qontract-reconcile 0.10.1rc701__py3-none-any.whl → 0.10.1rc703__py3-none-any.whl

reconcile/aws_account_manager/reconciler.py

@@ -0,0 +1,353 @@
+import logging
+from collections.abc import Iterable
+from textwrap import dedent
+from typing import Any, Protocol
+
+from reconcile.aws_account_manager.utils import state_key
+from reconcile.utils.aws_api_typed.api import AWSApi
+from reconcile.utils.aws_api_typed.iam import (
+    AWSAccessKey,
+)
+from reconcile.utils.aws_api_typed.organization import AwsOrganizationOU
+from reconcile.utils.aws_api_typed.service_quotas import (
+    AWSResourceAlreadyExistsException,
+)
+from reconcile.utils.aws_api_typed.support import SUPPORT_PLAN
+from reconcile.utils.state import AbortStateTransaction, State
+
+TASK_CREATE_ACCOUNT = "create-account"
+TASK_DESCRIBE_ACCOUNT = "describe-account"
+TASK_TAG_ACCOUNT = "tag-account"
+TASK_MOVE_ACCOUNT = "move-account"
+TASK_ACCOUNT_ALIAS = "account-alias"
+TASK_CREATE_IAM_USER = "create-iam-user"
+TASK_REQUEST_SERVICE_QUOTA = "request-service-quota"
+TASK_CHECK_SERVICE_QUOTA_STATUS = "check-service-quota-status"
+TASK_ENABLE_ENTERPRISE_SUPPORT = "enable-enterprise-support"
+TASK_CHECK_ENTERPRISE_SUPPORT_STATUS = "check-enterprise-support-status"
+
+
+class Quota(Protocol):
+    service_code: str
+    quota_code: str
+    value: float
+
+    def dict(self) -> dict[str, Any]: ...
+
+
+class AWSReconciler:
+    def __init__(self, state: State, dry_run: bool) -> None:
+        self.state = state
+        self.dry_run = dry_run
+
+    def _create_account(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        email: str,
+    ) -> str | None:
+        """Create the organization account and return the creation status ID."""
+        with self.state.transaction(state_key(name, TASK_CREATE_ACCOUNT)) as _state:
+            if _state.exists:
+                # account already exists, nothing to do
+                return _state.value
+
+            logging.info(f"Creating account {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            status = aws_api.organizations.create_account(email=email, name=name)
+            # store the status id for future reference
+            _state.value = status.id
+            return status.id
+
+    def _org_account_exists(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        create_account_request_id: str,
+    ) -> str | None:
+        """Check if the organization account exists and return its ID."""
+        with self.state.transaction(state_key(name, TASK_DESCRIBE_ACCOUNT)) as _state:
+            if _state.exists:
+                # account checked and exists, nothing to do
+                return _state.value
+
+            logging.info(f"Checking account creation status {name}")
+            status = aws_api.organizations.describe_create_account_status(
+                create_account_request_id=create_account_request_id
+            )
+            match status.state:
+                case "SUCCEEDED":
+                    _state.value = status.uid
+                    return status.uid
+                case "FAILED":
+                    raise RuntimeError(
+                        f"Account creation failed: {status.failure_reason}"
+                    )
+                case "IN_PROGRESS":
+                    raise AbortStateTransaction("Account creation still in progress")
+                case _:
+                    raise RuntimeError(
+                        f"Unexpected account creation status: {status.state}"
+                    )
+
+    def _tag_account(
+        self, aws_api: AWSApi, name: str, uid: str, tags: dict[str, str]
+    ) -> None:
+        with self.state.transaction(state_key(name, TASK_TAG_ACCOUNT)) as _state:
+            if _state.exists and _state.value == tags:
+                # account already tagged, nothing to do
+                return
+
+            logging.info(f"Tagging account {name}: {tags}")
+            _state.value = tags
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            if _state.exists:
+                aws_api.organizations.untag_resource(
+                    resource_id=uid, tag_keys=_state.value.keys()
+                )
+            aws_api.organizations.tag_resource(resource_id=uid, tags=tags)
+
+    def _get_destination_ou(
+        self, aws_api: AWSApi, destination_path: str
+    ) -> AwsOrganizationOU:
+        org_tree_root = aws_api.organizations.get_organizational_units_tree()
+        return org_tree_root.find(destination_path)
+
+    def _move_account(self, aws_api: AWSApi, name: str, uid: str, ou: str) -> None:
+        with self.state.transaction(state_key(name, TASK_MOVE_ACCOUNT)) as _state:
+            if _state.exists and _state.value == ou:
+                # account already moved, nothing to do
+                return
+
+            logging.info(f"Moving account {name} to {ou}")
+            destination = self._get_destination_ou(aws_api, destination_path=ou)
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            aws_api.organizations.move_account(
+                uid=uid,
+                destination_parent_id=destination.id,
+            )
+            _state.value = ou
+
+    def _set_account_alias(self, aws_api: AWSApi, name: str, alias: str | None) -> None:
+        """Create an account alias."""
+        new_alias = alias or name
+        with self.state.transaction(
+            state_key(name, TASK_ACCOUNT_ALIAS), new_alias
+        ) as _state:
+            if _state.exists and _state.value == new_alias:
+                return
+
+            logging.info(f"Set account alias '{new_alias}' for {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            aws_api.iam.set_account_alias(account_alias=new_alias)
+
+    def _request_quotas(
+        self, aws_api: AWSApi, name: str, quotas: Iterable[Quota]
+    ) -> list[str] | None:
+        """Request service quota changes."""
+        quotas_dict = [q.dict() for q in quotas]
+        with self.state.transaction(
+            state_key(name, TASK_REQUEST_SERVICE_QUOTA)
+        ) as _state:
+            if _state.exists and _state.value["last_applied_quotas"] == quotas_dict:
+                return _state.value["ids"]
+
+            # ATTENTION: reverting previously applied quotas or lowering them is not supported
+            new_quotas = []
+            for q in quotas:
+                quota = aws_api.service_quotas.get_service_quota(
+                    service_code=q.service_code, quota_code=q.quota_code
+                )
+                if quota.value > q.value:
+                    # a quota can be already higher than requested, because it was may set manually or enforced by the payer account
+                    logging.info(
+                        f"Cannot lower quota {q.service_code=}, {q.quota_code=}: {quota.value} -> {q.value}. Skipping."
+                    )
+                elif quota.value < q.value:
+                    quota.value = q.value
+                    new_quotas.append(quota)
+
+            for q in new_quotas:
+                logging.info(
+                    f"Setting quota for {name}: {q.service_name}/{q.quota_name} ({q.service_code}/{q.quota_code}) -> {q.value}"
+                )
+
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            ids = []
+            for new_quota in new_quotas:
+                try:
+                    req = aws_api.service_quotas.request_service_quota_change(
+                        service_code=new_quota.service_code,
+                        quota_code=new_quota.quota_code,
+                        desired_value=new_quota.value,
+                    )
+                except AWSResourceAlreadyExistsException:
+                    raise AbortStateTransaction(
+                        f"A quota increase for this {new_quota.service_code}/{new_quota.quota_code} already exists. Try it again later."
+                    )
+                ids.append(req.id)
+
+            _state.value = {"last_applied_quotas": quotas_dict, "ids": ids}
+            return ids
+
+    def _check_quota_change_requests(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        request_ids: Iterable[str],
+    ) -> None:
+        """Check the status of the quota change requests."""
+        with self.state.transaction(
+            state_key(name, TASK_CHECK_SERVICE_QUOTA_STATUS)
+        ) as _state:
+            if _state.exists and _state.value == request_ids:
+                return
+
+            logging.info(f"Checking quota change requests for {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            _state.value = []
+            for request_id in request_ids:
+                req = aws_api.service_quotas.get_requested_service_quota_change(
+                    request_id=request_id
+                )
+                match req.status:
+                    case "CASE_CLOSED" | "APPROVED":
+                        _state.value.append(request_id)
+                    case "DENIED" | "INVALID_REQUEST" | "NOT_APPROVED":
+                        raise RuntimeError(
+                            f"Quota change request {request_id} failed: {req.status}"
+                        )
+                    case _:
+                        # everything else is considered in progress
+                        pass
+
+    def _enable_enterprise_support(
+        self, aws_api: AWSApi, name: str, uid: str
+    ) -> str | None:
+        """Enable enterprise support for the account."""
+        with self.state.transaction(
+            state_key(name, TASK_ENABLE_ENTERPRISE_SUPPORT), ""
+        ) as _state:
+            if _state.exists:
+                return _state.value
+
+            if aws_api.support.get_support_level() == SUPPORT_PLAN.ENTERPRISE:
+                if self.dry_run:
+                    raise AbortStateTransaction("Dry run")
+                return None
+
+            logging.info(f"Enabling enterprise support for {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            case_id = aws_api.support.create_case(
+                subject=f"Add account {uid} to Enterprise Support",
+                message=dedent(f"""
+                    Hello AWS,
+
+                    Please enable Enterprise Support on AWS account {uid} and resolve this support case.
+
+                    Thanks.
+
+                    [rh-internal-account-name: {name}]
+                """),
+            )
+            _state.value = case_id
+            return case_id
+
+    def _check_enterprise_support_status(self, aws_api: AWSApi, case_id: str) -> None:
+        """Check the status of the enterprise support case."""
+        with self.state.transaction(
+            state_key(case_id, TASK_CHECK_ENTERPRISE_SUPPORT_STATUS), True
+        ) as _state:
+            if _state.exists:
+                return
+
+            logging.info(f"Checking enterprise support case {case_id}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            case = aws_api.support.describe_case(case_id=case_id)
+            if case.status == "resolved":
+                return
+
+            logging.info(
+                f"Enterprise support case {case_id} is still open. Current status: {case.status}"
+            )
+            raise AbortStateTransaction("Enterprise support case still open")
+
+    #
+    # Public methods
+    #
+    def create_organization_account(
+        self, aws_api: AWSApi, name: str, email: str
+    ) -> str | None:
+        """Create an organization account and return the creation status ID."""
+        if create_account_request_id := self._create_account(aws_api, name, email):
+            if uid := self._org_account_exists(
+                aws_api, name, create_account_request_id
+            ):
+                return uid
+        return None
+
+    def create_iam_user(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        user_name: str,
+        user_policy_arn: str,
+    ) -> AWSAccessKey | None:
+        """Create an IAM user and return its access key."""
+        with self.state.transaction(
+            state_key(name, TASK_CREATE_IAM_USER), user_name
+        ) as _state:
+            if _state.exists and _state.value == user_name:
+                return None
+
+            logging.info(f"Creating IAM user '{user_name}' for {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            aws_api.iam.create_user(user_name=user_name)
+            aws_api.iam.attach_user_policy(
+                user_name=user_name,
+                policy_arn=user_policy_arn,
+            )
+            return aws_api.iam.create_access_key(user_name=user_name)
+
+    def reconcile_organization_account(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        uid: str,
+        ou: str,
+        tags: dict[str, str],
+        enterprise_support: bool,
+    ) -> None:
+        """Reconcile the AWS account on the organization level."""
+        self._tag_account(aws_api, name, uid, tags)
+        self._move_account(aws_api, name, uid, ou)
+        if enterprise_support and (
+            case_id := self._enable_enterprise_support(aws_api, name, uid)
+        ):
+            self._check_enterprise_support_status(aws_api, case_id)
+
+    def reconcile_account(
+        self, aws_api: AWSApi, name: str, alias: str | None, quotas: Iterable[Quota]
+    ) -> None:
+        """Reconcile/update the AWS account. Return the initial user access key if a new user was created."""
+        self._set_account_alias(aws_api, name, alias)
+        if request_ids := self._request_quotas(aws_api, name, quotas):
+            self._check_quota_change_requests(aws_api, name, request_ids)

reconcile/aws_account_manager/utils.py

@@ -0,0 +1,38 @@
+from collections import Counter
+
+from reconcile.gql_definitions.aws_account_manager.aws_accounts import (
+    AWSAccountV1,
+)
+
+
+def validate(account: AWSAccountV1) -> bool:
+    """Validate the account configurations."""
+    # check referenced quotas don't overlap
+    quotas = Counter([
+        (quota.service_code, quota.quota_code)
+        for quota_limit in account.quota_limits or []
+        for quota in quota_limit.quotas or []
+    ])
+    errors = [
+        ValueError(
+            f"Quota service_code={service_code}, quota_code={quota_code} is referenced multiple times in account {account.name}"
+        )
+        for (service_code, quota_code), cnt in quotas.items()
+        if cnt > 1
+    ]
+    if errors:
+        raise ExceptionGroup("Multiple quotas are referenced in the account", errors)
+
+    if account.organization_accounts or account.account_requests:
+        # it's payer account
+        if not account.premium_support:
+            raise ValueError(
+                f"Premium support is required for payer account {account.name}"
+            )
+
+    return True
+
+
+def state_key(account: str, task: str) -> str:
+    """Compute a state key based on the organization account and the task name."""
+    return f"task.{account}.{task}"

reconcile/cli.py

@@ -951,6 +951,85 @@ def aws_saml_roles(
     )
 
 
+@integration.command(short_help="Create and manage AWS accounts.")
+@account_name
+@click.option(
+    "--flavor",
+    help="Flavor of the AWS account manager.",
+    required=True,
+    default="app-interface-commercial",
+)
+@click.option(
+    "--tag",
+    "-t",
+    type=(str, str),
+    multiple=True,
+    default=[("managed-by", "app-interface")],
+)
+@click.option(
+    "--initial-user-name",
+    help="The name of the initial user to be created in the account.",
+    required=True,
+    default="terraform",
+)
+@click.option(
+    "--initial-user-policy-arn",
+    help="The ARN of the policy that is attached to the initial user.",
+    required=True,
+    default="arn:aws:iam::aws:policy/AdministratorAccess",
+)
+@click.option(
+    "--initial-user-secret-vault-path",
+    help="The path in Vault to store the initial user secret. Python format string with access to 'account_name' attribute.",
+    required=True,
+    default="app-sre/creds/terraform/{account_name}/config",
+)
+@click.option(
+    "--account-tmpl-resource",
+    help="Resource name of the account template-collection template in the app-interface.",
+    required=True,
+    default="/aws-account-manager/account-tmpl.yml",
+)
+@click.option(
+    "--template-collection-root-path",
+    help="File path to the root directory to store new account template-collections.",
+    required=True,
+    default="data/templating/collections/aws-account",
+)
+@click.pass_context
+def aws_account_manager(
+    ctx,
+    account_name,
+    flavor,
+    tag,
+    initial_user_name,
+    initial_user_policy_arn,
+    initial_user_secret_vault_path,
+    account_tmpl_resource,
+    template_collection_root_path,
+):
+    from reconcile.aws_account_manager.integration import (
+        AwsAccountMgmtIntegration,
+        AwsAccountMgmtIntegrationParams,
+    )
+
+    run_class_integration(
+        integration=AwsAccountMgmtIntegration(
+            AwsAccountMgmtIntegrationParams(
+                account_name=account_name,
+                flavor=flavor,
+                default_tags=dict(tag),
+                initial_user_name=initial_user_name,
+                initial_user_policy_arn=initial_user_policy_arn,
+                initial_user_secret_vault_path=initial_user_secret_vault_path,
+                account_tmpl_resource=account_tmpl_resource,
+                template_collection_root_path=template_collection_root_path,
+            )
+        ),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Manage Jenkins roles association via REST API.")
 @click.pass_context
 def jenkins_roles(ctx):

reconcile/gql_definitions/aws_account_manager/aws_accounts.py

@@ -0,0 +1,163 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.aws_account_managed import AWSAccountManaged
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment AWSAccountManaged on AWSAccount_v1 {
+  name
+  uid
+  alias
+  premiumSupport
+  organization {
+    ou
+    tags
+  }
+  quotaLimits {
+    name
+    quotas {
+      serviceCode
+      quotaCode
+      value
+    }
+  }
+}
+
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query AWSAccountManagerAccounts {
+  accounts: awsaccounts_v1 {
+    ... AWSAccountManaged
+    resourcesDefaultRegion
+    automationToken {
+      ...VaultSecret
+    }
+    disable {
+      integrations
+    }
+    automationRole {
+      awsAccountManager
+    }
+    # for the requests via "payer account"
+    account_requests {
+      path
+      name
+      description
+      accountOwner {
+        name
+        email
+      }
+      organization {
+        ou
+        tags
+        payerAccount {
+          path
+        }
+      }
+      quotaLimits {
+        path
+      }
+    }
+    organization_accounts {
+      ... AWSAccountManaged
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class DisableClusterAutomationsV1(ConfiguredBaseModel):
+    integrations: Optional[list[str]] = Field(..., alias="integrations")
+
+
+class AWSAutomationRoleV1(ConfiguredBaseModel):
+    aws_account_manager: Optional[str] = Field(..., alias="awsAccountManager")
+
+
+class OwnerV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    email: str = Field(..., alias="email")
+
+
+class AWSOrganizationV1_AWSAccountV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class AWSOrganizationV1(ConfiguredBaseModel):
+    ou: str = Field(..., alias="ou")
+    tags: Json = Field(..., alias="tags")
+    payer_account: AWSOrganizationV1_AWSAccountV1 = Field(..., alias="payerAccount")
+
+
+class AWSQuotaLimitsV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class AWSAccountRequestV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+    name: str = Field(..., alias="name")
+    description: str = Field(..., alias="description")
+    account_owner: OwnerV1 = Field(..., alias="accountOwner")
+    organization: AWSOrganizationV1 = Field(..., alias="organization")
+    quota_limits: Optional[list[AWSQuotaLimitsV1]] = Field(..., alias="quotaLimits")
+
+
+class AWSAccountV1(AWSAccountManaged):
+    resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
+    automation_token: VaultSecret = Field(..., alias="automationToken")
+    disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
+    automation_role: Optional[AWSAutomationRoleV1] = Field(..., alias="automationRole")
+    account_requests: Optional[list[AWSAccountRequestV1]] = Field(..., alias="account_requests")
+    organization_accounts: Optional[list[AWSAccountManaged]] = Field(..., alias="organization_accounts")
+
+
+class AWSAccountManagerAccountsQueryData(ConfiguredBaseModel):
+    accounts: Optional[list[AWSAccountV1]] = Field(..., alias="accounts")
+
+
+def query(query_func: Callable, **kwargs: Any) -> AWSAccountManagerAccountsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        AWSAccountManagerAccountsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return AWSAccountManagerAccountsQueryData(**raw_data)

reconcile/gql_definitions/fragments/aws_account_managed.py

@@ -0,0 +1,49 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class AWSOrganizationV1(ConfiguredBaseModel):
+    ou: str = Field(..., alias="ou")
+    tags: Json = Field(..., alias="tags")
+
+
+class AWSQuotaV1(ConfiguredBaseModel):
+    service_code: str = Field(..., alias="serviceCode")
+    quota_code: str = Field(..., alias="quotaCode")
+    value: float = Field(..., alias="value")
+
+
+class AWSQuotaLimitsV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    quotas: list[AWSQuotaV1] = Field(..., alias="quotas")
+
+
+class AWSAccountManaged(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    uid: str = Field(..., alias="uid")
+    alias: Optional[str] = Field(..., alias="alias")
+    premium_support: bool = Field(..., alias="premiumSupport")
+    organization: Optional[AWSOrganizationV1] = Field(..., alias="organization")
+    quota_limits: Optional[list[AWSQuotaLimitsV1]] = Field(..., alias="quotaLimits")

reconcile/ocm_internal_notifications/integration.py

@@ -63,7 +63,7 @@ class OcmInternalNotifications(QontractReconcileIntegration[NoParams]):
         for env in environments:
             ocm = init_ocm_base_client(env, self.secret_reader)
 
-            if not (env.labels and env.labels["internal_notifications"]):
+            if not (env.labels and env.labels.get("internal_notifications")):
                 logging.info(
                     f"skipping environment {env.name} due to no internal_notifications label"
                 )