qontract-reconcile 0.10.1rc696__py3-none-any.whl → 0.10.1rc702__py3-none-any.whl

reconcile/aws_account_manager/reconciler.py

@@ -0,0 +1,353 @@
+import logging
+from collections.abc import Iterable
+from textwrap import dedent
+from typing import Any, Protocol
+
+from reconcile.aws_account_manager.utils import state_key
+from reconcile.utils.aws_api_typed.api import AWSApi
+from reconcile.utils.aws_api_typed.iam import (
+    AWSAccessKey,
+)
+from reconcile.utils.aws_api_typed.organization import AwsOrganizationOU
+from reconcile.utils.aws_api_typed.service_quotas import (
+    AWSResourceAlreadyExistsException,
+)
+from reconcile.utils.aws_api_typed.support import SUPPORT_PLAN
+from reconcile.utils.state import AbortStateTransaction, State
+
+TASK_CREATE_ACCOUNT = "create-account"
+TASK_DESCRIBE_ACCOUNT = "describe-account"
+TASK_TAG_ACCOUNT = "tag-account"
+TASK_MOVE_ACCOUNT = "move-account"
+TASK_ACCOUNT_ALIAS = "account-alias"
+TASK_CREATE_IAM_USER = "create-iam-user"
+TASK_REQUEST_SERVICE_QUOTA = "request-service-quota"
+TASK_CHECK_SERVICE_QUOTA_STATUS = "check-service-quota-status"
+TASK_ENABLE_ENTERPRISE_SUPPORT = "enable-enterprise-support"
+TASK_CHECK_ENTERPRISE_SUPPORT_STATUS = "check-enterprise-support-status"
+
+
+class Quota(Protocol):
+    service_code: str
+    quota_code: str
+    value: float
+
+    def dict(self) -> dict[str, Any]: ...
+
+
+class AWSReconciler:
+    def __init__(self, state: State, dry_run: bool) -> None:
+        self.state = state
+        self.dry_run = dry_run
+
+    def _create_account(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        email: str,
+    ) -> str | None:
+        """Create the organization account and return the creation status ID."""
+        with self.state.transaction(state_key(name, TASK_CREATE_ACCOUNT)) as _state:
+            if _state.exists:
+                # account already exists, nothing to do
+                return _state.value
+
+            logging.info(f"Creating account {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            status = aws_api.organizations.create_account(email=email, name=name)
+            # store the status id for future reference
+            _state.value = status.id
+            return status.id
+
+    def _org_account_exists(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        create_account_request_id: str,
+    ) -> str | None:
+        """Check if the organization account exists and return its ID."""
+        with self.state.transaction(state_key(name, TASK_DESCRIBE_ACCOUNT)) as _state:
+            if _state.exists:
+                # account checked and exists, nothing to do
+                return _state.value
+
+            logging.info(f"Checking account creation status {name}")
+            status = aws_api.organizations.describe_create_account_status(
+                create_account_request_id=create_account_request_id
+            )
+            match status.state:
+                case "SUCCEEDED":
+                    _state.value = status.uid
+                    return status.uid
+                case "FAILED":
+                    raise RuntimeError(
+                        f"Account creation failed: {status.failure_reason}"
+                    )
+                case "IN_PROGRESS":
+                    raise AbortStateTransaction("Account creation still in progress")
+                case _:
+                    raise RuntimeError(
+                        f"Unexpected account creation status: {status.state}"
+                    )
+
+    def _tag_account(
+        self, aws_api: AWSApi, name: str, uid: str, tags: dict[str, str]
+    ) -> None:
+        with self.state.transaction(state_key(name, TASK_TAG_ACCOUNT)) as _state:
+            if _state.exists and _state.value == tags:
+                # account already tagged, nothing to do
+                return
+
+            logging.info(f"Tagging account {name}: {tags}")
+            _state.value = tags
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            if _state.exists:
+                aws_api.organizations.untag_resource(
+                    resource_id=uid, tag_keys=_state.value.keys()
+                )
+            aws_api.organizations.tag_resource(resource_id=uid, tags=tags)
+
+    def _get_destination_ou(
+        self, aws_api: AWSApi, destination_path: str
+    ) -> AwsOrganizationOU:
+        org_tree_root = aws_api.organizations.get_organizational_units_tree()
+        return org_tree_root.find(destination_path)
+
+    def _move_account(self, aws_api: AWSApi, name: str, uid: str, ou: str) -> None:
+        with self.state.transaction(state_key(name, TASK_MOVE_ACCOUNT)) as _state:
+            if _state.exists and _state.value == ou:
+                # account already moved, nothing to do
+                return
+
+            logging.info(f"Moving account {name} to {ou}")
+            destination = self._get_destination_ou(aws_api, destination_path=ou)
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            aws_api.organizations.move_account(
+                uid=uid,
+                destination_parent_id=destination.id,
+            )
+            _state.value = ou
+
+    def _set_account_alias(self, aws_api: AWSApi, name: str, alias: str | None) -> None:
+        """Create an account alias."""
+        new_alias = alias or name
+        with self.state.transaction(
+            state_key(name, TASK_ACCOUNT_ALIAS), new_alias
+        ) as _state:
+            if _state.exists and _state.value == new_alias:
+                return
+
+            logging.info(f"Set account alias '{new_alias}' for {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            aws_api.iam.set_account_alias(account_alias=new_alias)
+
+    def _request_quotas(
+        self, aws_api: AWSApi, name: str, quotas: Iterable[Quota]
+    ) -> list[str] | None:
+        """Request service quota changes."""
+        quotas_dict = [q.dict() for q in quotas]
+        with self.state.transaction(
+            state_key(name, TASK_REQUEST_SERVICE_QUOTA)
+        ) as _state:
+            if _state.exists and _state.value["last_applied_quotas"] == quotas_dict:
+                return _state.value["ids"]
+
+            # ATTENTION: reverting previously applied quotas or lowering them is not supported
+            new_quotas = []
+            for q in quotas:
+                quota = aws_api.service_quotas.get_service_quota(
+                    service_code=q.service_code, quota_code=q.quota_code
+                )
+                if quota.value > q.value:
+                    # a quota can be already higher than requested, because it was may set manually or enforced by the payer account
+                    logging.info(
+                        f"Cannot lower quota {q.service_code=}, {q.quota_code=}: {quota.value} -> {q.value}. Skipping."
+                    )
+                elif quota.value < q.value:
+                    quota.value = q.value
+                    new_quotas.append(quota)
+
+            for q in new_quotas:
+                logging.info(
+                    f"Setting quota for {name}: {q.service_name}/{q.quota_name} ({q.service_code}/{q.quota_code}) -> {q.value}"
+                )
+
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            ids = []
+            for new_quota in new_quotas:
+                try:
+                    req = aws_api.service_quotas.request_service_quota_change(
+                        service_code=new_quota.service_code,
+                        quota_code=new_quota.quota_code,
+                        desired_value=new_quota.value,
+                    )
+                except AWSResourceAlreadyExistsException:
+                    raise AbortStateTransaction(
+                        f"A quota increase for this {new_quota.service_code}/{new_quota.quota_code} already exists. Try it again later."
+                    )
+                ids.append(req.id)
+
+            _state.value = {"last_applied_quotas": quotas_dict, "ids": ids}
+            return ids
+
+    def _check_quota_change_requests(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        request_ids: Iterable[str],
+    ) -> None:
+        """Check the status of the quota change requests."""
+        with self.state.transaction(
+            state_key(name, TASK_CHECK_SERVICE_QUOTA_STATUS)
+        ) as _state:
+            if _state.exists and _state.value == request_ids:
+                return
+
+            logging.info(f"Checking quota change requests for {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            _state.value = []
+            for request_id in request_ids:
+                req = aws_api.service_quotas.get_requested_service_quota_change(
+                    request_id=request_id
+                )
+                match req.status:
+                    case "CASE_CLOSED" | "APPROVED":
+                        _state.value.append(request_id)
+                    case "DENIED" | "INVALID_REQUEST" | "NOT_APPROVED":
+                        raise RuntimeError(
+                            f"Quota change request {request_id} failed: {req.status}"
+                        )
+                    case _:
+                        # everything else is considered in progress
+                        pass
+
+    def _enable_enterprise_support(
+        self, aws_api: AWSApi, name: str, uid: str
+    ) -> str | None:
+        """Enable enterprise support for the account."""
+        with self.state.transaction(
+            state_key(name, TASK_ENABLE_ENTERPRISE_SUPPORT), ""
+        ) as _state:
+            if _state.exists:
+                return _state.value
+
+            if aws_api.support.get_support_level() == SUPPORT_PLAN.ENTERPRISE:
+                if self.dry_run:
+                    raise AbortStateTransaction("Dry run")
+                return None
+
+            logging.info(f"Enabling enterprise support for {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            case_id = aws_api.support.create_case(
+                subject=f"Add account {uid} to Enterprise Support",
+                message=dedent(f"""
+                    Hello AWS,
+
+                    Please enable Enterprise Support on AWS account {uid} and resolve this support case.
+
+                    Thanks.
+
+                    [rh-internal-account-name: {name}]
+                """),
+            )
+            _state.value = case_id
+            return case_id
+
+    def _check_enterprise_support_status(self, aws_api: AWSApi, case_id: str) -> None:
+        """Check the status of the enterprise support case."""
+        with self.state.transaction(
+            state_key(case_id, TASK_CHECK_ENTERPRISE_SUPPORT_STATUS), True
+        ) as _state:
+            if _state.exists:
+                return
+
+            logging.info(f"Checking enterprise support case {case_id}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            case = aws_api.support.describe_case(case_id=case_id)
+            if case.status == "resolved":
+                return
+
+            logging.info(
+                f"Enterprise support case {case_id} is still open. Current status: {case.status}"
+            )
+            raise AbortStateTransaction("Enterprise support case still open")
+
+    #
+    # Public methods
+    #
+    def create_organization_account(
+        self, aws_api: AWSApi, name: str, email: str
+    ) -> str | None:
+        """Create an organization account and return the creation status ID."""
+        if create_account_request_id := self._create_account(aws_api, name, email):
+            if uid := self._org_account_exists(
+                aws_api, name, create_account_request_id
+            ):
+                return uid
+        return None
+
+    def create_iam_user(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        user_name: str,
+        user_policy_arn: str,
+    ) -> AWSAccessKey | None:
+        """Create an IAM user and return its access key."""
+        with self.state.transaction(
+            state_key(name, TASK_CREATE_IAM_USER), user_name
+        ) as _state:
+            if _state.exists and _state.value == user_name:
+                return None
+
+            logging.info(f"Creating IAM user '{user_name}' for {name}")
+            if self.dry_run:
+                raise AbortStateTransaction("Dry run")
+
+            aws_api.iam.create_user(user_name=user_name)
+            aws_api.iam.attach_user_policy(
+                user_name=user_name,
+                policy_arn=user_policy_arn,
+            )
+            return aws_api.iam.create_access_key(user_name=user_name)
+
+    def reconcile_organization_account(
+        self,
+        aws_api: AWSApi,
+        name: str,
+        uid: str,
+        ou: str,
+        tags: dict[str, str],
+        enterprise_support: bool,
+    ) -> None:
+        """Reconcile the AWS account on the organization level."""
+        self._tag_account(aws_api, name, uid, tags)
+        self._move_account(aws_api, name, uid, ou)
+        if enterprise_support and (
+            case_id := self._enable_enterprise_support(aws_api, name, uid)
+        ):
+            self._check_enterprise_support_status(aws_api, case_id)
+
+    def reconcile_account(
+        self, aws_api: AWSApi, name: str, alias: str | None, quotas: Iterable[Quota]
+    ) -> None:
+        """Reconcile/update the AWS account. Return the initial user access key if a new user was created."""
+        self._set_account_alias(aws_api, name, alias)
+        if request_ids := self._request_quotas(aws_api, name, quotas):
+            self._check_quota_change_requests(aws_api, name, request_ids)

reconcile/aws_account_manager/utils.py

@@ -0,0 +1,38 @@
+from collections import Counter
+
+from reconcile.gql_definitions.aws_account_manager.aws_accounts import (
+    AWSAccountV1,
+)
+
+
+def validate(account: AWSAccountV1) -> bool:
+    """Validate the account configurations."""
+    # check referenced quotas don't overlap
+    quotas = Counter([
+        (quota.service_code, quota.quota_code)
+        for quota_limit in account.quota_limits or []
+        for quota in quota_limit.quotas or []
+    ])
+    errors = [
+        ValueError(
+            f"Quota service_code={service_code}, quota_code={quota_code} is referenced multiple times in account {account.name}"
+        )
+        for (service_code, quota_code), cnt in quotas.items()
+        if cnt > 1
+    ]
+    if errors:
+        raise ExceptionGroup("Multiple quotas are referenced in the account", errors)
+
+    if account.organization_accounts or account.account_requests:
+        # it's payer account
+        if not account.premium_support:
+            raise ValueError(
+                f"Premium support is required for payer account {account.name}"
+            )
+
+    return True
+
+
+def state_key(account: str, task: str) -> str:
+    """Compute a state key based on the organization account and the task name."""
+    return f"task.{account}.{task}"

reconcile/aws_saml_idp/integration.py

@@ -76,6 +76,8 @@ class AwsSamlIdpIntegration(QontractReconcileIntegration[AwsSamlIdpIntegrationPa
             for account in data.accounts or []
             if integration_is_enabled(self.name, account)
             and (not account_name or account.name == account_name)
+            # a new account does not have a terraform state yet, ignore it until terraform-init does its job
+            and account.terraform_state
         ]
 
     def build_saml_idp_config(

reconcile/aws_version_sync/integration.py

@@ -19,6 +19,7 @@ from reconcile.aws_version_sync.merge_request_manager.merge_request import (
 )
 from reconcile.aws_version_sync.merge_request_manager.merge_request_manager import (
     MergeRequestManager,
+    MrData,
 )
 from reconcile.aws_version_sync.utils import (
     get_values,
@@ -337,8 +338,6 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
         external_resources_aws: AwsExternalResources,
         external_resources_app_interface: AppInterfaceExternalResources,
     ) -> None:
-        # initialize the merge request manager
-        merge_request_manager.fetch_avs_managed_open_merge_requests()
         # housekeeping: close old/bad MRs
         merge_request_manager.housekeeping()
         diff = diff_iterables(
@@ -361,15 +360,17 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
             # make mypy happy
             assert app_interface_resource.namespace_file
             assert app_interface_resource.provisioner.path
-            merge_request_manager.create_avs_merge_request(
-                namespace_file=app_interface_resource.namespace_file,
-                provider=app_interface_resource.provider,
-                provisioner_ref=app_interface_resource.provisioner.path,
-                provisioner_uid=app_interface_resource.provisioner.uid,
-                resource_provider=app_interface_resource.resource_provider,
-                resource_identifier=app_interface_resource.resource_identifier,
-                resource_engine=app_interface_resource.resource_engine,
-                resource_engine_version=aws_resource.resource_engine_version_string,
+            merge_request_manager.create_merge_request(
+                MrData(
+                    namespace_file=app_interface_resource.namespace_file,
+                    provider=app_interface_resource.provider,
+                    provisioner_ref=app_interface_resource.provisioner.path,
+                    provisioner_uid=app_interface_resource.provisioner.uid,
+                    resource_provider=app_interface_resource.resource_provider,
+                    resource_identifier=app_interface_resource.resource_identifier,
+                    resource_engine=app_interface_resource.resource_engine,
+                    resource_engine_version=aws_resource.resource_engine_version_string,
+                )
             )
 
     @defer

reconcile/aws_version_sync/merge_request_manager/merge_request_manager.py

@@ -1,9 +1,7 @@
 import logging
-import re
-from dataclasses import dataclass
 
 from gitlab.exceptions import GitlabGetError
-from gitlab.v4.objects import ProjectMergeRequest
+from pydantic import BaseModel
 
 from reconcile.aws_version_sync.merge_request_manager.merge_request import (
     AVS_LABEL,
@@ -12,18 +10,14 @@ from reconcile.aws_version_sync.merge_request_manager.merge_request import (
     Renderer,
 )
 from reconcile.utils.gitlab_api import GitLabApi
-from reconcile.utils.merge_request_manager.parser import ParserError, ParserVersionError
+from reconcile.utils.merge_request_manager.merge_request_manager import (
+    MergeRequestManagerBase,
+)
 from reconcile.utils.mr.base import MergeRequestBase
 from reconcile.utils.mr.labels import AUTO_MERGE
 from reconcile.utils.vcs import VCS
 
 
-@dataclass
-class OpenMergeRequest:
-    raw: ProjectMergeRequest
-    avs_info: AVSInfo
-
-
 class AVSMR(MergeRequestBase):
     name = "AVS"
 
@@ -54,7 +48,18 @@ class AVSMR(MergeRequestBase):
         )
 
 
-class MergeRequestManager:
+class MrData(BaseModel):
+    namespace_file: str
+    provider: str
+    provisioner_ref: str
+    provisioner_uid: str
+    resource_provider: str
+    resource_identifier: str
+    resource_engine: str
+    resource_engine_version: str
+
+
+class MergeRequestManager(MergeRequestManagerBase[AVSInfo]):
     """
     Manager for AVS merge requests. This class
     is responsible for housekeeping (closing old/bad MRs) and
@@ -68,113 +73,35 @@ class MergeRequestManager:
     def __init__(
         self, vcs: VCS, renderer: Renderer, parser: Parser, auto_merge_enabled: bool
     ):
-        self._vcs = vcs
-        self._open_mrs: list[OpenMergeRequest] = []
-        self._open_mrs_with_problems: list[OpenMergeRequest] = []
-        self._open_raw_mrs: list[ProjectMergeRequest] = []
+        super().__init__(vcs, parser, AVS_LABEL)
         self._renderer = renderer
-        self._parser = parser
         self._auto_merge_enabled = auto_merge_enabled
 
-    def _apply_regex(self, pattern: re.Pattern, promotion_data: str) -> str:
-        matches = pattern.search(promotion_data)
-        if not matches:
-            return ""
-        groups = matches.groups()
-        if len(groups) != 1:
-            return ""
-        return groups[0]
-
-    def fetch_avs_managed_open_merge_requests(self) -> None:
-        all_open_mrs = self._vcs.get_open_app_interface_merge_requests()
-        self._open_raw_mrs = [mr for mr in all_open_mrs if AVS_LABEL in mr.labels]
-
-    def housekeeping(self) -> None:
-        """
-        Close bad MRs:
-        - bad description format
-        - old AVS version
-        - merge conflict
-
-        --> if we bump the AVS version, we automatically close
-        old open MRs and replace them with new ones.
-        """
-        for mr in self._open_raw_mrs:
-            attrs = mr.attributes
-            desc = attrs.get("description")
-            has_conflicts = attrs.get("has_conflicts", False)
-            if has_conflicts:
-                logging.info(
-                    "Merge-conflict detected. Closing %s",
-                    mr.attributes.get("web_url", "NO_WEBURL"),
-                )
-                self._vcs.close_app_interface_mr(
-                    mr, "Closing this MR because of a merge-conflict."
-                )
-                continue
-            try:
-                avs_info = self._parser.parse(description=desc)
-            except ParserVersionError:
-                logging.info(
-                    "Old MR version detected! Closing %s",
-                    mr.attributes.get("web_url", "NO_WEBURL"),
-                )
-                self._vcs.close_app_interface_mr(
-                    mr, "Closing this MR because it has an outdated AVS version"
-                )
-                continue
-            except ParserError:
-                logging.info(
-                    "Bad MR description format. Closing %s",
-                    mr.attributes.get("web_url", "NO_WEBURL"),
-                )
-                self._vcs.close_app_interface_mr(
-                    mr, "Closing this MR because of bad description format."
-                )
-                continue
-
-            self._open_mrs.append(OpenMergeRequest(raw=mr, avs_info=avs_info))
-
-    def _merge_request_already_exists(
+    def create_merge_request(
         self,
-        provider: str,
-        account_id: str,
-        resource_provider: str,
-        resource_identifier: str,
-        resource_engine: str,
-    ) -> OpenMergeRequest | None:
-        for mr in self._open_mrs:
-            if (
-                mr.avs_info.provider == provider
-                and mr.avs_info.account_id == account_id
-                and mr.avs_info.resource_provider == resource_provider
-                and mr.avs_info.resource_identifier == resource_identifier
-                and mr.avs_info.resource_engine == resource_engine
-            ):
-                return mr
-
-        return None
-
-    def create_avs_merge_request(
-        self,
-        namespace_file: str,
-        provider: str,
-        provisioner_ref: str,
-        provisioner_uid: str,
-        resource_provider: str,
-        resource_identifier: str,
-        resource_engine: str,
-        resource_engine_version: str,
+        data: MrData,
     ) -> None:
         """Open new MR (if not already present) for an external resource and close any outdated before."""
-        if mr := self._merge_request_already_exists(
-            provider=provider,
-            account_id=provisioner_uid,
-            resource_provider=resource_provider,
-            resource_identifier=resource_identifier,
-            resource_engine=resource_engine,
-        ):
-            if mr.avs_info.resource_engine_version == resource_engine_version:
+        if not self._housekeeping_ran:
+            self.housekeeping()
+
+        namespace_file = data.namespace_file
+        provider = data.provider
+        provisioner_ref = data.provisioner_ref
+        provisioner_uid = data.provisioner_uid
+        resource_provider = data.resource_provider
+        resource_identifier = data.resource_identifier
+        resource_engine = data.resource_engine
+        resource_engine_version = data.resource_engine_version
+
+        if mr := self._merge_request_already_exists({
+            "provider": provider,
+            "account_id": provisioner_uid,
+            "resource_provider": resource_provider,
+            "resource_identifier": resource_identifier,
+            "resource_engine": resource_engine,
+        }):
+            if mr.mr_info.resource_engine_version == resource_engine_version:
                 # an MR for this external resource already exists
                 return None
             logging.info(