qontract-reconcile 0.10.1rc696__py3-none-any.whl → 0.10.1rc702__py3-none-any.whl

reconcile/cli.py

@@ -951,6 +951,85 @@ def aws_saml_roles(
     )
 
 
+@integration.command(short_help="Create and manage AWS accounts.")
+@account_name
+@click.option(
+    "--flavor",
+    help="Flavor of the AWS account manager.",
+    required=True,
+    default="app-interface-commercial",
+)
+@click.option(
+    "--tag",
+    "-t",
+    type=(str, str),
+    multiple=True,
+    default=[("managed-by", "app-interface")],
+)
+@click.option(
+    "--initial-user-name",
+    help="The name of the initial user to be created in the account.",
+    required=True,
+    default="terraform",
+)
+@click.option(
+    "--initial-user-policy-arn",
+    help="The ARN of the policy that is attached to the initial user.",
+    required=True,
+    default="arn:aws:iam::aws:policy/AdministratorAccess",
+)
+@click.option(
+    "--initial-user-secret-vault-path",
+    help="The path in Vault to store the initial user secret. Python format string with access to 'account_name' attribute.",
+    required=True,
+    default="app-sre/creds/terraform/{account_name}/config",
+)
+@click.option(
+    "--account-tmpl-resource",
+    help="Resource name of the account template-collection template in the app-interface.",
+    required=True,
+    default="/aws-account-manager/account-tmpl.yml",
+)
+@click.option(
+    "--template-collection-root-path",
+    help="File path to the root directory to store new account template-collections.",
+    required=True,
+    default="data/templating/collections/aws-account",
+)
+@click.pass_context
+def aws_account_manager(
+    ctx,
+    account_name,
+    flavor,
+    tag,
+    initial_user_name,
+    initial_user_policy_arn,
+    initial_user_secret_vault_path,
+    account_tmpl_resource,
+    template_collection_root_path,
+):
+    from reconcile.aws_account_manager.integration import (
+        AwsAccountMgmtIntegration,
+        AwsAccountMgmtIntegrationParams,
+    )
+
+    run_class_integration(
+        integration=AwsAccountMgmtIntegration(
+            AwsAccountMgmtIntegrationParams(
+                account_name=account_name,
+                flavor=flavor,
+                default_tags=dict(tag),
+                initial_user_name=initial_user_name,
+                initial_user_policy_arn=initial_user_policy_arn,
+                initial_user_secret_vault_path=initial_user_secret_vault_path,
+                account_tmpl_resource=account_tmpl_resource,
+                template_collection_root_path=template_collection_root_path,
+            )
+        ),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Manage Jenkins roles association via REST API.")
 @click.pass_context
 def jenkins_roles(ctx):

reconcile/gql_definitions/aws_account_manager/aws_accounts.py

@@ -0,0 +1,163 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.aws_account_managed import AWSAccountManaged
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment AWSAccountManaged on AWSAccount_v1 {
+  name
+  uid
+  alias
+  premiumSupport
+  organization {
+    ou
+    tags
+  }
+  quotaLimits {
+    name
+    quotas {
+      serviceCode
+      quotaCode
+      value
+    }
+  }
+}
+
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query AWSAccountManagerAccounts {
+  accounts: awsaccounts_v1 {
+    ... AWSAccountManaged
+    resourcesDefaultRegion
+    automationToken {
+      ...VaultSecret
+    }
+    disable {
+      integrations
+    }
+    automationRole {
+      awsAccountManager
+    }
+    # for the requests via "payer account"
+    account_requests {
+      path
+      name
+      description
+      accountOwner {
+        name
+        email
+      }
+      organization {
+        ou
+        tags
+        payerAccount {
+          path
+        }
+      }
+      quotaLimits {
+        path
+      }
+    }
+    organization_accounts {
+      ... AWSAccountManaged
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class DisableClusterAutomationsV1(ConfiguredBaseModel):
+    integrations: Optional[list[str]] = Field(..., alias="integrations")
+
+
+class AWSAutomationRoleV1(ConfiguredBaseModel):
+    aws_account_manager: Optional[str] = Field(..., alias="awsAccountManager")
+
+
+class OwnerV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    email: str = Field(..., alias="email")
+
+
+class AWSOrganizationV1_AWSAccountV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class AWSOrganizationV1(ConfiguredBaseModel):
+    ou: str = Field(..., alias="ou")
+    tags: Json = Field(..., alias="tags")
+    payer_account: AWSOrganizationV1_AWSAccountV1 = Field(..., alias="payerAccount")
+
+
+class AWSQuotaLimitsV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+
+
+class AWSAccountRequestV1(ConfiguredBaseModel):
+    path: str = Field(..., alias="path")
+    name: str = Field(..., alias="name")
+    description: str = Field(..., alias="description")
+    account_owner: OwnerV1 = Field(..., alias="accountOwner")
+    organization: AWSOrganizationV1 = Field(..., alias="organization")
+    quota_limits: Optional[list[AWSQuotaLimitsV1]] = Field(..., alias="quotaLimits")
+
+
+class AWSAccountV1(AWSAccountManaged):
+    resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
+    automation_token: VaultSecret = Field(..., alias="automationToken")
+    disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
+    automation_role: Optional[AWSAutomationRoleV1] = Field(..., alias="automationRole")
+    account_requests: Optional[list[AWSAccountRequestV1]] = Field(..., alias="account_requests")
+    organization_accounts: Optional[list[AWSAccountManaged]] = Field(..., alias="organization_accounts")
+
+
+class AWSAccountManagerAccountsQueryData(ConfiguredBaseModel):
+    accounts: Optional[list[AWSAccountV1]] = Field(..., alias="accounts")
+
+
+def query(query_func: Callable, **kwargs: Any) -> AWSAccountManagerAccountsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        AWSAccountManagerAccountsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return AWSAccountManagerAccountsQueryData(**raw_data)

reconcile/gql_definitions/cost_report/app_names.py

@@ -0,0 +1,68 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+DEFINITION = """
+query AppNames {
+  apps: apps_v1 {
+    name
+    parentApp {
+      name
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class AppV1_AppV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+
+
+class AppV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    parent_app: Optional[AppV1_AppV1] = Field(..., alias="parentApp")
+
+
+class AppNamesQueryData(ConfiguredBaseModel):
+    apps: Optional[list[AppV1]] = Field(..., alias="apps")
+
+
+def query(query_func: Callable, **kwargs: Any) -> AppNamesQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        AppNamesQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return AppNamesQueryData(**raw_data)

reconcile/gql_definitions/cost_report/settings.py

@@ -0,0 +1,77 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+
+
+DEFINITION = """
+fragment VaultSecret on VaultSecret_v1 {
+    path
+    field
+    version
+    format
+}
+
+query CostReportAppInterfaceSettings {
+  settings: app_interface_settings_v1 {
+    costReport {
+      credentials {
+        ...VaultSecret
+      }
+    }
+  }
+}
+"""
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class CostReportSettingsV1(ConfiguredBaseModel):
+    credentials: VaultSecret = Field(..., alias="credentials")
+
+
+class AppInterfaceSettingsV1(ConfiguredBaseModel):
+    cost_report: Optional[CostReportSettingsV1] = Field(..., alias="costReport")
+
+
+class CostReportAppInterfaceSettingsQueryData(ConfiguredBaseModel):
+    settings: Optional[list[AppInterfaceSettingsV1]] = Field(..., alias="settings")
+
+
+def query(query_func: Callable, **kwargs: Any) -> CostReportAppInterfaceSettingsQueryData:
+    """
+    This is a convenience function which queries and parses the data into
+    concrete types. It should be compatible with most GQL clients.
+    You do not have to use it to consume the generated data classes.
+    Alternatively, you can also mime and alternate the behavior
+    of this function in the caller.
+
+    Parameters:
+        query_func (Callable): Function which queries your GQL Server
+        kwargs: optional arguments that will be passed to the query function
+
+    Returns:
+        CostReportAppInterfaceSettingsQueryData: queried data parsed into generated classes
+    """
+    raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
+    return CostReportAppInterfaceSettingsQueryData(**raw_data)

reconcile/gql_definitions/fragments/aws_account_managed.py

@@ -0,0 +1,49 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class AWSOrganizationV1(ConfiguredBaseModel):
+    ou: str = Field(..., alias="ou")
+    tags: Json = Field(..., alias="tags")
+
+
+class AWSQuotaV1(ConfiguredBaseModel):
+    service_code: str = Field(..., alias="serviceCode")
+    quota_code: str = Field(..., alias="quotaCode")
+    value: float = Field(..., alias="value")
+
+
+class AWSQuotaLimitsV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    quotas: list[AWSQuotaV1] = Field(..., alias="quotas")
+
+
+class AWSAccountManaged(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
+    uid: str = Field(..., alias="uid")
+    alias: Optional[str] = Field(..., alias="alias")
+    premium_support: bool = Field(..., alias="premiumSupport")
+    organization: Optional[AWSOrganizationV1] = Field(..., alias="organization")
+    quota_limits: Optional[list[AWSQuotaLimitsV1]] = Field(..., alias="quotaLimits")

reconcile/queries.py

@@ -583,7 +583,13 @@ def get_aws_accounts(
         ecrs=ecrs,
         cleanup=cleanup,
     )
-    return gqlapi.query(query)["accounts"]
+    accounts = gqlapi.query(query)["accounts"]
+    if terraform_state:
+        # a new account does not have a terraform state yet, ignore it until terraform-init does its job
+        return [
+            account for account in accounts if account.get("terraformState") is not None
+        ]
+    return accounts
 
 
 def get_state_aws_accounts(reset_passwords=False):

reconcile/templating/lib/merge_request_manager.py

@@ -1,17 +1,16 @@
 import logging
 import re
 import string
-from dataclasses import dataclass
 
-from gitlab.v4.objects import ProjectMergeRequest
 from pydantic import BaseModel
 
 from reconcile.templating.lib.model import TemplateOutput
 from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.merge_request_manager.merge_request_manager import (
+    MergeRequestManagerBase,
+)
 from reconcile.utils.merge_request_manager.parser import (
     Parser,
-    ParserError,
-    ParserVersionError,
 )
 from reconcile.utils.mr import MergeRequestBase
 from reconcile.utils.vcs import VCS
@@ -79,12 +78,6 @@ class TemplateInfo(BaseModel):
     collection_hash: str
 
 
-@dataclass
-class OpenMergeRequest:
-    raw: ProjectMergeRequest
-    template_info: TemplateInfo
-
-
 class TemplateRenderingMR(MergeRequestBase):
     name = "TemplateRendering"
 
@@ -127,78 +120,11 @@ class TemplateRenderingMR(MergeRequestBase):
                 )
 
 
-class MergeRequestManager:
-    # TODO: Create base class for Merge Request Manager, to make it reusable
-    """ """
-
+class MergeRequestManager(MergeRequestManagerBase[TemplateInfo]):
     def __init__(self, vcs: VCS, parser: Parser):
-        self._vcs = vcs
-        self._parser = parser
-        self._open_mrs: list[OpenMergeRequest] = []
-        self._open_mrs_with_problems: list[OpenMergeRequest] = []
-        self._housekeeping_ran = False
-
-    def _merge_request_already_exists(
-        self,
-        collection: str,
-    ) -> OpenMergeRequest | None:
-        for mr in self._open_mrs:
-            if mr.template_info.collection == collection:
-                return mr
-
-        return None
-
-    def _fetch_avs_managed_open_merge_requests(self) -> list[ProjectMergeRequest]:
-        all_open_mrs = self._vcs.get_open_app_interface_merge_requests()
-        return [mr for mr in all_open_mrs if TR_LABEL in mr.labels]
-
-    def housekeeping(self) -> None:
-        """
-        Close bad MRs:
-        - bad description format
-        - wrong version
-        - merge conflict
-
-        --> if we update the template output, we automatically close
-        old open MRs and replace them with new ones.
-        """
-        for mr in self._fetch_avs_managed_open_merge_requests():
-            attrs = mr.attributes
-            desc = attrs.get("description")
-            has_conflicts = attrs.get("has_conflicts", False)
-            if has_conflicts:
-                logging.info(
-                    "Merge-conflict detected. Closing %s",
-                    mr.attributes.get("web_url", "NO_WEBURL"),
-                )
-                self._vcs.close_app_interface_mr(
-                    mr, "Closing this MR because of a merge-conflict."
-                )
-                continue
-            try:
-                template_info = self._parser.parse(description=desc)
-            except ParserVersionError:
-                logging.info(
-                    "Old MR version detected! Closing %s",
-                    mr.attributes.get("web_url", "NO_WEBURL"),
-                )
-                self._vcs.close_app_interface_mr(
-                    mr, "Closing this MR because it has an outdated integration version"
-                )
-                continue
-            except ParserError:
-                logging.info(
-                    "Bad MR description format. Closing %s",
-                    mr.attributes.get("web_url", "NO_WEBURL"),
-                )
-                self._vcs.close_app_interface_mr(
-                    mr, "Closing this MR because of bad description format."
-                )
-                continue
-            self._open_mrs.append(OpenMergeRequest(raw=mr, template_info=template_info))
-        self._housekeeping_ran = True
+        super().__init__(vcs, parser, TR_LABEL)
 
-    def create_tr_merge_request(self, output: list[TemplateOutput]) -> None:
+    def create_merge_request(self, output: list[TemplateOutput]) -> None:
         if not self._housekeeping_ran:
             self.housekeeping()
 
@@ -211,8 +137,8 @@ class MergeRequestManager:
         collection_hash = collection_hashes.pop()
 
         """Create a new MR with the rendered template."""
-        if mr := self._merge_request_already_exists(collection):
-            if mr.template_info.collection_hash == collection_hash:
+        if mr := self._merge_request_already_exists({"collection": collection}):
+            if mr.mr_info.collection_hash == collection_hash:
                 logging.info(
                     "MR already exists and has the same template hash. Skipping",
                 )

reconcile/templating/renderer.py

@@ -97,7 +97,7 @@ class GitlabFilePersistence(FilePersistence):
 
     def write(self, outputs: list[TemplateOutput]) -> None:
         self.mr_manager.housekeeping()
-        self.mr_manager.create_tr_merge_request(outputs)
+        self.mr_manager.create_merge_request(outputs)
 
     def read(self, path: str) -> Optional[str]:
         try:
@@ -182,10 +182,10 @@ class TemplateRendererIntegration(QontractReconcileIntegration):
         ruamel_instance: yaml.YAML,
         state: Optional[State] = None,
     ) -> None:
-        outputs: list[TemplateOutput] = []
         gql_no_validation = init_from_config(validate_schemas=False)
 
         for c in get_template_collections():
+            outputs: list[TemplateOutput] = []
             variables = {}
             if c.variables:
                 variables = {

reconcile/typed_queries/cost_report/app_names.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+from reconcile.gql_definitions.cost_report.app_names import query
+from reconcile.utils.gql import GqlApi
+
+
+class App(BaseModel):
+    name: str
+    parent_app_name: str | None
+
+
+def get_app_names(
+    gql_api: GqlApi,
+) -> list[App]:
+    apps = query(gql_api.query).apps or []
+    return [
+        App(
+            name=app.name,
+            parent_app_name=app.parent_app.name if app.parent_app else None,
+        )
+        for app in apps
+    ]

reconcile/typed_queries/cost_report/settings.py

@@ -0,0 +1,15 @@
+from reconcile.gql_definitions.cost_report.settings import CostReportSettingsV1, query
+from reconcile.utils.exceptions import AppInterfaceSettingsError
+from reconcile.utils.gql import GqlApi
+
+
+def get_cost_report_settings(
+    gql_api: GqlApi,
+) -> CostReportSettingsV1:
+    data = query(gql_api.query)
+    if not data.settings:
+        raise AppInterfaceSettingsError("No settings configured")
+    cost_report_settings = data.settings[0].cost_report
+    if cost_report_settings is None:
+        raise AppInterfaceSettingsError("No cost report configured")
+    return cost_report_settings