qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/terraform_vpc_resources/integration.py

@@ -0,0 +1,220 @@
+import logging
+import sys
+from collections.abc import Iterable, Mapping, MutableMapping
+from typing import Any
+
+import jinja2
+
+from reconcile.gql_definitions.fragments.aws_vpc_request import (
+    AWSAccountV1,
+    VPCRequest,
+)
+from reconcile.status import ExitCodes
+from reconcile.terraform_vpc_resources.merge_request import Renderer, create_parser
+from reconcile.terraform_vpc_resources.merge_request_manager import (
+    MergeRequestManager,
+    MrData,
+)
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.aws_vpc_requests import get_aws_vpc_requests
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.utils import gql
+from reconcile.utils.runtime.integration import (
+    DesiredStateShardConfig,
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.terraform_client import TerraformClient
+from reconcile.utils.terrascript_aws_client import TerrascriptClient
+from reconcile.utils.vcs import VCS
+
+QONTRACT_INTEGRATION = "terraform_vpc_resources"
+QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
+QONTRACT_TF_PREFIX = "qrtvr"
+AWS_PROVIDER_VERSION = "5.7.1"
+
+
+class TerraformVpcResourcesParams(PydanticRunParams):
+    account_name: str | None
+    print_to_file: str | None
+    thread_pool_size: int
+    enable_deletion: bool = False
+
+
+class TerraformVpcResources(QontractReconcileIntegration[TerraformVpcResourcesParams]):
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION.replace("_", "-")
+
+    def _filter_accounts(
+        self, data: Iterable[VPCRequest], account_name: str | None
+    ) -> list[AWSAccountV1]:
+        """Return a list of accounts extracted from the provided VPCRequests.
+        If account_name is given returns the account object with that name."""
+        accounts = [vpc.account for vpc in data]
+
+        if account_name:
+            accounts = [account for account in accounts if account.name == account_name]
+
+        return accounts
+
+    def _handle_outputs(
+        self, requests: Iterable[VPCRequest], outputs: Mapping[str, Any]
+    ) -> Mapping[str, Any]:
+        """Receives a terraform outputs dict and returns a map of outputs per VPC requests"""
+        outputs_per_request: MutableMapping[str, Any] = {}
+        for request in requests:
+            # Skiping requests that don't have outputs,
+            # this happens because we are not filtering the requests
+            # when running the integration for a single account with --account-name.
+            # We also don't want to create outputs for deleted requets.
+            if request.account.name not in outputs or request.delete:
+                continue
+
+            outputs_per_request[request.identifier] = []
+
+            outputs_per_account = outputs[request.account.name]
+
+            # If the output exists for that request get its value
+            # Else get None
+            private_subnets = outputs_per_account.get(
+                f"{request.identifier}-private_subnets", {}
+            ).get("value", [])
+            public_subnets = outputs_per_account.get(
+                f"{request.identifier}-public_subnets", {}
+            ).get("value", [])
+
+            values = {
+                "static": {
+                    "vpc_id": outputs_per_account.get(
+                        f"{request.identifier}-vpc_id", {}
+                    ).get("value"),
+                    "subnets": {
+                        "private": private_subnets,
+                        "public": public_subnets,
+                    },
+                    "account_name": request.account.name,
+                    "region": request.region,
+                    "cidr_block": request.cidr_block.network_address,
+                    "identifier": request.identifier,
+                }
+            }
+
+            outputs_per_request[request.identifier] = values
+
+        return outputs_per_request
+
+    def _render_template(self, template: str, data: Mapping[str, Any]) -> str:
+        return jinja2.Template(
+            template,
+            undefined=jinja2.StrictUndefined,
+            trim_blocks=False,
+            lstrip_blocks=False,
+            keep_trailing_newline=False,
+        ).render(data)
+
+    def run(self, dry_run: bool) -> None:
+        account_name = self.params.account_name
+        thread_pool_size = self.params.thread_pool_size
+        enable_deletion = self.params.enable_deletion
+
+        vault_settings = get_app_interface_vault_settings()
+        secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+
+        gql_api = gql.get_api()
+        data = get_aws_vpc_requests(gql_api=gql_api)
+
+        if data:
+            accounts = self._filter_accounts(data, account_name)
+            if account_name and not accounts:
+                msg = f"The account {account_name} doesn't have any managed vpc. Verify your input"
+                logging.debug(msg)
+                sys.exit(ExitCodes.SUCCESS)
+        else:
+            logging.debug("No VPC requests found, nothing to do.")
+            sys.exit(ExitCodes.SUCCESS)
+
+        accounts_untyped: list[dict] = [acc.dict(by_alias=True) for acc in accounts]
+        with TerrascriptClient(
+            integration=QONTRACT_INTEGRATION,
+            integration_prefix=QONTRACT_TF_PREFIX,
+            thread_pool_size=thread_pool_size,
+            accounts=accounts_untyped,
+            secret_reader=secret_reader,
+        ) as ts_client:
+            ts_client.populate_vpc_requests(data, AWS_PROVIDER_VERSION)
+
+        working_dirs = ts_client.dump(print_to_file=self.params.print_to_file)
+
+        if self.params.print_to_file:
+            sys.exit(ExitCodes.SUCCESS)
+
+        tf_client = TerraformClient(
+            integration=QONTRACT_INTEGRATION,
+            integration_version=QONTRACT_INTEGRATION_VERSION,
+            integration_prefix=QONTRACT_TF_PREFIX,
+            accounts=accounts_untyped,
+            working_dirs=working_dirs,
+            thread_pool_size=thread_pool_size,
+        )
+
+        tf_client.safe_plan(enable_deletion=enable_deletion)
+
+        if dry_run:
+            sys.exit(ExitCodes.SUCCESS)
+
+        tf_client.apply()
+
+        tf_client.init_outputs()
+
+        handled_output = self._handle_outputs(data, tf_client.outputs)
+
+        # MR and template Management
+        vcs = VCS(
+            secret_reader=secret_reader,
+            github_orgs=get_github_orgs(),
+            gitlab_instances=get_gitlab_instances(),
+            app_interface_repo_url=get_app_interface_repo_url(),
+            dry_run=dry_run,
+            allow_deleting_mrs=False,
+            allow_opening_mrs=True,
+        )
+
+        mr_manager = MergeRequestManager(
+            vcs=vcs,
+            renderer=Renderer(),
+            parser=create_parser(),
+            auto_merge_enabled=True,
+        )
+
+        mr_manager._fetch_managed_open_merge_requests()
+
+        # Create a MR for each vpc request if the MR don't exist yet
+        for _, outputs in handled_output.items():
+            template = gql_api.get_template(
+                path="/templating/templates/terraform-vpc-resources/vpc.yml"
+            )["template"]
+            content = self._render_template(template=template, data=outputs)
+
+            mr_manager.create_merge_request(
+                MrData(
+                    account=outputs["static"]["account_name"],
+                    content=content,
+                    path=f"data/aws/{outputs['static']['account_name']}/vpcs/{outputs['static']['identifier']}.yml",
+                )
+            )
+
+    def get_desired_state_shard_config(self) -> DesiredStateShardConfig:
+        return DesiredStateShardConfig(
+            shard_arg_name="account_name",
+            shard_path_selectors={
+                "accounts[*].name",
+            },
+            sharded_run_review=lambda proposal: len(proposal.proposed_shards) <= 2,
+        )

reconcile/terraform_vpc_resources/merge_request.py

@@ -0,0 +1,57 @@
+import re
+import string
+
+from pydantic import BaseModel
+
+from reconcile.utils.merge_request_manager.parser import Parser
+
+PROMOTION_DATA_SEPARATOR = "**DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
+VERSION = "0.1.0"
+LABEL = "terraform-vpc-resources"
+
+VERSION_REF = "tf_vpc_resources_version"
+ACCOUNT_REF = "account"
+COMPILED_REGEXES = {
+    i: re.compile(rf".*{i}: (.*)$", re.MULTILINE) for i in [VERSION_REF, ACCOUNT_REF]
+}
+
+DESC = string.Template(
+    f"""
+This MR is triggered by app-interface's [terraform-vpc-resources](https://github.com/app-sre/qontract-reconcile/tree/master/reconcile/terraform_vpc_request).
+
+Please **do not remove** the **{LABEL}** label from this MR!
+
+Parts of this description are used by integration to manage the MR.
+
+{PROMOTION_DATA_SEPARATOR}
+
+* {VERSION_REF}: {VERSION}
+* {ACCOUNT_REF}: $account
+"""
+)
+
+
+class Info(BaseModel):
+    account: str
+
+
+def create_parser() -> Parser:
+    """Create a parser for MRs created by terraform-vpc-resources."""
+
+    return Parser[Info](
+        klass=Info,
+        compiled_regexes=COMPILED_REGEXES,
+        version_ref=VERSION_REF,
+        expected_version=VERSION,
+        data_separator=PROMOTION_DATA_SEPARATOR,
+    )
+
+
+class Renderer:
+    """This class is only concerned with rendering text for MRs."""
+
+    def render_description(self, account: str) -> str:
+        return DESC.safe_substitute(account=account)
+
+    def render_title(self, account: str) -> str:
+        return f"[auto] VPC data file creation to {account}"

reconcile/terraform_vpc_resources/merge_request_manager.py

@@ -0,0 +1,107 @@
+import logging
+
+from gitlab.exceptions import GitlabGetError
+from pydantic import BaseModel
+
+from reconcile.terraform_vpc_resources.merge_request import (
+    LABEL,
+    Info,
+    Renderer,
+)
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.merge_request_manager.merge_request_manager import (
+    MergeRequestManagerBase,
+)
+from reconcile.utils.merge_request_manager.parser import Parser
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
+from reconcile.utils.vcs import VCS
+
+
+class VPCRequestMR(MergeRequestBase):
+    name = "VPCRequest"
+
+    def __init__(
+        self,
+        title: str,
+        description: str,
+        vpc_tmpl_file_path: str,
+        vpc_tmpl_file_content: str,
+        labels: list[str],
+    ):
+        super().__init__()
+        self._title = title
+        self._description = description
+        self._vpc_tmpl_file_path = vpc_tmpl_file_path
+        self._vpc_tmpl_file_content = vpc_tmpl_file_content
+        self.labels = labels
+
+    @property
+    def title(self) -> str:
+        return self._title
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        gitlab_cli.create_file(
+            branch_name=self.branch,
+            file_path=self._vpc_tmpl_file_path,
+            commit_message="add vpc datafile",
+            content=self._vpc_tmpl_file_content,
+        )
+
+
+class MrData(BaseModel):
+    account: str
+    content: str
+    path: str
+
+
+class MergeRequestManager(MergeRequestManagerBase[Info]):
+    """Manager for the merge requests.
+
+    This class is responsible for housekeeping (closing old/bad MRs) and
+    opening new MRs.
+    """
+
+    def __init__(
+        self, vcs: VCS, renderer: Renderer, parser: Parser, auto_merge_enabled: bool
+    ):
+        super().__init__(vcs, parser, LABEL)
+        self._renderer = renderer
+        self._auto_merge_enabled = auto_merge_enabled
+
+    def create_merge_request(self, data: MrData) -> None:
+        """Open a new MR, if not already present, for a VPC datafile and close any outdated before."""
+        if not self._housekeeping_ran:
+            self.housekeeping()
+
+        if self._merge_request_already_exists({"account": data.account}):
+            logging.info("MR already exists for %s", data.account)
+            return None
+
+        try:
+            self._vcs.get_file_content_from_app_interface_master(file_path=data.path)
+            # the file exists, nothing to do
+            return None
+        except GitlabGetError as e:
+            if e.response_code != 404:
+                raise
+
+        description = self._renderer.render_description(account=data.account)
+        title = self._renderer.render_title(account=data.account)
+        logging.info("Open MR for %s", data.account)
+        mr_labels = [LABEL]
+        if self._auto_merge_enabled:
+            mr_labels.append(AUTO_MERGE)
+        self._vcs.open_app_interface_merge_request(
+            mr=VPCRequestMR(
+                vpc_tmpl_file_path=data.path,
+                title=title,
+                description=description,
+                vpc_tmpl_file_content=data.content,
+                labels=mr_labels,
+            )
+        )

reconcile/typed_queries/alerting_services_settings.py

@@ -1,12 +1,11 @@
 from collections.abc import Callable
-from typing import Optional
 
 from reconcile.gql_definitions.common.alerting_services_settings import query
 from reconcile.utils import gql
 from reconcile.utils.exceptions import AppInterfaceSettingsError
 
 
-def get_alerting_services(query_func: Optional[Callable] = None) -> set[str]:
+def get_alerting_services(query_func: Callable | None = None) -> set[str]:
     """Get alertingServices from app-interface settings"""
 
     if not query_func:

reconcile/typed_queries/app_interface_custom_messages.py

@@ -0,0 +1,24 @@
+from collections.abc import Callable
+
+from reconcile.gql_definitions.common.app_interface_custom_messages import (
+    query,
+)
+from reconcile.utils import gql
+
+
+def get_app_interface_custom_message(
+    desired_id: str,
+    query_func: Callable | None = None,
+) -> str | None:
+    """Returns App Interface Custom Message by ID or None if not found"""
+    if not query_func:
+        query_func = gql.get_api().query
+    data = query(query_func=query_func)
+    for item in (
+        data.settings[0].custom_messages
+        if data.settings and data.settings[0].custom_messages
+        else []
+    ):
+        if item.q_id == desired_id:
+            return item.content
+    return None

reconcile/typed_queries/app_interface_deadmanssnitch_settings.py

@@ -0,0 +1,17 @@
+from reconcile.gql_definitions.common.app_interface_dms_settings import (
+    DeadMansSnitchSettingsV1,
+    query,
+)
+from reconcile.utils import gql
+from reconcile.utils.exceptions import AppInterfaceSettingsError
+from reconcile.utils.gql import GqlApi
+
+
+def get_deadmanssnitch_settings(
+    gql_api: GqlApi | None = None,
+) -> DeadMansSnitchSettingsV1:
+    api = gql_api if gql_api else gql.get_api()
+    data = query(query_func=api.query)
+    if data.settings and data.settings[0].dead_mans_snitch_settings is not None:
+        return data.settings[0].dead_mans_snitch_settings
+    raise AppInterfaceSettingsError("deadmanssnitch settings missing")

reconcile/typed_queries/app_interface_metrics_exporter/onboarding_status.py

@@ -0,0 +1,13 @@
+from collections import Counter
+
+from reconcile.gql_definitions.app_interface_metrics_exporter.onboarding_status import (
+    query,
+)
+from reconcile.utils.gql import GqlApi
+
+
+def get_onboarding_status(
+    gql: GqlApi,
+) -> Counter:
+    apps = query(gql.query).apps or []
+    return Counter(a.onboarding_status for a in apps)

reconcile/typed_queries/app_interface_repo_url.py

@@ -1,12 +1,11 @@
 from collections.abc import Callable
-from typing import Optional
 
 from reconcile.gql_definitions.common.app_interface_repo_settings import query
 from reconcile.utils import gql
 from reconcile.utils.exceptions import AppInterfaceSettingsError
 
 
-def get_app_interface_repo_url(query_func: Optional[Callable] = None) -> str:
+def get_app_interface_repo_url(query_func: Callable | None = None) -> str:
     if not query_func:
         gqlapi = gql.get_api()
         query_func = gqlapi.query

reconcile/typed_queries/app_interface_state_settings.py

@@ -1,5 +1,3 @@
-from typing import Optional
-
 from reconcile.gql_definitions.common.app_interface_state_settings import (
     AppInterfaceStateConfigurationV1,
     query,
@@ -7,7 +5,7 @@ from reconcile.gql_definitions.common.app_interface_state_settings import (
 from reconcile.utils import gql
 
 
-def get_app_interface_state_settings() -> Optional[AppInterfaceStateConfigurationV1]:
+def get_app_interface_state_settings() -> AppInterfaceStateConfigurationV1 | None:
     """Returns App Interface Settings"""
     gqlapi = gql.get_api()
     data = query(gqlapi.query)

reconcile/typed_queries/app_interface_vault_settings.py

@@ -1,5 +1,4 @@
 from collections.abc import Callable
-from typing import Optional
 
 from reconcile.gql_definitions.common.app_interface_vault_settings import (
     AppInterfaceSettingsV1,
@@ -10,7 +9,7 @@ from reconcile.utils.exceptions import AppInterfaceSettingsError
 
 
 def get_app_interface_vault_settings(
-    query_func: Optional[Callable] = None,
+    query_func: Callable | None = None,
 ) -> AppInterfaceSettingsV1:
     """Returns App Interface Settings and raises err if none are found"""
     if not query_func:

reconcile/typed_queries/app_quay_repos_escalation_policies.py

@@ -0,0 +1,14 @@
+from reconcile.gql_definitions.common.app_quay_repos_escalation_policies import (
+    AppV1,
+    query,
+)
+from reconcile.utils import gql
+from reconcile.utils.gql import GqlApi
+
+
+def get_apps_quay_repos_escalation_policies(
+    gql_api: GqlApi | None = None,
+) -> list[AppV1]:
+    api = gql_api if gql_api else gql.get_api()
+    data = query(query_func=api.query)
+    return list(data.apps or [])

reconcile/typed_queries/apps.py

@@ -0,0 +1,11 @@
+from collections.abc import Callable
+
+from reconcile.gql_definitions.common.apps import AppV1, query
+from reconcile.utils import gql
+
+
+def get_apps(query_func: Callable | None = None) -> list[AppV1]:
+    if not query_func:
+        gqlapi = gql.get_api()
+        query_func = gqlapi.query
+    return query(query_func=query_func).apps or []

reconcile/typed_queries/aws_vpc_requests.py

@@ -0,0 +1,9 @@
+from reconcile.gql_definitions.common.aws_vpc_requests import VPCRequest, query
+from reconcile.utils import gql
+from reconcile.utils.gql import GqlApi
+
+
+def get_aws_vpc_requests(gql_api: GqlApi | None = None) -> list[VPCRequest]:
+    api = gql_api if gql_api else gql.get_api()
+    data = query(query_func=api.query)
+    return list(data.vpc_requests or [])

reconcile/typed_queries/aws_vpcs.py

@@ -0,0 +1,12 @@
+from reconcile.gql_definitions.common.aws_vpcs import (
+    AWSVPC,
+    query,
+)
+from reconcile.utils import gql
+from reconcile.utils.gql import GqlApi
+
+
+def get_aws_vpcs(gql_api: GqlApi | None = None) -> list[AWSVPC]:
+    api = gql_api if gql_api else gql.get_api()
+    data = query(query_func=api.query)
+    return list(data.vpcs or [])

reconcile/typed_queries/cloudflare.py

@@ -0,0 +1,10 @@
+from reconcile.gql_definitions.terraform_cloudflare_resources.terraform_cloudflare_accounts import (
+    CloudflareAccountV1,
+    query,
+)
+from reconcile.utils import gql
+
+
+def get_cloudflare_accounts() -> list[CloudflareAccountV1]:
+    data = query(gql.get_api().query)
+    return list(data.accounts or [])

reconcile/typed_queries/clusters.py

@@ -1,16 +1,18 @@
-from typing import Optional
-
 from reconcile.gql_definitions.common.clusters import (
     ClusterV1,
     query,
 )
 from reconcile.utils import gql
+from reconcile.utils.gql import GqlApi
 
 
-def get_clusters(name: Optional[str] = None) -> list[ClusterV1]:
+def get_clusters(
+    gql_api: GqlApi | None = None,
+    name: str | None = None,
+) -> list[ClusterV1]:
     variables = {}
     if name:
         variables["name"] = name
-    gqlapi = gql.get_api()
-    data = query(gqlapi.query, variables=variables)
+    api = gql_api if gql_api else gql.get_api()
+    data = query(query_func=api.query, variables=variables)
     return list(data.clusters or [])

reconcile/typed_queries/clusters_minimal.py

@@ -1,16 +1,17 @@
-from typing import Optional
-
 from reconcile.gql_definitions.common.clusters_minimal import (
     ClusterV1,
     query,
 )
 from reconcile.utils import gql
+from reconcile.utils.gql import GqlApi
 
 
-def get_clusters_minimal(name: Optional[str] = None) -> list[ClusterV1]:
+def get_clusters_minimal(
+    gql_api: GqlApi | None = None, name: str | None = None
+) -> list[ClusterV1]:
     variables = {}
     if name:
         variables["name"] = name
-    gqlapi = gql.get_api()
-    data = query(gqlapi.query, variables=variables)
+    api = gql_api if gql_api else gql.get_api()
+    data = query(query_func=api.query, variables=variables)
     return list(data.clusters or [])

reconcile/typed_queries/clusters_with_dms.py

@@ -0,0 +1,16 @@
+from reconcile.gql_definitions.common.clusters_with_dms import (
+    ClusterV1,
+    query,
+)
+from reconcile.utils import gql
+from reconcile.utils.gql import GqlApi
+
+
+def get_clusters_with_dms(
+    gql_api: GqlApi | None = None,
+) -> list[ClusterV1]:
+    # get the clusters containing the filed enableDeadMansSnitch
+    variable = {"filter": {"enableDeadMansSnitch": {"ne": None}}}
+    api = gql_api if gql_api else gql.get_api()
+    data = query(query_func=api.query, variables=variable)
+    return data.clusters or []

reconcile/typed_queries/cost_report/app_names.py

@@ -0,0 +1,22 @@
+from pydantic import BaseModel
+
+from reconcile.gql_definitions.cost_report.app_names import query
+from reconcile.utils.gql import GqlApi
+
+
+class App(BaseModel):
+    name: str
+    parent_app_name: str | None
+
+
+def get_app_names(
+    gql_api: GqlApi,
+) -> list[App]:
+    apps = query(gql_api.query).apps or []
+    return [
+        App(
+            name=app.name,
+            parent_app_name=app.parent_app.name if app.parent_app else None,
+        )
+        for app in apps
+    ]