dbt-platform-helper 11.0.1__tar.gz → 11.1.0__tar.gz

{dbt_platform_helper-11.0.1 → dbt_platform_helper-11.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 11.0.1
+Version: 11.1.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team
@@ -34,6 +34,7 @@ Requires-Dist: schema (==0.7.5)
 Requires-Dist: semver (>=3.0.2,<4.0.0)
 Requires-Dist: slack-sdk (>=3.27.1,<4.0.0)
 Requires-Dist: tomlkit (>=0.12.2,<0.13.0)
+Requires-Dist: yamllint (>=1.35.1,<2.0.0)
 Description-Content-Type: text/markdown
 
 # DBT Platform Helper

{dbt_platform_helper-11.0.1 → dbt_platform_helper-11.1.0}/dbt_platform_helper/COMMANDS.md

@@ -42,6 +42,7 @@
 - [platform-helper database](#platform-helper-database)
 - [platform-helper database dump](#platform-helper-database-dump)
 - [platform-helper database load](#platform-helper-database-load)
+- [platform-helper database copy](#platform-helper-database-copy)
 - [platform-helper version](#platform-helper-version)
 - [platform-helper version get-platform-helper-for-project](#platform-helper-version-get-platform-helper-for-project)
 
@@ -962,10 +963,12 @@ platform-helper notify add-comment <slack_channel_id> <slack_token>
 
 [↩ Parent](#platform-helper)
 
+    Commands to copy data between databases.
+
 ## Usage
 
 ```
-platform-helper database (dump|load) 
+platform-helper database (dump|load|copy) 
 ```
 
 ## Options
@@ -975,6 +978,7 @@ platform-helper database (dump|load)
 
 ## Commands
 
+- [`copy` ↪](#platform-helper-database-copy)
 - [`dump` ↪](#platform-helper-database-dump)
 - [`load` ↪](#platform-helper-database-load)
 
@@ -987,23 +991,20 @@ platform-helper database (dump|load)
 ## Usage
 
 ```
-platform-helper database dump --account-id <account_id> --app <application> 
-                              --env <environment> --database <database> 
-                              --vpc-name <vpc_name> 
+platform-helper database dump --from <from_env> --database <database> 
+                              [--app <application>] [--from-vpc <from_vpc>] 
 ```
 
 ## Options
 
-- `--account-id <text>`
-
 - `--app <text>`
-
-- `--env <text>`
-
+  - The application name. Required unless you are running the command from your deploy repo
+- `--from <text>`
+  - The environment you are dumping data from
 - `--database <text>`
-
-- `--vpc-name <text>`
-
+  - The name of the database you are dumping data from
+- `--from-vpc <text>`
+  - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
@@ -1016,22 +1017,62 @@ platform-helper database dump --account-id <account_id> --app <application>
 ## Usage
 
 ```
-platform-helper database load --account-id <account_id> --app <application> 
-                              --env <environment> --database <database> 
-                              --vpc-name <vpc_name> 
+platform-helper database load --to <to_env> --database <database> 
+                              [--app <application>] [--to-vpc <to_vpc>] 
+                              [--auto-approve] 
 ```
 
 ## Options
 
-- `--account-id <text>`
-
 - `--app <text>`
+  - The application name. Required unless you are running the command from your deploy repo
+- `--to <text>`
+  - The environment you are loading data into
+- `--database <text>`
+  - The name of the database you are loading data into
+- `--to-vpc <text>`
+  - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
+- `--auto-approve <boolean>` _Defaults to False._
 
-- `--env <text>`
+- `--help <boolean>` _Defaults to False._
+  - Show this message and exit.
+
+# platform-helper database copy
+
+[↩ Parent](#platform-helper-database)
+
+    Copy a database between environments.
+
+## Usage
 
+```
+platform-helper database copy --from <from_env> --to <to_env> --database <database> 
+                              --svc <service> [--app <application>] [--from-vpc <from_vpc>] 
+                              [--to-vpc <to_vpc>] [--template (default|migration|dmas-migration)] 
+                              [--auto-approve] [--no-maintenance-page] 
+```
+
+## Options
+
+- `--app <text>`
+  - The application name. Required unless you are running the command from your deploy repo
+- `--from <text>`
+  - The environment you are copying data from
+- `--to <text>`
+  - The environment you are copying data into
 - `--database <text>`
+  - The name of the database you are copying
+- `--from-vpc <text>`
+  - The vpc the environment you are copying from is running in. Required unless you are running the command from your deploy repo
+- `--to-vpc <text>`
+  - The vpc the environment you are copying into is running in. Required unless you are running the command from your deploy repo
+- `--auto-approve <boolean>` _Defaults to False._
 
-- `--vpc-name <text>`
+- `--svc <text>` _Defaults to ['web']._
+
+- `--template <choice>` _Defaults to default._
+  - The maintenance page you wish to put up.
+- `--no-maintenance-page <boolean>` _Defaults to False._
 
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.

dbt_platform_helper-11.1.0/dbt_platform_helper/commands/database.py

@@ -0,0 +1,112 @@
+import click
+
+from dbt_platform_helper.commands.environment import AVAILABLE_TEMPLATES
+from dbt_platform_helper.domain.database_copy import DatabaseCopy
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+
+
+@click.group(chain=True, cls=ClickDocOptGroup)
+def database():
+    """Commands to copy data between databases."""
+
+
+@database.command(name="dump")
+@click.option(
+    "--app",
+    type=str,
+    help="The application name. Required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--from",
+    "from_env",
+    type=str,
+    required=True,
+    help="The environment you are dumping data from",
+)
+@click.option(
+    "--database", type=str, required=True, help="The name of the database you are dumping data from"
+)
+@click.option(
+    "--from-vpc",
+    type=str,
+    help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
+)
+def dump(app, from_env, database, from_vpc):
+    """Dump a database into an S3 bucket."""
+    data_copy = DatabaseCopy(app, database)
+    data_copy.dump(from_env, from_vpc)
+
+
+@database.command(name="load")
+@click.option(
+    "--app",
+    type=str,
+    help="The application name. Required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--to", "to_env", type=str, required=True, help="The environment you are loading data into"
+)
+@click.option(
+    "--database", type=str, required=True, help="The name of the database you are loading data into"
+)
+@click.option(
+    "--to-vpc",
+    type=str,
+    help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
+)
+@click.option("--auto-approve/--no-auto-approve", default=False)
+def load(app, to_env, database, to_vpc, auto_approve):
+    """Load a database from an S3 bucket."""
+    data_copy = DatabaseCopy(app, database, auto_approve)
+    data_copy.load(to_env, to_vpc)
+
+
+@database.command(name="copy")
+@click.option(
+    "--app",
+    type=str,
+    help="The application name. Required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--from", "from_env", type=str, required=True, help="The environment you are copying data from"
+)
+@click.option(
+    "--to", "to_env", type=str, required=True, help="The environment you are copying data into"
+)
+@click.option(
+    "--database", type=str, required=True, help="The name of the database you are copying"
+)
+@click.option(
+    "--from-vpc",
+    type=str,
+    help="The vpc the environment you are copying from is running in. Required unless you are running the command from your deploy repo",
+)
+@click.option(
+    "--to-vpc",
+    type=str,
+    help="The vpc the environment you are copying into is running in. Required unless you are running the command from your deploy repo",
+)
+@click.option("--auto-approve/--no-auto-approve", default=False)
+@click.option("--svc", type=str, required=True, multiple=True, default=["web"])
+@click.option(
+    "--template",
+    type=click.Choice(AVAILABLE_TEMPLATES),
+    default="default",
+    help="The maintenance page you wish to put up.",
+)
+@click.option("--no-maintenance-page", flag_value=True)
+def copy(
+    app,
+    from_env,
+    to_env,
+    database,
+    from_vpc,
+    to_vpc,
+    auto_approve,
+    svc,
+    template,
+    no_maintenance_page,
+):
+    """Copy a database between environments."""
+    data_copy = DatabaseCopy(app, database, auto_approve)
+    data_copy.copy(from_env, to_env, from_vpc, to_vpc, svc, template, no_maintenance_page)

dbt_platform_helper-11.1.0/dbt_platform_helper/commands/environment.py

@@ -0,0 +1,218 @@
+import boto3
+import click
+from schema import SchemaError
+
+from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
+from dbt_platform_helper.providers.load_balancers import find_https_listener
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+from dbt_platform_helper.utils.files import apply_environment_defaults
+from dbt_platform_helper.utils.files import mkfile
+from dbt_platform_helper.utils.platform_config import is_terraform_project
+from dbt_platform_helper.utils.template import setup_templates
+from dbt_platform_helper.utils.validation import load_and_validate_platform_config
+from dbt_platform_helper.utils.versioning import (
+    check_platform_helper_version_needs_update,
+)
+
+AVAILABLE_TEMPLATES = ["default", "migration", "dmas-migration"]
+
+
+@click.group(cls=ClickDocOptGroup)
+def environment():
+    """Commands affecting environments."""
+    check_platform_helper_version_needs_update()
+
+
+@environment.command()
+@click.option("--app", type=str, required=True)
+@click.option("--env", type=str, required=True)
+@click.option("--svc", type=str, required=True, multiple=True, default=["web"])
+@click.option(
+    "--template",
+    type=click.Choice(AVAILABLE_TEMPLATES),
+    default="default",
+    help="The maintenance page you wish to put up.",
+)
+@click.option("--vpc", type=str)
+def offline(app, env, svc, template, vpc):
+    """Take load-balanced web services offline with a maintenance page."""
+    maintenance_page = MaintenancePageProvider()
+    maintenance_page.activate(app, env, svc, template, vpc)
+
+
+@environment.command()
+@click.option("--app", type=str, required=True)
+@click.option("--env", type=str, required=True)
+def online(app, env):
+    """Remove a maintenance page from an environment."""
+    maintenance_page = MaintenancePageProvider()
+    maintenance_page.deactivate(app, env)
+
+
+def get_vpc_id(session, env_name, vpc_name=None):
+    if not vpc_name:
+        vpc_name = f"{session.profile_name}-{env_name}"
+
+    filters = [{"Name": "tag:Name", "Values": [vpc_name]}]
+    vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
+
+    if not vpcs:
+        filters[0]["Values"] = [session.profile_name]
+        vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
+
+    if not vpcs:
+        click.secho(
+            f"No VPC found with name {vpc_name} in AWS account {session.profile_name}.", fg="red"
+        )
+        raise click.Abort
+
+    return vpcs[0]["VpcId"]
+
+
+def get_subnet_ids(session, vpc_id):
+    subnets = session.client("ec2").describe_subnets(
+        Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
+    )["Subnets"]
+
+    if not subnets:
+        click.secho(f"No subnets found for VPC with id: {vpc_id}.", fg="red")
+        raise click.Abort
+
+    public_tag = {"Key": "subnet_type", "Value": "public"}
+    public = [subnet["SubnetId"] for subnet in subnets if public_tag in subnet["Tags"]]
+    private_tag = {"Key": "subnet_type", "Value": "private"}
+    private = [subnet["SubnetId"] for subnet in subnets if private_tag in subnet["Tags"]]
+
+    return public, private
+
+
+def get_cert_arn(session, application, env_name):
+    try:
+        arn = find_https_certificate(session, application, env_name)
+    except:
+        click.secho(
+            f"No certificate found with domain name matching environment {env_name}.", fg="red"
+        )
+        raise click.Abort
+
+    return arn
+
+
+@environment.command()
+@click.option("--vpc-name", hidden=True)
+@click.option("--name", "-n", required=True)
+def generate(name, vpc_name):
+    if vpc_name:
+        click.secho(
+            f"This option is deprecated. Please add the VPC name for your envs to {PLATFORM_CONFIG_FILE}",
+            fg="red",
+        )
+        raise click.Abort
+
+    try:
+        conf = load_and_validate_platform_config()
+    except SchemaError as ex:
+        click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
+        raise click.Abort
+
+    env_config = apply_environment_defaults(conf)["environments"][name]
+    profile_for_environment = env_config.get("accounts", {}).get("deploy", {}).get("name")
+    click.secho(f"Using {profile_for_environment} for this AWS session")
+    session = get_aws_session_or_abort(profile_for_environment)
+
+    _generate_copilot_environment_manifests(name, conf["application"], env_config, session)
+
+
+@environment.command(help="Generate terraform manifest for the specified environment.")
+@click.option(
+    "--name", "-n", required=True, help="The name of the environment to generate a manifest for."
+)
+@click.option(
+    "--terraform-platform-modules-version",
+    help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
+)
+def generate_terraform(name, terraform_platform_modules_version):
+    if not is_terraform_project():
+        click.secho("This is not a terraform project. Exiting.", fg="red")
+        exit(1)
+
+    conf = load_and_validate_platform_config()
+
+    env_config = apply_environment_defaults(conf)["environments"][name]
+    _generate_terraform_environment_manifests(
+        conf["application"], name, env_config, terraform_platform_modules_version
+    )
+
+
+def _generate_copilot_environment_manifests(name, application, env_config, session):
+    env_template = setup_templates().get_template("env/manifest.yml")
+    vpc_name = env_config.get("vpc", None)
+    vpc_id = get_vpc_id(session, name, vpc_name)
+    pub_subnet_ids, priv_subnet_ids = get_subnet_ids(session, vpc_id)
+    cert_arn = get_cert_arn(session, application, name)
+    contents = env_template.render(
+        {
+            "name": name,
+            "vpc_id": vpc_id,
+            "pub_subnet_ids": pub_subnet_ids,
+            "priv_subnet_ids": priv_subnet_ids,
+            "certificate_arn": cert_arn,
+        }
+    )
+    click.echo(mkfile(".", f"copilot/environments/{name}/manifest.yml", contents, overwrite=True))
+
+
+def _generate_terraform_environment_manifests(
+    application, env, env_config, cli_terraform_platform_modules_version
+):
+    env_template = setup_templates().get_template("environments/main.tf")
+
+    terraform_platform_modules_version = _determine_terraform_platform_modules_version(
+        env_config, cli_terraform_platform_modules_version
+    )
+
+    contents = env_template.render(
+        {
+            "application": application,
+            "environment": env,
+            "config": env_config,
+            "terraform_platform_modules_version": terraform_platform_modules_version,
+        }
+    )
+
+    click.echo(mkfile(".", f"terraform/environments/{env}/main.tf", contents, overwrite=True))
+
+
+def _determine_terraform_platform_modules_version(env_conf, cli_terraform_platform_modules_version):
+    cli_terraform_platform_modules_version = cli_terraform_platform_modules_version
+    env_conf_terraform_platform_modules_version = env_conf.get("versions", {}).get(
+        "terraform-platform-modules"
+    )
+    version_preference_order = [
+        cli_terraform_platform_modules_version,
+        env_conf_terraform_platform_modules_version,
+        DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
+    ]
+    return [version for version in version_preference_order if version][0]
+
+
+def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
+    listener_arn = find_https_listener(session, app, env)
+    cert_client = session.client("elbv2")
+    certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
+        "Certificates"
+    ]
+
+    try:
+        certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
+    except StopIteration:
+        raise CertificateNotFoundError()
+
+    return certificate_arn
+
+
+class CertificateNotFoundError(Exception):
+    pass