dbt-platform-helper 10.11.2__tar.gz → 11.0.0__tar.gz

{dbt_platform_helper-10.11.2 → dbt_platform_helper-11.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 10.11.2
+Version: 11.0.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-10.11.2 → dbt_platform_helper-11.0.0}/dbt_platform_helper/COMMANDS.md

@@ -40,7 +40,8 @@
 - [platform-helper notify environment-progress](#platform-helper-notify-environment-progress)
 - [platform-helper notify add-comment](#platform-helper-notify-add-comment)
 - [platform-helper database](#platform-helper-database)
-- [platform-helper database copy](#platform-helper-database-copy)
+- [platform-helper database dump](#platform-helper-database-dump)
+- [platform-helper database load](#platform-helper-database-load)
 - [platform-helper version](#platform-helper-version)
 - [platform-helper version get-platform-helper-for-project](#platform-helper-version-get-platform-helper-for-project)
 
@@ -964,7 +965,7 @@ platform-helper notify add-comment <slack_channel_id> <slack_token>
 ## Usage
 
 ```
-platform-helper database copy 
+platform-helper database (dump|load) 
 ```
 
 ## Options
@@ -974,27 +975,64 @@ platform-helper database copy
 
 ## Commands
 
-- [`copy` ↪](#platform-helper-database-copy)
+- [`dump` ↪](#platform-helper-database-dump)
+- [`load` ↪](#platform-helper-database-load)
 
-# platform-helper database copy
+# platform-helper database dump
 
 [↩ Parent](#platform-helper-database)
 
-    Copy source database to target database.
+    Dump a database into an S3 bucket.
 
 ## Usage
 
 ```
-platform-helper database copy <source_db> <target_db> 
+platform-helper database dump --account-id <account_id> --app <application> 
+                              --env <environment> --database <database> 
+                              --vpc-name <vpc_name> 
 ```
 
-## Arguments
+## Options
+
+- `--account-id <text>`
+
+- `--app <text>`
+
+- `--env <text>`
+
+- `--database <text>`
 
-- `source_db <text>`
-- `target_db <text>`
+- `--vpc-name <text>`
+
+- `--help <boolean>` _Defaults to False._
+  - Show this message and exit.
+
+# platform-helper database load
+
+[↩ Parent](#platform-helper-database)
+
+    Load a database from an S3 bucket.
+
+## Usage
+
+```
+platform-helper database load --account-id <account_id> --app <application> 
+                              --env <environment> --database <database> 
+                              --vpc-name <vpc_name> 
+```
 
 ## Options
 
+- `--account-id <text>`
+
+- `--app <text>`
+
+- `--env <text>`
+
+- `--database <text>`
+
+- `--vpc-name <text>`
+
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
@@ -1002,8 +1040,8 @@ platform-helper database copy <source_db> <target_db>
 
 [↩ Parent](#platform-helper)
 
-    Contains subcommands for getting version information about the current
-    project.
+    Contains subcommands for getting version information about the
+        current project.
 
 ## Usage
 

{dbt_platform_helper-10.11.2 → dbt_platform_helper-11.0.0}/dbt_platform_helper/commands/conduit.py

@@ -11,7 +11,9 @@ from cfn_tools import load_yaml
 
 from dbt_platform_helper.utils.application import Application
 from dbt_platform_helper.utils.application import load_application
-from dbt_platform_helper.utils.aws import update_postgres_parameter_with_master_secret
+from dbt_platform_helper.utils.aws import (
+    get_postgres_connection_data_updated_with_master_secret,
+)
 from dbt_platform_helper.utils.click import ClickDocOptCommand
 from dbt_platform_helper.utils.messages import abort_with_error
 from dbt_platform_helper.utils.platform_config import is_terraform_project
@@ -171,7 +173,7 @@ def create_postgres_admin_task(
         Name=master_secret_name, WithDecryption=True
     )["Parameter"]["Value"]
     connection_string = json.dumps(
-        update_postgres_parameter_with_master_secret(
+        get_postgres_connection_data_updated_with_master_secret(
             session, read_only_secret_name, master_secret_arn
         )
     )
@@ -416,7 +418,6 @@ def conduit(addon_name: str, app: str, env: str, access: str):
 
     try:
         addon_type = get_addon_type(application, env, addon_name)
-        start_conduit(application, env, addon_type, addon_name, access)
     except ParameterNotFoundConduitError:
         click.secho(
             f"""No parameter called "/copilot/applications/{app}/environments/{env}/addons". Try deploying the "{app}" "{env}" environment.""",
@@ -435,6 +436,9 @@ def conduit(addon_name: str, app: str, env: str, access: str):
             fg="red",
         )
         exit(1)
+
+    try:
+        start_conduit(application, env, addon_type, addon_name, access)
     except NoClusterConduitError:
         click.secho(f"""No ECS cluster found for "{app}" in "{env}" environment.""", fg="red")
         exit(1)

dbt_platform_helper-11.0.0/dbt_platform_helper/commands/database.py

@@ -0,0 +1,33 @@
+import click
+
+from dbt_platform_helper.commands.database_helpers import DatabaseCopy
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+
+
+@click.group(chain=True, cls=ClickDocOptGroup)
+def database():
+    pass
+
+
+@database.command(name="dump")
+@click.option("--account-id", type=str, required=True)
+@click.option("--app", type=str, required=True)
+@click.option("--env", type=str, required=True)
+@click.option("--database", type=str, required=True)
+@click.option("--vpc-name", type=str, required=True)
+def dump(account_id, app, env, database, vpc_name):
+    """Dump a database into an S3 bucket."""
+    data_copy = DatabaseCopy(account_id, app, env, database, vpc_name)
+    data_copy.dump()
+
+
+@database.command(name="load")
+@click.option("--account-id", type=str, required=True)
+@click.option("--app", type=str, required=True)
+@click.option("--env", type=str, required=True)
+@click.option("--database", type=str, required=True)
+@click.option("--vpc-name", type=str, required=True)
+def load(account_id, app, env, database, vpc_name):
+    """Load a database from an S3 bucket."""
+    data_copy = DatabaseCopy(account_id, app, env, database, vpc_name)
+    data_copy.load()

dbt_platform_helper-11.0.0/dbt_platform_helper/commands/database_helpers.py

@@ -0,0 +1,145 @@
+import boto3
+import click
+
+from dbt_platform_helper.utils.aws import Vpc
+from dbt_platform_helper.utils.aws import get_aws_session_or_abort
+from dbt_platform_helper.utils.aws import get_connection_string
+from dbt_platform_helper.utils.aws import get_vpc_info_by_name
+
+
+def run_database_copy_task(
+    session: boto3.session.Session,
+    account_id: str,
+    app: str,
+    env: str,
+    database: str,
+    vpc_config: Vpc,
+    is_dump: bool,
+    db_connection_string: str,
+):
+    client = session.client("ecs")
+    action = "dump" if is_dump else "load"
+    response = client.run_task(
+        taskDefinition=f"arn:aws:ecs:eu-west-2:{account_id}:task-definition/{app}-{env}-{database}-{action}",
+        cluster=f"{app}-{env}",
+        capacityProviderStrategy=[
+            {"capacityProvider": "FARGATE", "weight": 1, "base": 0},
+        ],
+        networkConfiguration={
+            "awsvpcConfiguration": {
+                "subnets": vpc_config.subnets,
+                "securityGroups": vpc_config.security_groups,
+                "assignPublicIp": "DISABLED",
+            }
+        },
+        overrides={
+            "containerOverrides": [
+                {
+                    "name": f"{app}-{env}-{database}-{action}",
+                    "environment": [
+                        {"name": "DATA_COPY_OPERATION", "value": action.upper()},
+                        {"name": "DB_CONNECTION_STRING", "value": db_connection_string},
+                    ],
+                }
+            ]
+        },
+    )
+
+    return response.get("tasks", [{}])[0].get("taskArn")
+
+
+class DatabaseCopy:
+    def __init__(
+        self,
+        account_id,
+        app,
+        env,
+        database,
+        vpc_name,
+        get_session_fn=get_aws_session_or_abort,
+        run_database_copy_fn=run_database_copy_task,
+        vpc_config_fn=get_vpc_info_by_name,
+        db_connection_string_fn=get_connection_string,
+        input_fn=click.prompt,
+        echo_fn=click.secho,
+    ):
+        self.account_id = account_id
+        self.app = app
+        self.env = env
+        self.database = database
+        self.vpc_name = vpc_name
+        self.get_session_fn = get_session_fn
+        self.run_database_copy_fn = run_database_copy_fn
+        self.vpc_config_fn = vpc_config_fn
+        self.db_connection_string_fn = db_connection_string_fn
+        self.input_fn = input_fn
+        self.echo_fn = echo_fn
+
+    def _execute_operation(self, is_dump):
+        session = self.get_session_fn()
+        vpc_config = self.vpc_config_fn(session, self.app, self.env, self.vpc_name)
+        database_identifier = f"{self.app}-{self.env}-{self.database}"
+        db_connection_string = self.db_connection_string_fn(
+            session, self.app, self.env, database_identifier
+        )
+        task_arn = self.run_database_copy_fn(
+            session,
+            self.account_id,
+            self.app,
+            self.env,
+            self.database,
+            vpc_config,
+            is_dump,
+            db_connection_string,
+        )
+
+        self.echo_fn(
+            f"Task {task_arn} started. Waiting for it to complete (this may take some time)...",
+            fg="green",
+        )
+        self.tail_logs(is_dump)
+        self.wait_for_task_to_stop(task_arn)
+
+    def dump(self):
+        self._execute_operation(True)
+
+    def load(self):
+        if self.is_confirmed_ready_to_load():
+            self._execute_operation(False)
+
+    def is_confirmed_ready_to_load(self):
+        user_input = self.input_fn(
+            f"Are all tasks using {self.database} in the {self.env} environment stopped? (y/n)"
+        )
+        return user_input.lower().strip() in ["y", "yes"]
+
+    def tail_logs(self, is_dump: bool):
+        action = "dump" if is_dump else "load"
+        log_group_name = f"/ecs/{self.app}-{self.env}-{self.database}-{action}"
+        log_group_arn = f"arn:aws:logs:eu-west-2:{self.account_id}:log-group:{log_group_name}"
+        self.echo_fn(f"Tailing logs for {log_group_name}", fg="yellow")
+        session = self.get_session_fn()
+        response = session.client("logs").start_live_tail(logGroupIdentifiers=[log_group_arn])
+
+        stopped = False
+        for data in response["responseStream"]:
+            if stopped:
+                break
+            results = data.get("sessionUpdate", {}).get("sessionResults", [])
+            for result in results:
+                message = result.get("message")
+
+                if message:
+                    if message.startswith("Stopping data "):
+                        stopped = True
+                    self.echo_fn(message)
+
+    def wait_for_task_to_stop(self, task_arn):
+        self.echo_fn("Waiting for task to complete", fg="yellow")
+        client = self.get_session_fn().client("ecs")
+        waiter = client.get_waiter("tasks_stopped")
+        waiter.wait(
+            cluster=f"{self.app}-{self.env}",
+            tasks=[task_arn],
+            WaiterConfig={"Delay": 6, "MaxAttempts": 300},
+        )

dbt_platform_helper-11.0.0/dbt_platform_helper/commands/version.py

@@ -0,0 +1,38 @@
+import click
+
+from dbt_platform_helper.utils.click import ClickDocOptGroup
+from dbt_platform_helper.utils.platform_config import get_environment_pipeline_names
+from dbt_platform_helper.utils.versioning import get_required_platform_helper_version
+
+
+class VersionCommand:
+    def __init__(self):
+        self.command_group = self.version
+        self.command = self.get_platform_helper_for_project
+
+    @click.group(chain=True, cls=ClickDocOptGroup)
+    def version():
+        """Contains subcommands for getting version information about the
+        current project."""
+
+    @version.command(help="Print the version of platform-tools required by the current project")
+    @click.option(
+        "--pipeline",
+        required=False,
+        type=click.Choice(get_environment_pipeline_names()),
+        help="Take into account platform-tools version overrides in the specified pipeline",
+    )
+    def get_platform_helper_for_project(pipeline):
+        """
+        Version precedence is in this order:
+            - if the --pipeline option is supplied, the version in 'platform-config.yml' in:
+                environment_pipelines:
+                    <pipeline>:
+                        ...
+                        versions:
+                            platform-helper
+            - The version from default_versions/platform-helper in 'platform-config.yml'
+            - Fall back on the version in the deprecated '.platform-helper-version' file
+        """
+        required_version = get_required_platform_helper_version(pipeline)
+        click.secho(required_version)

{dbt_platform_helper-10.11.2 → dbt_platform_helper-11.0.0}/dbt_platform_helper/exceptions.py

@@ -2,6 +2,10 @@ class ValidationException(Exception):
     pass
 
 
+class AWSException(Exception):
+    pass
+
+
 class IncompatibleMajorVersion(ValidationException):
     def __init__(self, app_version: str, check_version: str):
         super().__init__()

{dbt_platform_helper-10.11.2 → dbt_platform_helper-11.0.0}/dbt_platform_helper/utils/aws.py

@@ -11,6 +11,7 @@ import click
 import yaml
 from boto3 import Session
 
+from dbt_platform_helper.exceptions import AWSException
 from dbt_platform_helper.exceptions import ValidationException
 
 SSM_BASE_PATH = "/copilot/{app}/{env}/secrets/"
@@ -321,7 +322,7 @@ def get_load_balancer_configuration(
     return response
 
 
-def update_postgres_parameter_with_master_secret(session, parameter_name, secret_arn):
+def get_postgres_connection_data_updated_with_master_secret(session, parameter_name, secret_arn):
     ssm_client = session.client("ssm")
     secrets_manager_client = session.client("secretsmanager")
     response = ssm_client.get_parameter(Name=parameter_name, WithDecryption=True)
@@ -336,3 +337,74 @@ def update_postgres_parameter_with_master_secret(session, parameter_name, secret
     parameter_data["password"] = urllib.parse.quote(secret_value["password"])
 
     return parameter_data
+
+
+def get_connection_string(
+    session: Session,
+    app: str,
+    env: str,
+    db_identifier: str,
+    connection_data_fn=get_postgres_connection_data_updated_with_master_secret,
+) -> str:
+    addon_name = db_identifier.split(f"{app}-{env}-", 1)[1]
+    normalised_addon_name = addon_name.replace("-", "_").upper()
+    connection_string_parameter = (
+        f"/copilot/{app}/{env}/secrets/{normalised_addon_name}_READ_ONLY_USER"
+    )
+    master_secret_name = f"/copilot/{app}/{env}/secrets/{normalised_addon_name}_RDS_MASTER_ARN"
+    master_secret_arn = session.client("ssm").get_parameter(
+        Name=master_secret_name, WithDecryption=True
+    )["Parameter"]["Value"]
+
+    conn = connection_data_fn(session, connection_string_parameter, master_secret_arn)
+
+    return f"postgres://{conn['username']}:{conn['password']}@{conn['host']}:{conn['port']}/{conn['dbname']}"
+
+
+class Vpc:
+    def __init__(self, subnets, security_groups):
+        self.subnets = subnets
+        self.security_groups = security_groups
+
+
+def get_vpc_info_by_name(session, app, env, vpc_name):
+    ec2_client = session.client("ec2")
+    vpc_response = ec2_client.describe_vpcs(Filters=[{"Name": "tag:Name", "Values": [vpc_name]}])
+
+    matching_vpcs = vpc_response.get("Vpcs", [])
+
+    if not matching_vpcs:
+        raise AWSException(f"VPC not found for name '{vpc_name}'")
+
+    vpc_id = vpc_response["Vpcs"][0].get("VpcId")
+
+    if not vpc_id:
+        raise AWSException(f"VPC id not present in vpc '{vpc_name}'")
+
+    ec2_resource = session.resource("ec2")
+    vpc = ec2_resource.Vpc(vpc_id)
+
+    route_tables = ec2_client.describe_route_tables(
+        Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
+    )["RouteTables"]
+
+    subnets = []
+    for route_table in route_tables:
+        private_routes = [route for route in route_table["Routes"] if "NatGatewayId" in route]
+        if not private_routes:
+            continue
+        for association in route_table["Associations"]:
+            if "SubnetId" in association:
+                subnet_id = association["SubnetId"]
+                subnets.append(subnet_id)
+
+    if not subnets:
+        raise AWSException(f"No private subnets found in vpc '{vpc_name}'")
+
+    tag_value = {"Key": "Name", "Value": f"copilot-{app}-{env}-env"}
+    sec_groups = [sg.id for sg in vpc.security_groups.all() if sg.tags and tag_value in sg.tags]
+
+    if not sec_groups:
+        raise AWSException(f"No matching security groups found in vpc '{vpc_name}'")
+
+    return Vpc(subnets, sec_groups)

dbt_platform_helper-11.0.0/dbt_platform_helper/utils/platform_config.py

@@ -0,0 +1,32 @@
+import yaml
+
+from pathlib import Path
+
+from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
+
+
+def _read_config_file_contents():
+    if Path(PLATFORM_CONFIG_FILE).exists():
+        return Path(PLATFORM_CONFIG_FILE).read_text()
+
+
+def load_unvalidated_config_file():
+    file_contents = _read_config_file_contents()
+    if not file_contents:
+        return {}
+    try:
+        return yaml.safe_load(file_contents)
+    except yaml.parser.ParserError:
+        return {}
+
+
+def get_environment_pipeline_names():
+    pipelines_config = load_unvalidated_config_file().get("environment_pipelines")
+    if pipelines_config:
+        return pipelines_config.keys()
+    return {}
+
+
+def is_terraform_project() -> bool:
+    config = yaml.safe_load(Path(PLATFORM_CONFIG_FILE).read_text())
+    return not config.get("legacy_project", False)

{dbt_platform_helper-10.11.2 → dbt_platform_helper-11.0.0}/dbt_platform_helper/utils/validation.py

@@ -222,6 +222,9 @@ RETENTION_POLICY = Or(
         Or("days", "years", only_one=True): int,
     },
 )
+
+DATABASE_COPY = {"from": ENV_NAME, "to": ENV_NAME}
+
 POSTGRES_DEFINITION = {
     "type": "postgres",
     "version": NUMBER,
@@ -239,6 +242,7 @@ POSTGRES_DEFINITION = {
             Optional("backup_retention_days"): int_between(1, 35),
         }
     },
+    Optional("database_copy"): [DATABASE_COPY],
     Optional("objects"): [
         {
             "key": str,
@@ -387,10 +391,27 @@ ALB_DEFINITION = {
     Optional("environments"): {
         ENV_NAME: Or(
             {
+                Optional("additional_address_list"): list,
+                Optional("allowed_methods"): list,
+                Optional("cached_methods"): list,
+                Optional("cdn_compress"): bool,
+                Optional("cdn_domains_list"): dict,
+                Optional("cdn_geo_locations"): list,
+                Optional("cdn_geo_restriction_type"): str,
+                Optional("cdn_logging_bucket"): str,
+                Optional("cdn_logging_bucket_prefix"): str,
+                Optional("default_waf"): str,
                 Optional("domain_prefix"): str,
+                Optional("enable_logging"): bool,
                 Optional("env_root"): str,
-                Optional("cdn_domains_list"): dict,
-                Optional("additional_address_list"): list,
+                Optional("forwarded_values_forward"): str,
+                Optional("forwarded_values_headers"): list,
+                Optional("forwarded_values_query_string"): bool,
+                Optional("origin_protocol_policy"): str,
+                Optional("origin_ssl_protocols"): list,
+                Optional("viewer_certificate_minimum_protocol_version"): str,
+                Optional("viewer_certificate_ssl_support_method"): str,
+                Optional("viewer_protocol_policy"): str,
             },
             None,
         )
@@ -531,10 +552,62 @@ def validate_platform_config(config, disable_aws_validation=False):
     _validate_environment_pipelines(enriched_config)
     _validate_environment_pipelines_triggers(enriched_config)
     _validate_codebase_pipelines(enriched_config)
+    validate_database_copy_section(enriched_config)
     if not disable_aws_validation:
         _validate_s3_bucket_uniqueness(enriched_config)
 
 
+def validate_database_copy_section(config):
+    extensions = config.get("extensions", {})
+    if not extensions:
+        return
+
+    postgres_extensions = {
+        key: ext for key, ext in extensions.items() if ext.get("type", None) == "postgres"
+    }
+
+    if not postgres_extensions:
+        return
+
+    errors = []
+
+    for extension_name, extension in postgres_extensions.items():
+        database_copy_sections = extension.get("database_copy", [])
+
+        if not database_copy_sections:
+            return
+
+        all_environments = [env for env in config.get("environments", {}).keys() if not env == "*"]
+        all_envs_string = ", ".join(all_environments)
+
+        for section in database_copy_sections:
+            from_env = section["from"]
+            to_env = section["to"]
+
+            if from_env == to_env:
+                errors.append(
+                    f"database_copy 'to' and 'from' cannot be the same environment in extension '{extension_name}'."
+                )
+
+            if "prod" in to_env:
+                errors.append(
+                    f"Copying to a prod environment is not supported: database_copy 'to' cannot be '{to_env}' in extension '{extension_name}'."
+                )
+
+            if from_env not in all_environments:
+                errors.append(
+                    f"database_copy 'from' parameter must be a valid environment ({all_envs_string}) but was '{from_env}' in extension '{extension_name}'."
+                )
+
+            if to_env not in all_environments:
+                errors.append(
+                    f"database_copy 'to' parameter must be a valid environment ({all_envs_string}) but was '{to_env}' in extension '{extension_name}'."
+                )
+
+    if errors:
+        abort_with_error("\n".join(errors))
+
+
 def _validate_environment_pipelines(config):
     bad_pipelines = {}
     for pipeline_name, pipeline in config.get("environment_pipelines", {}).items():
@@ -615,6 +688,8 @@ def load_and_validate_platform_config(
         return conf
     except ParserError:
         abort_with_error(f"{PLATFORM_CONFIG_FILE} is not valid YAML")
+    except SchemaError as e:
+        abort_with_error(f"Schema error in {PLATFORM_CONFIG_FILE}. {e}")
 
 
 def config_file_check(path=PLATFORM_CONFIG_FILE):