dbt-platform-helper 11.3.0__tar.gz → 12.0.0__tar.gz

{dbt_platform_helper-11.3.0 → dbt_platform_helper-12.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 11.3.0
+Version: 12.0.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-11.3.0 → dbt_platform_helper-12.0.0}/dbt_platform_helper/COMMANDS.md

@@ -4,13 +4,6 @@
 - [platform-helper application](#platform-helper-application)
 - [platform-helper application container-stats](#platform-helper-application-container-stats)
 - [platform-helper application task-stats](#platform-helper-application-task-stats)
-- [platform-helper cdn](#platform-helper-cdn)
-- [platform-helper cdn assign](#platform-helper-cdn-assign)
-- [platform-helper cdn delete](#platform-helper-cdn-delete)
-- [platform-helper cdn list](#platform-helper-cdn-list)
-- [platform-helper check-cloudformation](#platform-helper-check-cloudformation)
-- [platform-helper check-cloudformation lint](#platform-helper-check-cloudformation-lint)
-- [platform-helper check-cloudformation check-security](#platform-helper-check-cloudformation-check-security)
 - [platform-helper codebase](#platform-helper-codebase)
 - [platform-helper codebase prepare](#platform-helper-codebase-prepare)
 - [platform-helper codebase list](#platform-helper-codebase-list)
@@ -22,9 +15,6 @@
 - [platform-helper config aws](#platform-helper-config-aws)
 - [platform-helper copilot](#platform-helper-copilot)
 - [platform-helper copilot make-addons](#platform-helper-copilot-make-addons)
-- [platform-helper domain](#platform-helper-domain)
-- [platform-helper domain configure](#platform-helper-domain-configure)
-- [platform-helper domain assign](#platform-helper-domain-assign)
 - [platform-helper environment](#platform-helper-environment)
 - [platform-helper environment offline](#platform-helper-environment-offline)
 - [platform-helper environment online](#platform-helper-environment-online)
@@ -64,14 +54,11 @@ platform-helper <command> [--version]
 ## Commands
 
 - [`application` ↪](#platform-helper-application)
-- [`cdn` ↪](#platform-helper-cdn)
-- [`check-cloudformation` ↪](#platform-helper-check-cloudformation)
 - [`codebase` ↪](#platform-helper-codebase)
 - [`conduit` ↪](#platform-helper-conduit)
 - [`config` ↪](#platform-helper-config)
 - [`copilot` ↪](#platform-helper-copilot)
 - [`database` ↪](#platform-helper-database)
-- [`domain` ↪](#platform-helper-domain)
 - [`environment` ↪](#platform-helper-environment)
 - [`generate` ↪](#platform-helper-generate)
 - [`notify` ↪](#platform-helper-notify)
@@ -83,7 +70,7 @@ platform-helper <command> [--version]
 
 [↩ Parent](#platform-helper)
 
-    Application metrics.
+    [DEPRECATED] Application metrics.
 
 ## Usage
 
@@ -105,7 +92,7 @@ platform-helper application (container-stats|task-stats)
 
 [↩ Parent](#platform-helper-application)
 
-    Command to get application container level metrics.
+    [DEPRECATED] Command to get application container level metrics.
 
 ## Usage
 
@@ -131,7 +118,7 @@ platform-helper application container-stats --env <environment> --app <applicati
 
 [↩ Parent](#platform-helper-application)
 
-    Command to get application task level metrics.
+    [DEPRECATED] Command to get application task level metrics.
 
 ## Usage
 
@@ -152,171 +139,6 @@ platform-helper application task-stats --env <environment> --app <application> [
 
 - `--network <boolean>` _Defaults to False._
 
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-# platform-helper cdn
-
-[↩ Parent](#platform-helper)
-
-## Usage
-
-```
-platform-helper cdn (assign|delete|list) 
-```
-
-## Options
-
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-## Commands
-
-- [`assign` ↪](#platform-helper-cdn-assign)
-- [`delete` ↪](#platform-helper-cdn-delete)
-- [`list` ↪](#platform-helper-cdn-list)
-
-# platform-helper cdn assign
-
-[↩ Parent](#platform-helper-cdn)
-
-    Assigns a CDN domain name to application loadbalancer.
-
-## Usage
-
-```
-platform-helper cdn assign --project-profile <project_profile> --env <environment> 
-                           --app <application> --svc <service> 
-```
-
-## Options
-
-- `--project-profile <text>`
-  - AWS account profile name for certificates account
-- `--env <text>`
-  - AWS Copilot environment name
-- `--app <text>`
-  - Application Name
-- `--svc <text>`
-  - Service Name
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-# platform-helper cdn delete
-
-[↩ Parent](#platform-helper-cdn)
-
-    Assigns a CDN domain name to application loadbalancer.
-
-## Usage
-
-```
-platform-helper cdn delete --project-profile <project_profile> --env <environment> 
-                           --app <application> --svc <service> 
-```
-
-## Options
-
-- `--project-profile <text>`
-  - AWS account profile name for certificates account
-- `--env <text>`
-  - AWS Copilot environment name
-- `--app <text>`
-  - Application Name
-- `--svc <text>`
-  - Service Name
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-# platform-helper cdn list
-
-[↩ Parent](#platform-helper-cdn)
-
-    List CDN domain name attached to application loadbalancer.
-
-## Usage
-
-```
-platform-helper cdn list --project-profile <project_profile> --env <environment> 
-                         --app <application> --svc <service> 
-```
-
-## Options
-
-- `--project-profile <text>`
-  - AWS account profile name for certificates account
-- `--env <text>`
-  - AWS Copilot environment name
-- `--app <text>`
-  - Application Name
-- `--svc <text>`
-  - Service Name
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-# platform-helper check-cloudformation
-
-[↩ Parent](#platform-helper)
-
-    Runs the checks passed in the command arguments.
-
-    If no argument is passed, it will run all the checks.
-
-## Usage
-
-```
-platform-helper check-cloudformation (lint|check-security) 
-                                     [-d <directory>] 
-```
-
-## Options
-
-- `-d
---directory <text>` _Defaults to copilot._
-
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-## Commands
-
-- [`check-security` ↪](#platform-helper-check-cloudformation-check-security)
-- [`lint` ↪](#platform-helper-check-cloudformation-lint)
-
-# platform-helper check-cloudformation lint
-
-[↩ Parent](#platform-helper-check-cloudformation)
-
-    Runs cfn-lint against the generated CloudFormation templates.
-
-## Usage
-
-```
-platform-helper check-cloudformation lint [-d <directory>] 
-```
-
-## Options
-
-- `-d
---directory <text>` _Defaults to copilot._
-
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-# platform-helper check-cloudformation check-security
-
-[↩ Parent](#platform-helper-check-cloudformation)
-
-## Usage
-
-```
-platform-helper check-cloudformation check-security [-d <directory>] 
-```
-
-## Options
-
-- `-d
---directory <text>` _Defaults to copilot._
-
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
@@ -558,77 +380,6 @@ platform-helper copilot make-addons
 - `--help <boolean>` _Defaults to False._
   - Show this message and exit.
 
-# platform-helper domain
-
-[↩ Parent](#platform-helper)
-
-## Usage
-
-```
-platform-helper domain (configure|assign) 
-```
-
-## Options
-
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-## Commands
-
-- [`assign` ↪](#platform-helper-domain-assign)
-- [`configure` ↪](#platform-helper-domain-configure)
-
-# platform-helper domain configure
-
-[↩ Parent](#platform-helper-domain)
-
-    Creates subdomains if they do not exist and then creates certificates for
-    them.
-
-## Usage
-
-```
-platform-helper domain configure --project-profile <project_profile> 
-                                 --env <environment> 
-```
-
-## Options
-
-- `--project-profile <text>`
-  - AWS account profile name for certificates account
-- `--env <text>`
-  - AWS Copilot environment name
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
-# platform-helper domain assign
-
-[↩ Parent](#platform-helper-domain)
-
-    Assigns the load balancer for a service to its domain name.
-
-## Usage
-
-```
-platform-helper domain assign --app <application> --env <environment> --svc <service> 
-                              --domain-profile (dev|live) --project-profile <project_profile> 
-```
-
-## Options
-
-- `--app <text>`
-  - Application Name
-- `--env <text>`
-  - Environment
-- `--svc <text>`
-  - Service Name
-- `--domain-profile <choice>`
-  - AWS account profile name for Route53 domains account
-- `--project-profile <text>`
-  - AWS account profile name for application account
-- `--help <boolean>` _Defaults to False._
-  - Show this message and exit.
-
 # platform-helper environment
 
 [↩ Parent](#platform-helper)

dbt_platform_helper-12.0.0/dbt_platform_helper/addons-template-map.yml

@@ -0,0 +1,29 @@
+# This file maps addon types to svc level templates
+
+# explanation:
+
+# {addons-type}:
+#   svc:
+#     - template: path/to/template.yml
+
+redis: {}
+postgres: {}
+opensearch: {}
+s3:
+  svc:
+    - template: addons/svc/s3-policy.yml
+s3-policy:
+  svc:
+    - template: addons/svc/s3-policy.yml
+appconfig-ipfilter:
+  svc:
+    - template: addons/svc/appconfig-ipfilter.yml
+subscription-filter:
+  svc:
+    - template: addons/svc/subscription-filter.yml
+monitoring: {}
+vpc: {}
+alb: {}
+prometheus-policy:
+  svc:
+    - template: addons/svc/prometheus-policy.yml

{dbt_platform_helper-11.3.0 → dbt_platform_helper-12.0.0}/dbt_platform_helper/commands/application.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+# application commands are deprecated, do not spend time refactoring them
+
 import time
 from datetime import datetime
 from datetime import timedelta
@@ -45,7 +47,6 @@ def get_query_results(env, app, query_string, timeout):
 
     # Container stats are 5 mins behind realtime.
     date_time = datetime.now()
-    end_time = int(date_time.timestamp())
     start_time = int((date_time - timedelta(minutes=5)).timestamp())
     end_time = int((date_time - timedelta(minutes=4)).timestamp())
 
@@ -92,19 +93,19 @@ def get_query_results(env, app, query_string, timeout):
     return cpu_response
 
 
-@click.group(chain=True, cls=ClickDocOptGroup)
+@click.group(chain=True, cls=ClickDocOptGroup, deprecated=True)
 def application():
-    """Application metrics."""
+    """[DEPRECATED] Application metrics."""
     check_platform_helper_version_needs_update()
 
 
-@application.command()
+@application.command(deprecated=True)
 @click.option("--env", type=str, required=True)
 @click.option("--app", type=str, required=True)
 @click.option("--storage", is_flag=True)
 @click.option("--network", is_flag=True)
 def container_stats(env, app, storage, network):
-    """Command to get application container level metrics."""
+    """[DEPRECATED] Command to get application container level metrics."""
 
     # Query string to get the required container stats
     query_string = "stats max(CpuUtilized), max(MemoryUtilized) by TaskId, ContainerName, TaskDefinitionFamily, TaskDefinitionRevision, Image, StorageReadBytes, StorageWriteBytes, NetworkRxPackets, NetworkTxBytes | filter Type='Container' | sort TaskId, ContainerName desc"
@@ -218,14 +219,14 @@ def container_stats(env, app, storage, network):
         index = index + 1
 
 
-@application.command()
+@application.command(deprecated=True)
 @click.option("--env", type=str, required=True)
 @click.option("--app", type=str, required=True)
 @click.option("--disk", is_flag=True)
 @click.option("--storage", is_flag=True)
 @click.option("--network", is_flag=True)
 def task_stats(env, app, disk, storage, network):
-    """Command to get application task level metrics."""
+    """[DEPRECATED] Command to get application task level metrics."""
 
     # Query string to get the required container stats
     query_string = "stats max(CpuUtilized), max(MemoryUtilized), max(EphemeralStorageUtilized) by TaskId, TaskDefinitionFamily, TaskDefinitionRevision, StorageReadBytes, StorageWriteBytes, NetworkRxPackets, NetworkTxBytes, KnownStatus | filter Type='Task' | sort TaskId desc"

{dbt_platform_helper-11.3.0 → dbt_platform_helper-12.0.0}/dbt_platform_helper/commands/conduit.py

@@ -16,7 +16,6 @@ from dbt_platform_helper.utils.aws import (
 )
 from dbt_platform_helper.utils.click import ClickDocOptCommand
 from dbt_platform_helper.utils.messages import abort_with_error
-from dbt_platform_helper.utils.platform_config import is_terraform_project
 from dbt_platform_helper.utils.versioning import (
     check_platform_helper_version_needs_update,
 )
@@ -54,8 +53,6 @@ class AddonNotFoundConduitError(ConduitError):
 CONDUIT_DOCKER_IMAGE_LOCATION = "public.ecr.aws/uktrade/tunnel"
 CONDUIT_ADDON_TYPES = [
     "opensearch",
-    "rds-postgres",
-    "aurora-postgres",
     "postgres",
     "redis",
 ]
@@ -205,7 +202,7 @@ def create_addon_client_task(
             secret_name += "_READ_ONLY_USER"
         elif access == "write":
             secret_name += "_APPLICATION_USER"
-        elif access == "admin" and is_terraform_project():
+        elif access == "admin":
             create_postgres_admin_task(app, env, secret_name, task_name, addon_type, addon_name)
             return
     elif addon_type == "redis" or addon_type == "opensearch":

{dbt_platform_helper-11.3.0 → dbt_platform_helper-12.0.0}/dbt_platform_helper/commands/copilot.py

@@ -2,8 +2,6 @@
 
 import copy
 import json
-from os import listdir
-from os.path import isfile
 from pathlib import Path
 from pathlib import PosixPath
 
@@ -17,7 +15,6 @@ from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.files import generate_override_files
 from dbt_platform_helper.utils.files import mkfile
-from dbt_platform_helper.utils.platform_config import is_terraform_project
 from dbt_platform_helper.utils.template import camel_case
 from dbt_platform_helper.utils.template import setup_templates
 from dbt_platform_helper.utils.validation import config_file_check
@@ -244,7 +241,6 @@ def make_addons():
     """Generate addons CloudFormation for each environment."""
     output_dir = Path(".").absolute()
     config_file_check()
-    is_terraform = is_terraform_project()
 
     templates = setup_templates()
     config = _get_config()
@@ -255,20 +251,14 @@ def make_addons():
     with open(PACKAGE_DIR / "addons-template-map.yml") as fd:
         addon_template_map = yaml.safe_load(fd)
 
-    if is_terraform:
-        click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
-    else:
-        click.echo("\n>>> Generating addons CloudFormation\n")
+    click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
 
     env_path = Path(f"copilot/environments/")
     env_addons_path = env_path / "addons"
     env_overrides_path = env_path / "overrides"
 
-    (output_dir / env_addons_path).mkdir(parents=True, exist_ok=True)
-
     _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path)
-    _generate_env_overrides(output_dir, is_terraform)
-    custom_resources = _get_custom_resources()
+    _generate_env_overrides(output_dir)
 
     svc_names = list_copilot_local_services()
     base_path = Path(".")
@@ -276,26 +266,17 @@ def make_addons():
         _generate_svc_overrides(base_path, templates, svc_name)
 
     services = []
-    has_addons_parameters = False
-    has_postgres_addon = False
     for addon_name, addon_config in config.items():
         print(f">>>>>>>>> {addon_name}")
         addon_type = addon_config.pop("type")
         environments = addon_config.pop("environments")
         if addon_template_map[addon_type].get("requires_addons_parameters", False):
-            has_addons_parameters = True
-        if addon_type in ["aurora-postgres", "postgres"]:
-            has_postgres_addon = True
-
-        for environment_name, environment_config in environments.items():
-            if not environment_config.get("deletion_policy"):
-                environments[environment_name]["deletion_policy"] = addon_config.get(
-                    "deletion_policy", "Delete"
-                )
+            pass
+        if addon_type in ["postgres"]:
+            pass
 
         environment_addon_config = {
             "addon_type": addon_type,
-            "custom_resources": custom_resources,
             "environments": environments,
             "name": addon_config.get("name", None) or addon_name,
             "prefix": camel_case(addon_name),
@@ -318,28 +299,13 @@ def make_addons():
         if addon_type in ["s3", "s3-policy"]:
             if config[addon_name].get("serve_static_content"):
                 continue
-            if is_terraform:
-                s3_kms_arns = _get_s3_kms_alias_arns(session, application_name, environments)
-                for environment_name in environments:
-                    environments[environment_name]["kms_key_arn"] = s3_kms_arns.get(
-                        environment_name, "kms-key-not-found"
-                    )
-            else:
-                service_addon_config["kms_key_reference"] = service_addon_config["prefix"].rsplit(
-                    "BucketAccess", 1
-                )[0]
-
-        if not is_terraform:
-            _generate_env_addons(
-                addon_name,
-                addon_template_map,
-                config.items(),
-                env_addons_path,
-                environment_addon_config,
-                output_dir,
-                templates,
-                log_destination_arns,
-            )
+
+            s3_kms_arns = _get_s3_kms_alias_arns(session, application_name, environments)
+            for environment_name in environments:
+                environments[environment_name]["kms_key_arn"] = s3_kms_arns.get(
+                    environment_name, "kms-key-not-found"
+                )
+
         _generate_service_addons(
             addon_config,
             addon_name,
@@ -349,20 +315,6 @@ def make_addons():
             service_addon_config,
             templates,
             log_destination_arns,
-            is_terraform=is_terraform,
-        )
-
-        if addon_type in ["aurora-postgres", "postgres"] and not is_terraform:
-            click.secho(
-                "\nNote: The key DATABASE_CREDENTIALS may need to be changed to match your Django settings configuration.",
-                fg="yellow",
-            )
-
-    if has_addons_parameters and not is_terraform:
-        template = templates.get_template("addons/env/addons.parameters.yml")
-        contents = template.render({"has_postgres_addon": has_postgres_addon})
-        click.echo(
-            mkfile(output_dir, env_addons_path / "addons.parameters.yml", contents, overwrite=True)
         )
 
     click.echo(templates.get_template("addon-instructions.txt").render(services=services))
@@ -375,8 +327,8 @@ def _get_config():
     return config
 
 
-def _generate_env_overrides(output_dir, is_terraform):
-    path = "templates/env/terraform-overrides" if is_terraform else "templates/env/overrides"
+def _generate_env_overrides(output_dir):
+    path = "templates/env/terraform-overrides"
     click.echo("\n>>> Generating Environment overrides\n")
     overrides_path = output_dir.joinpath(f"copilot/environments/overrides")
     overrides_path.mkdir(parents=True, exist_ok=True)
@@ -384,33 +336,6 @@ def _generate_env_overrides(output_dir, is_terraform):
     generate_override_files(Path("."), template_overrides_path, overrides_path)
 
 
-def _generate_env_addons(
-    addon_name,
-    addon_template_map,
-    addons,
-    env_addons_path,
-    environment_addon_config,
-    output_dir,
-    templates,
-    log_destination_arns,
-):
-    # generate env addons
-    addon_type = environment_addon_config["addon_type"]
-    for addon in addon_template_map[addon_type].get("env", []):
-        template = templates.get_template(addon["template"])
-        contents = template.render(
-            {
-                "addon_config": environment_addon_config,
-                "addons": addons,
-                "log_destination": log_destination_arns,
-            }
-        )
-
-        filename = addon.get("filename", f"{addon_name}.yml")
-
-        click.echo(mkfile(output_dir, env_addons_path / filename, contents, overwrite=True))
-
-
 def _generate_service_addons(
     addon_config,
     addon_name,
@@ -420,7 +345,6 @@ def _generate_service_addons(
     service_addon_config,
     templates,
     log_destination_arns,
-    is_terraform,
 ):
     # generate svc addons
     for addon in addon_template_map[addon_type].get("svc", []):
@@ -433,7 +357,6 @@ def _generate_service_addons(
                 {
                     "addon_config": service_addon_config,
                     "log_destination": log_destination_arns,
-                    "is_terraform": is_terraform,
                 }
             )
 
@@ -443,25 +366,6 @@ def _generate_service_addons(
             click.echo(mkfile(output_dir, service_path / filename, contents, overwrite=True))
 
 
-def _get_custom_resources():
-    custom_resources = {}
-    custom_resource_path = (Path(__file__).parent / "../custom_resources/").resolve()
-    for file in listdir(custom_resource_path):
-        file_path = custom_resource_path / file
-        if isfile(file_path) and file_path.name.endswith(".py") and file_path.name != "__init__.py":
-            custom_resource_contents = file_path.read_text()
-
-            def file_with_formatting_options(padding=0):
-                lines = [
-                    (" " * padding) + line if line.strip() else line.strip()
-                    for line in custom_resource_contents.splitlines(True)
-                ]
-                return "".join(lines)
-
-            custom_resources[file_path.name.rstrip(".py")] = file_with_formatting_options
-    return custom_resources
-
-
 def _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path):
     def _rmdir(path):
         if not path.exists():

{dbt_platform_helper-11.3.0 → dbt_platform_helper-12.0.0}/dbt_platform_helper/commands/environment.py

@@ -10,7 +10,6 @@ from dbt_platform_helper.utils.aws import get_aws_session_or_abort
 from dbt_platform_helper.utils.click import ClickDocOptGroup
 from dbt_platform_helper.utils.files import apply_environment_defaults
 from dbt_platform_helper.utils.files import mkfile
-from dbt_platform_helper.utils.platform_config import is_terraform_project
 from dbt_platform_helper.utils.template import setup_templates
 from dbt_platform_helper.utils.validation import load_and_validate_platform_config
 from dbt_platform_helper.utils.versioning import (
@@ -135,10 +134,6 @@ def generate(name, vpc_name):
     help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
 )
 def generate_terraform(name, terraform_platform_modules_version):
-    if not is_terraform_project():
-        click.secho("This is not a terraform project. Exiting.", fg="red")
-        exit(1)
-
     conf = load_and_validate_platform_config()
 
     env_config = apply_environment_defaults(conf)["environments"][name]