dbt-platform-helper 11.0.1__py3-none-any.whl → 11.2.0__py3-none-any.whl

dbt_platform_helper/utils/validation.py

@@ -1,4 +1,5 @@
 import ipaddress
+import os
 import re
 from pathlib import Path
 
@@ -11,6 +12,8 @@ from schema import Regex
 from schema import Schema
 from schema import SchemaError
 from yaml.parser import ParserError
+from yamllint import config
+from yamllint import linter
 
 from dbt_platform_helper.constants import CODEBASE_PIPELINES_KEY
 from dbt_platform_helper.constants import ENVIRONMENTS_KEY
@@ -677,6 +680,25 @@ def _validate_environment_pipelines_triggers(config):
         abort_with_error(error_message + "\n  ".join(errors))
 
 
+def lint_yaml_for_duplicate_keys(file_path):
+    lint_yaml_config = """
+rules:
+  key-duplicates: enable
+"""
+    yaml_config = config.YamlLintConfig(lint_yaml_config)
+
+    with open(file_path, "r") as yaml_file:
+        file_contents = yaml_file.read()
+        results = linter.run(file_contents, yaml_config)
+
+    parsed_results = [
+        "\t" + f"Line {result.line}: {result.message}".replace(" in mapping (key-duplicates)", "")
+        for result in results
+    ]
+
+    return parsed_results
+
+
 def load_and_validate_platform_config(
     path=PLATFORM_CONFIG_FILE, disable_aws_validation=False, disable_file_check=False
 ):
@@ -684,6 +706,13 @@ def load_and_validate_platform_config(
         config_file_check(path)
     try:
         conf = yaml.safe_load(Path(path).read_text())
+        duplicate_keys = lint_yaml_for_duplicate_keys(path)
+        if duplicate_keys:
+            abort_with_error(
+                "Duplicate keys found in platform-config:"
+                + os.linesep
+                + os.linesep.join(duplicate_keys)
+            )
         validate_platform_config(conf, disable_aws_validation)
         return conf
     except ParserError:

{dbt_platform_helper-11.0.1.dist-info → dbt_platform_helper-11.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 11.0.1
+Version: 11.2.0
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team
@@ -34,6 +34,7 @@ Requires-Dist: schema (==0.7.5)
 Requires-Dist: semver (>=3.0.2,<4.0.0)
 Requires-Dist: slack-sdk (>=3.27.1,<4.0.0)
 Requires-Dist: tomlkit (>=0.12.2,<0.13.0)
+Requires-Dist: yamllint (>=1.35.1,<2.0.0)
 Description-Content-Type: text/markdown
 
 # DBT Platform Helper

{dbt_platform_helper-11.0.1.dist-info → dbt_platform_helper-11.2.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-dbt_platform_helper/COMMANDS.md,sha256=u9g0Zf1RYk-pzE4hjSKg15EMpXglzSA9qEg3QWv559M,23843
+dbt_platform_helper/COMMANDS.md,sha256=e4_kYpZLidhQB18sYT7O1FQYDljicJvoWGhYz-1RkeY,27580
 dbt_platform_helper/README.md,sha256=B0qN2_u_ASqqgkGDWY2iwNGZt_9tUgMb9XqtaTuzYjw,1530
 dbt_platform_helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbt_platform_helper/addon-plans.yml,sha256=O46a_ODsGG9KXmQY_1XbSGqrpSaHSLDe-SdROzHx8Go,4545
@@ -10,21 +10,24 @@ dbt_platform_helper/commands/codebase.py,sha256=NchJzH-yxv5mXCe2rPyXVNHmXGEvjFUv
 dbt_platform_helper/commands/conduit.py,sha256=BC2cxIX14e7_BEdvuMiq8z85k_4OWL-RwX0QJFJrUak,15436
 dbt_platform_helper/commands/config.py,sha256=NOHea7OAjrl6XHlW6HMLn0m0T5lFPyNH3HXoyCOWsJk,12070
 dbt_platform_helper/commands/copilot.py,sha256=euid0FTlVtwKmBQ6vxt_HxtBdRYiVQvb-9CyrK1-MWc,16724
-dbt_platform_helper/commands/database.py,sha256=r7GXfyt_1Wj7KlGilnPopfHXuJ8Q7x35RVh9XKiAZUY,1178
-dbt_platform_helper/commands/database_helpers.py,sha256=7TE6ojEmrIcJ-XQf3MPKeRSuTO5GRBwzm8IoWYmjE3M,4967
+dbt_platform_helper/commands/database.py,sha256=_HnuOxlfVIFGkDotGv0SGb6oWrnm517FSvLv0aGcLJQ,3542
 dbt_platform_helper/commands/dns.py,sha256=o7PkvHktZo0jmqbx0krJTL0R4GtWSf1rF2KDEWor8Ts,35211
-dbt_platform_helper/commands/environment.py,sha256=VNr7G1QstM8INGs8jOxL1jQRqDcWx2Q0jaaBXtbHhys,24819
+dbt_platform_helper/commands/environment.py,sha256=QrG2ewchP2D1BmG067QpO-rR77F9lbNn2oVlM_zVS-g,7953
 dbt_platform_helper/commands/generate.py,sha256=YLCPb-xcPapGcsLn-7d1Am7BpGp5l0iecIDTOdNGjHk,722
 dbt_platform_helper/commands/notify.py,sha256=kVJ0s78QMiaEWPVKu_bbMko4DW2uJy2fu8-HNJsglyk,3748
-dbt_platform_helper/commands/pipeline.py,sha256=jQGwCRpJ_hXK988XmLHzRBHDWmhFzZb37wa75KuSd0M,5945
+dbt_platform_helper/commands/pipeline.py,sha256=Rw7JTKDxF3sv1dv-8_o-ofq5YBW0Pf-q2e7dFqbhH5U,9993
 dbt_platform_helper/commands/secrets.py,sha256=2NtV5FGx-ErkMg2QMiDvFOp03cKVbLzgmY8Y504EKJw,3860
 dbt_platform_helper/commands/version.py,sha256=RcgOu04MzO5qohs8HLqFcGgwfel56rl_MV8cVNqaQYk,1572
 dbt_platform_helper/constants.py,sha256=fzN2VZt81mspNfdYpNef5_eEjDVsh8GUYmhBMTIfPvI,232
 dbt_platform_helper/custom_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbt_platform_helper/custom_resources/s3_object.py,sha256=0mhLuKD0-vwuN1qmnLCrLo2qL58FvtCjNNjH34kac6Y,2526
 dbt_platform_helper/default-extensions.yml,sha256=SU1ZitskbuEBpvE7efc3s56eAUF11j70brhj_XrNMMo,493
+dbt_platform_helper/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dbt_platform_helper/domain/database_copy.py,sha256=OY7X5wmlb-s0kOn77X24-QWLD13ru27xi9_2gV-st2k,8586
+dbt_platform_helper/domain/maintenance_page.py,sha256=g9rYOeV7-AOMGJL9J8EdB--1xhl8pihOGDcbNK2IKV8,15729
 dbt_platform_helper/exceptions.py,sha256=dCQkqmlt5yycZgblwTtpkwGrP7ANQB_1MRORUWIYI-U,541
 dbt_platform_helper/jinja2_tags.py,sha256=jFyN_Sxmko1GSfvrqRIGQ80CCW8EwlCV3su0ahJPfoE,541
+dbt_platform_helper/providers/load_balancers.py,sha256=e1SPrWbBWq95paSVd3Y5yORIrHAZxcVabBYDjPyUTsU,1430
 dbt_platform_helper/templates/.copilot/config.yml,sha256=J_bA9sCtBdCPBRImpCBRnYvhQd4vpLYIXIU-lq9vbkA,158
 dbt_platform_helper/templates/.copilot/image_build_run.sh,sha256=adYucYXEB-kAgZNjTQo0T6EIAY8sh_xCEvVhWKKQ8mw,164
 dbt_platform_helper/templates/.copilot/phases/build.sh,sha256=umKXePcRvx4XyrRY0fAWIyYFtNjqBI2L8vIJk-V7C60,121
@@ -60,6 +63,7 @@ dbt_platform_helper/templates/env/overrides/package.json,sha256=dLDirCV_IljHPl1j
 dbt_platform_helper/templates/env/overrides/stack.ts,sha256=mh5xBJ_enJGUflIDlD-aRLKB_IE5OcvM6W8dGUdB-io,1910
 dbt_platform_helper/templates/env/overrides/tsconfig.json,sha256=xOAruVlYLYq9vU1MiIruhrIXDWYVXjbxy9Y42bHh5xE,710
 dbt_platform_helper/templates/env/terraform-overrides/cfn.patches.yml,sha256=cFlg69fvi9kzpz13ZAeY1asseZ6TuUex-6s76jG3oL4,259
+dbt_platform_helper/templates/environment-pipelines/main.tf,sha256=R5xM5OHmWD1BjUyl67uOVQ_PbgR96PuiYXlCDXoaj8c,1914
 dbt_platform_helper/templates/environments/main.tf,sha256=6tgxyGLvllZCiN2ryYwNM6fcm6wijb-LMjwdb9UckyU,1579
 dbt_platform_helper/templates/pipelines/codebase/manifest.yml,sha256=Mby5ri_w0K9r3cJVdUN-c9adnu2cVvDgHpY4qvHggCE,1980
 dbt_platform_helper/templates/pipelines/codebase/overrides/.gitignore,sha256=duFipE4GPjdkrYo-sO1jjCXdmSMoYNoFJLZ19lUKfL0,169
@@ -84,7 +88,7 @@ dbt_platform_helper/templates/svc/overrides/cfn.patches.yml,sha256=W7-d017akuUq9
 dbt_platform_helper/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dbt_platform_helper/utils/application.py,sha256=FzEoyTUlRJ7bt01cZAg-JHRQM4CqosVLOojdfwaDl6w,4113
 dbt_platform_helper/utils/arn_parser.py,sha256=1jY0elpAe4YL3ulrrCf1YiKmjI-7YXz4gJASqkIFHTc,1294
-dbt_platform_helper/utils/aws.py,sha256=H4oFnzJf0WXQ8xi19QlC3zKKA3HWPp5wNfm9hZGgMsI,13683
+dbt_platform_helper/utils/aws.py,sha256=_NM76HOT2EvPwPe0ltuZ2_WH0NYdNLPWEnw7HgVQ6K8,14332
 dbt_platform_helper/utils/click.py,sha256=Fx4y4bbve1zypvog_sgK7tJtCocmzheoEFLBRv1lfdM,2943
 dbt_platform_helper/utils/cloudformation.py,sha256=A1z12IYpapCOngAvZUCc_Ig7S7tuSP3ceeVzVR-5BGY,1053
 dbt_platform_helper/utils/cloudfoundry.py,sha256=GnQ4fVLnDfOdNSrsJjI6ElZHqpgwINeoPn77cUH2UFY,484
@@ -94,11 +98,11 @@ dbt_platform_helper/utils/manifests.py,sha256=ji3UYHCxq9tTpkm4MlRa2y0-JOYYqq1pWZ
 dbt_platform_helper/utils/messages.py,sha256=aLx6s9utt__IqlDdeIYq4n82ERwludu2Zfqy0Q2t-x8,115
 dbt_platform_helper/utils/platform_config.py,sha256=zJYCIsgUk5kNerocTzD7Q2XIrkYhiCu65sx8KQ-d-o8,833
 dbt_platform_helper/utils/template.py,sha256=raRx4QUCVJtKfvJK08Egg6gwWcs3r3V4nPWcJW4xNhA,574
-dbt_platform_helper/utils/validation.py,sha256=fsUsEjabAtoswdEzyRBdVUdgRqYJeoE3PgNU-6kTb3k,26713
+dbt_platform_helper/utils/validation.py,sha256=5gjq6x_RWGcUcu_te7qGiSBsI563QmBG4e7NLOWN6Go,27545
 dbt_platform_helper/utils/versioning.py,sha256=IBxdocJ8ZyJib38d1ja87tTuFE0iJ4npaDcAHQAKQ58,10825
 platform_helper.py,sha256=1lvPwynKODyi2U-ePKzJyFwRdKPs6_6zAYUPDYzDKMo,2300
-dbt_platform_helper-11.0.1.dist-info/LICENSE,sha256=dP79lN73--7LMApnankTGLqDbImXg8iYFqWgnExGkGk,1090
-dbt_platform_helper-11.0.1.dist-info/METADATA,sha256=h7DzlcU6VkjDqM8j8yG_WvBIUkUkzzebIS2t_WU1jXk,3119
-dbt_platform_helper-11.0.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-dbt_platform_helper-11.0.1.dist-info/entry_points.txt,sha256=QhbY8F434A-onsg0-FsdMd2U6HKh6Q7yCFFZrGUh5-M,67
-dbt_platform_helper-11.0.1.dist-info/RECORD,,
+dbt_platform_helper-11.2.0.dist-info/LICENSE,sha256=dP79lN73--7LMApnankTGLqDbImXg8iYFqWgnExGkGk,1090
+dbt_platform_helper-11.2.0.dist-info/METADATA,sha256=vh3ZwqIDMtBr0jEoi04ZPW1kEUOw-IUW3uTYUpokL8Q,3161
+dbt_platform_helper-11.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+dbt_platform_helper-11.2.0.dist-info/entry_points.txt,sha256=QhbY8F434A-onsg0-FsdMd2U6HKh6Q7yCFFZrGUh5-M,67
+dbt_platform_helper-11.2.0.dist-info/RECORD,,

dbt_platform_helper/commands/database_helpers.py

@@ -1,145 +0,0 @@
-import boto3
-import click
-
-from dbt_platform_helper.utils.aws import Vpc
-from dbt_platform_helper.utils.aws import get_aws_session_or_abort
-from dbt_platform_helper.utils.aws import get_connection_string
-from dbt_platform_helper.utils.aws import get_vpc_info_by_name
-
-
-def run_database_copy_task(
-    session: boto3.session.Session,
-    account_id: str,
-    app: str,
-    env: str,
-    database: str,
-    vpc_config: Vpc,
-    is_dump: bool,
-    db_connection_string: str,
-):
-    client = session.client("ecs")
-    action = "dump" if is_dump else "load"
-    response = client.run_task(
-        taskDefinition=f"arn:aws:ecs:eu-west-2:{account_id}:task-definition/{app}-{env}-{database}-{action}",
-        cluster=f"{app}-{env}",
-        capacityProviderStrategy=[
-            {"capacityProvider": "FARGATE", "weight": 1, "base": 0},
-        ],
-        networkConfiguration={
-            "awsvpcConfiguration": {
-                "subnets": vpc_config.subnets,
-                "securityGroups": vpc_config.security_groups,
-                "assignPublicIp": "DISABLED",
-            }
-        },
-        overrides={
-            "containerOverrides": [
-                {
-                    "name": f"{app}-{env}-{database}-{action}",
-                    "environment": [
-                        {"name": "DATA_COPY_OPERATION", "value": action.upper()},
-                        {"name": "DB_CONNECTION_STRING", "value": db_connection_string},
-                    ],
-                }
-            ]
-        },
-    )
-
-    return response.get("tasks", [{}])[0].get("taskArn")
-
-
-class DatabaseCopy:
-    def __init__(
-        self,
-        account_id,
-        app,
-        env,
-        database,
-        vpc_name,
-        get_session_fn=get_aws_session_or_abort,
-        run_database_copy_fn=run_database_copy_task,
-        vpc_config_fn=get_vpc_info_by_name,
-        db_connection_string_fn=get_connection_string,
-        input_fn=click.prompt,
-        echo_fn=click.secho,
-    ):
-        self.account_id = account_id
-        self.app = app
-        self.env = env
-        self.database = database
-        self.vpc_name = vpc_name
-        self.get_session_fn = get_session_fn
-        self.run_database_copy_fn = run_database_copy_fn
-        self.vpc_config_fn = vpc_config_fn
-        self.db_connection_string_fn = db_connection_string_fn
-        self.input_fn = input_fn
-        self.echo_fn = echo_fn
-
-    def _execute_operation(self, is_dump):
-        session = self.get_session_fn()
-        vpc_config = self.vpc_config_fn(session, self.app, self.env, self.vpc_name)
-        database_identifier = f"{self.app}-{self.env}-{self.database}"
-        db_connection_string = self.db_connection_string_fn(
-            session, self.app, self.env, database_identifier
-        )
-        task_arn = self.run_database_copy_fn(
-            session,
-            self.account_id,
-            self.app,
-            self.env,
-            self.database,
-            vpc_config,
-            is_dump,
-            db_connection_string,
-        )
-
-        self.echo_fn(
-            f"Task {task_arn} started. Waiting for it to complete (this may take some time)...",
-            fg="green",
-        )
-        self.tail_logs(is_dump)
-        self.wait_for_task_to_stop(task_arn)
-
-    def dump(self):
-        self._execute_operation(True)
-
-    def load(self):
-        if self.is_confirmed_ready_to_load():
-            self._execute_operation(False)
-
-    def is_confirmed_ready_to_load(self):
-        user_input = self.input_fn(
-            f"Are all tasks using {self.database} in the {self.env} environment stopped? (y/n)"
-        )
-        return user_input.lower().strip() in ["y", "yes"]
-
-    def tail_logs(self, is_dump: bool):
-        action = "dump" if is_dump else "load"
-        log_group_name = f"/ecs/{self.app}-{self.env}-{self.database}-{action}"
-        log_group_arn = f"arn:aws:logs:eu-west-2:{self.account_id}:log-group:{log_group_name}"
-        self.echo_fn(f"Tailing logs for {log_group_name}", fg="yellow")
-        session = self.get_session_fn()
-        response = session.client("logs").start_live_tail(logGroupIdentifiers=[log_group_arn])
-
-        stopped = False
-        for data in response["responseStream"]:
-            if stopped:
-                break
-            results = data.get("sessionUpdate", {}).get("sessionResults", [])
-            for result in results:
-                message = result.get("message")
-
-                if message:
-                    if message.startswith("Stopping data "):
-                        stopped = True
-                    self.echo_fn(message)
-
-    def wait_for_task_to_stop(self, task_arn):
-        self.echo_fn("Waiting for task to complete", fg="yellow")
-        client = self.get_session_fn().client("ecs")
-        waiter = client.get_waiter("tasks_stopped")
-        waiter.wait(
-            cluster=f"{self.app}-{self.env}",
-            tasks=[task_arn],
-            WaiterConfig={"Delay": 6, "MaxAttempts": 300},
-        )