dbt-platform-helper 10.5.0__tar.gz → 10.6.1__tar.gz

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dbt-platform-helper
-Version: 10.5.0
+Version: 10.6.1
 Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
 License: MIT
 Author: Department for Business and Trade Platform Team

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/commands/codebase.py

@@ -86,6 +86,38 @@ def prepare():
         click.echo(mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True))
 
 
+def list_latest_images(ecr_client, ecr_repository_name, codebase_repository):
+    paginator = ecr_client.get_paginator("describe_images")
+    describe_images_response_iterator = paginator.paginate(
+        repositoryName=ecr_repository_name,
+        filter={"tagStatus": "TAGGED"},
+    )
+    images = []
+    for page in describe_images_response_iterator:
+        images += page["imageDetails"]
+
+    sorted_images = sorted(
+        images,
+        key=lambda i: i["imagePushedAt"],
+        reverse=True,
+    )
+
+    MAX_RESULTS = 20
+
+    for image in sorted_images[:MAX_RESULTS]:
+        try:
+            commit_tag = next(t for t in image["imageTags"] if t.startswith("commit-"))
+            if not commit_tag:
+                continue
+
+            commit_hash = commit_tag.replace("commit-", "")
+            click.echo(
+                f"  - https://github.com/{codebase_repository}/commit/{commit_hash} - published: {image['imagePushedAt']}"
+            )
+        except StopIteration:
+            continue
+
+
 @codebase.command()
 @click.option("--app", help="AWS application name", required=True)
 @click.option(
@@ -116,29 +148,9 @@ def list(app, with_images):
     for codebase in codebases:
         click.echo(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
         if with_images:
-            describe_images_response = ecr_client.describe_images(
-                repositoryName=f"{application.name}/{codebase['name']}",
-                maxResults=20,
-                filter={"tagStatus": "TAGGED"},
+            list_latest_images(
+                ecr_client, f"{application.name}/{codebase['name']}", codebase["repository"]
             )
-            images = sorted(
-                describe_images_response["imageDetails"],
-                key=lambda i: i["imagePushedAt"],
-                reverse=True,
-            )
-
-            for image in images:
-                try:
-                    commit_tag = next(t for t in image["imageTags"] if t.startswith("commit-"))
-                    if not commit_tag:
-                        continue
-
-                    commit_hash = commit_tag.replace("commit-", "")
-                    click.echo(
-                        f"  - https://github.com/{codebase['repository']}/commit/{commit_hash} - published: {image['imagePushedAt']}"
-                    )
-                except StopIteration:
-                    continue
 
     click.echo("")
 

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/commands/conduit.py

@@ -315,6 +315,7 @@ def update_conduit_stack_resources(
     addon_name: str,
     task_name: str,
     parameter_name: str,
+    access: str,
 ):
     session = app.environments[env].session
     cloudformation_client = session.client("cloudformation")
@@ -352,7 +353,7 @@ def update_conduit_stack_resources(
         Properties:
           RoleArn: {log_filter_role_arn}
           LogGroupName: /copilot/{task_name}
-          FilterName: /copilot/conduit/{app.name}/{env}/{addon_type}/{addon_name}/{task_name.rsplit("-", 1)[1]}
+          FilterName: /copilot/conduit/{app.name}/{env}/{addon_type}/{addon_name}/{task_name.rsplit("-", 1)[1]}/{access}
           FilterPattern: ''
           DestinationArn: {destination_arn}
         """
@@ -386,7 +387,7 @@ def start_conduit(
         create_addon_client_task(application, env, addon_type, addon_name, task_name, access)
         add_stack_delete_policy_to_task_role(application, env, task_name)
         update_conduit_stack_resources(
-            application, env, addon_type, addon_name, task_name, parameter_name
+            application, env, addon_type, addon_name, task_name, parameter_name, access
         )
 
     connect_to_addon_client_task(application, env, cluster_arn, task_name)

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/default-extensions.yml

@@ -17,3 +17,10 @@ monitoring:
   environments:
     "*":
       enable_ops_center: false
+
+prometheus:
+  type: prometheus-policy
+  services: __all__
+  environments:
+    "*":
+      role_arn: arn:aws:iam::480224066791:role/amp-prometheus-role

dbt_platform_helper-10.6.1/dbt_platform_helper/templates/.copilot/phases/pre_build.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+if [ -f "./.gitmodules" ]; then
+  echo ".gitmodules file exists. Modifying URLs..."
+    account_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d':' -f5)
+    connection_id=$(echo $CODESTAR_CONNECTION_ARN | cut -d'/' -f2)
+    git_clone_base_url="https://codestar-connections.eu-west-2.amazonaws.com/git-http/$account_id/eu-west-2/$connection_id/uktrade"
+
+    git config --global credential.helper '!aws codecommit credential-helper $@'
+    git config --global credential.UseHttpPath true
+
+    sed -i "s|url = git@github.com:uktrade/\(.*\).git|url = $git_clone_base_url/\1.git|g" ./.gitmodules
+
+    git submodule update --init --remote --recursive
+
+else
+  echo ".gitmodules file does not exist. No URLs to update."
+fi
+
+# Add commands below to run as part of the pre_build phase

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/templates/environments/main.tf

@@ -36,3 +36,12 @@ module "extensions" {
   environment = "{{ environment }}"
   vpc_name    = "{{ config.vpc }}"
 }
+
+/* 
+Clean up because terraform modules were initially deployed with a -tf suffix.  This block moves those modules to naming without a suffix.
+Can be removed once all services have moved to the new naming.
+*/
+moved {
+  from = module.extensions-tf
+  to   = module.extensions
+}

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.deploy.yml

@@ -19,7 +19,7 @@ phases:
         else
           exit 1
         fi
-  
+
   post_build:
     commands:
       - |

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/templates/pipelines/codebase/overrides/stack.ts

@@ -78,6 +78,7 @@ export class TransformedStack extends cdk.Stack {
         const envVars = [
             {name: 'AWS_ACCOUNT_ID', value: this.account},
             {name: 'ECR_REPOSITORY', value: this.ecrRepository()},
+            {name: 'CODESTAR_CONNECTION_ARN', value: this.codestarConnection.arn},
         ];
         if (this.additionalEcrRepository()){
             envVars.push({name: 'ADDITIONAL_ECR_REPOSITORY', value: this.additionalEcrRepository()});
@@ -115,7 +116,7 @@ export class TransformedStack extends cdk.Stack {
                 location: `https://github.com/${this.codebaseConfiguration.repository}.git`,
                 gitCloneDepth: 0,
                 auth: {type: 'OAUTH'},
-                gitSubmodulesConfig: {fetchSubmodules: true},
+                gitSubmodulesConfig: {fetchSubmodules: false},
                 buildSpec: stringify(parse(
                     readFileSync(path.join(__dirname, 'buildspec.image.yml')).toString('utf-8'),
                 )),

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/templates/svc/manifest-backend.yml

@@ -38,18 +38,8 @@ network:
 storage:
   readonly_fs: true       # Limit to read-only access to mounted root filesystems.
 
-observability:
-  tracing: awsxray
-
 variables:                    # Pass environment variables as key value pairs.
   PORT: 8080                  # The bootstrap container requires a $PORT env var
-  OTEL_PROPAGATORS: xray
-  OTEL_PYTHON_ID_GENERATOR: xray
-  OTEL_SERVICE_NAME: ${COPILOT_APPLICATION_NAME}-${COPILOT_ENVIRONMENT_NAME}-{{ name }}
-  OTEL_METRICS_EXPORTER: none
-  OTEL_TRACES_EXPORTER: otlp
-  OTEL_TRACES_SAMPLER: traceidratio
-  OTEL_TRACES_SAMPLER_ARG: "0.05"
 {%- for envvar, value in env_vars.items() %}
   {{ envvar }}: {{ value }}
 {%- endfor %}

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/templates/svc/manifest-public.yml

@@ -74,20 +74,10 @@ network:
 storage:
   readonly_fs: true       # Limit to read-only access to mounted root filesystems.
 
-observability:
-  tracing: awsxray
-
 # Optional fields for more advanced use-cases.
 #
 variables:                    # Pass environment variables as key value pairs.
   PORT: 8080
-  OTEL_PROPAGATORS: xray
-  OTEL_PYTHON_ID_GENERATOR: xray
-  OTEL_SERVICE_NAME: ${COPILOT_APPLICATION_NAME}-${COPILOT_ENVIRONMENT_NAME}-{{ name }}
-  OTEL_METRICS_EXPORTER: none
-  OTEL_TRACES_EXPORTER: otlp
-  OTEL_TRACES_SAMPLER: traceidratio
-  OTEL_TRACES_SAMPLER_ARG: "0.05"
 {%- for envvar, value in env_vars.items() %}
   {{ envvar }}: {{ value }}
 {%- endfor %}

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/utils/files.py

@@ -61,7 +61,7 @@ def generate_override_files_from_template(base_path, overrides_path, output_dir,
     templates = Environment(
         loader=FileSystemLoader(f"{overrides_path}"), keep_trailing_newline=True
     )
-    environments = ", ".join([env["name"] for env in template_data["environments"]])
+    environments = ",".join([env["name"] for env in template_data["environments"]])
     data = {"environments": environments}
 
     def generate_files_for_dir(pattern):

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/dbt_platform_helper/utils/versioning.py

@@ -90,7 +90,7 @@ def get_github_released_version(repository: str, tags: bool = False) -> Tuple[in
 
 
 def get_platform_helper_versions() -> PlatformHelperVersions:
-    local_version = parse_version(version("dbt-platform-helper"))
+    locally_installed_version = parse_version(version("dbt-platform-helper"))
 
     package_info = requests.get("https://pypi.org/pypi/dbt-platform-helper/json").json()
     released_versions = package_info["releases"].keys()
@@ -98,10 +98,16 @@ def get_platform_helper_versions() -> PlatformHelperVersions:
     parsed_released_versions.sort(reverse=True)
     latest_release = parsed_released_versions[0]
 
-    version_from_file = parse_version(Path(".platform-helper-version").read_text())
+    version_from_file = None
+    message = "Cannot get dbt-platform-helper version from file '.platform-helper-version'. Check if file exists."
+
+    try:
+        version_from_file = parse_version(Path(".platform-helper-version").read_text())
+    except FileNotFoundError:
+        click.secho(f"{message}", fg="yellow")
 
     return PlatformHelperVersions(
-        local_version=local_version,
+        local_version=locally_installed_version,
         latest_release=latest_release,
         platform_helper_file_version=version_from_file,
     )
@@ -156,8 +162,8 @@ def validate_template_version(app_version: Tuple[int, int, int], template_file_p
 
 def generate_platform_helper_version_file(directory="."):
     base_path = Path(directory)
-    platform_helper_version = string_version(get_platform_helper_versions().local_version)
-    click.echo(mkfile(base_path, ".platform-helper-version", f"{platform_helper_version}\n"))
+    local_version = string_version(get_platform_helper_versions().local_version)
+    click.echo(mkfile(base_path, ".platform-helper-version", f"{local_version}\n"))
 
 
 def check_platform_helper_version_needs_update():

{dbt_platform_helper-10.5.0 → dbt_platform_helper-10.6.1}/pyproject.toml

@@ -3,7 +3,7 @@ line-length = 100
 
 [tool.poetry]
 name = "dbt-platform-helper"
-version = "10.5.0"
+version = "10.6.1"
 description = "Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot."
 authors = ["Department for Business and Trade Platform Team <sre-team@digital.trade.gov.uk>"]
 license = "MIT"

dbt_platform_helper-10.5.0/dbt_platform_helper/templates/.copilot/phases/pre_build.sh

@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-# Exit early if something goes wrong
-set -e
-
-# Add commands below to run as part of the pre_build phase