dbt-platform-helper 12.2.4__tar.gz → 12.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbt-platform-helper might be problematic. Click here for more details.

Files changed (100) hide show
  1. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/PKG-INFO +1 -1
  2. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/COMMANDS.md +6 -1
  3. dbt_platform_helper-12.4.0/dbt_platform_helper/commands/codebase.py +75 -0
  4. dbt_platform_helper-12.4.0/dbt_platform_helper/commands/conduit.py +58 -0
  5. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/config.py +4 -4
  6. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/copilot.py +13 -15
  7. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/database.py +17 -4
  8. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/environment.py +3 -2
  9. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/secrets.py +1 -1
  10. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/codebase.py +81 -63
  11. dbt_platform_helper-12.4.0/dbt_platform_helper/domain/conduit.py +121 -0
  12. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/database_copy.py +48 -42
  13. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/maintenance_page.py +8 -8
  14. dbt_platform_helper-12.4.0/dbt_platform_helper/platform_exception.py +5 -0
  15. dbt_platform_helper-12.4.0/dbt_platform_helper/providers/aws.py +32 -0
  16. dbt_platform_helper-12.4.0/dbt_platform_helper/providers/cloudformation.py +134 -0
  17. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/copilot.py +33 -16
  18. dbt_platform_helper-12.4.0/dbt_platform_helper/providers/ecs.py +102 -0
  19. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/load_balancers.py +11 -5
  20. dbt_platform_helper-12.4.0/dbt_platform_helper/providers/secrets.py +126 -0
  21. dbt_platform_helper-12.4.0/dbt_platform_helper/providers/validation.py +19 -0
  22. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/application.py +14 -2
  23. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/arn_parser.py +1 -1
  24. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/aws.py +38 -12
  25. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/git.py +2 -2
  26. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/validation.py +57 -18
  27. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/versioning.py +8 -8
  28. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/pyproject.toml +1 -1
  29. dbt_platform_helper-12.2.4/dbt_platform_helper/addons-template-map.yml +0 -29
  30. dbt_platform_helper-12.2.4/dbt_platform_helper/commands/codebase.py +0 -146
  31. dbt_platform_helper-12.2.4/dbt_platform_helper/commands/conduit.py +0 -78
  32. dbt_platform_helper-12.2.4/dbt_platform_helper/domain/conduit.py +0 -172
  33. dbt_platform_helper-12.2.4/dbt_platform_helper/exceptions.py +0 -81
  34. dbt_platform_helper-12.2.4/dbt_platform_helper/providers/cloudformation.py +0 -105
  35. dbt_platform_helper-12.2.4/dbt_platform_helper/providers/ecs.py +0 -79
  36. dbt_platform_helper-12.2.4/dbt_platform_helper/providers/secrets.py +0 -85
  37. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/LICENSE +0 -0
  38. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/README.md +0 -0
  39. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/__init__.py +0 -0
  40. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/addon-plans.yml +0 -0
  41. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/__init__.py +0 -0
  42. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/application.py +0 -0
  43. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/generate.py +0 -0
  44. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/notify.py +0 -0
  45. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/pipeline.py +0 -0
  46. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/version.py +0 -0
  47. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/constants.py +0 -0
  48. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/default-extensions.yml +0 -0
  49. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/__init__.py +0 -0
  50. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/jinja2_tags.py +0 -0
  51. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/__init__.py +0 -0
  52. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/config.yml +0 -0
  53. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/image_build_run.sh +0 -0
  54. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/phases/build.sh +0 -0
  55. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/phases/install.sh +0 -0
  56. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/phases/post_build.sh +0 -0
  57. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/phases/pre_build.sh +0 -0
  58. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/COMMANDS.md.jinja +0 -0
  59. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addon-instructions.txt +0 -0
  60. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/README.md +0 -0
  61. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/svc/appconfig-ipfilter.yml +0 -0
  62. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/svc/prometheus-policy.yml +0 -0
  63. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/svc/s3-policy.yml +0 -0
  64. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/svc/subscription-filter.yml +0 -0
  65. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/ci-codebuild-role-policy.json +0 -0
  66. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/create-codebuild-role.json +0 -0
  67. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/custom-codebuild-role-policy.json +0 -0
  68. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/env/manifest.yml +0 -0
  69. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/env/terraform-overrides/cfn.patches.yml +0 -0
  70. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/environment-pipelines/main.tf +0 -0
  71. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/environments/main.tf +0 -0
  72. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/manifest.yml +0 -0
  73. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/.gitignore +0 -0
  74. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/bin/override.ts +0 -0
  75. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.deploy.yml +0 -0
  76. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.image.yml +0 -0
  77. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/cdk.json +0 -0
  78. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json +0 -0
  79. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/package.json +0 -0
  80. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/stack.ts +0 -0
  81. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/tsconfig.json +0 -0
  82. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/types.ts +0 -0
  83. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/environments/buildspec.yml +0 -0
  84. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/environments/manifest.yml +0 -0
  85. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/environments/overrides/cfn.patches.yml +0 -0
  86. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/maintenance_pages/default.html +0 -0
  87. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/maintenance_pages/dmas-migration.html +0 -0
  88. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/maintenance_pages/migration.html +0 -0
  89. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/manifest-backend.yml +0 -0
  90. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/manifest-public.yml +0 -0
  91. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +0 -0
  92. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/__init__.py +0 -0
  93. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/click.py +0 -0
  94. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/cloudfoundry.py +0 -0
  95. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/files.py +0 -0
  96. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/manifests.py +0 -0
  97. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/messages.py +0 -0
  98. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/platform_config.py +0 -0
  99. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/template.py +0 -0
  100. {dbt_platform_helper-12.2.4 → dbt_platform_helper-12.4.0}/platform_helper.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dbt-platform-helper
3
- Version: 12.2.4
3
+ Version: 12.4.0
4
4
  Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
5
5
  License: MIT
6
6
  Author: Department for Business and Trade Platform Team
@@ -765,6 +765,7 @@ platform-helper database (dump|load|copy)
765
765
  ```
766
766
  platform-helper database dump --from <from_env> --database <database>
767
767
  [--app <application>] [--from-vpc <from_vpc>]
768
+ [--filename <filename>]
768
769
  ```
769
770
 
770
771
  ## Options
@@ -777,6 +778,8 @@ platform-helper database dump --from <from_env> --database <database>
777
778
  - The name of the database you are dumping data from
778
779
  - `--from-vpc <text>`
779
780
  - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
781
+ - `--filename <text>`
782
+ - Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments
780
783
  - `--help <boolean>` _Defaults to False._
781
784
  - Show this message and exit.
782
785
 
@@ -791,7 +794,7 @@ platform-helper database dump --from <from_env> --database <database>
791
794
  ```
792
795
  platform-helper database load --to <to_env> --database <database>
793
796
  [--app <application>] [--to-vpc <to_vpc>]
794
- [--auto-approve]
797
+ [--filename <filename>] [--auto-approve]
795
798
  ```
796
799
 
797
800
  ## Options
@@ -806,6 +809,8 @@ platform-helper database load --to <to_env> --database <database>
806
809
  - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
807
810
  - `--auto-approve <boolean>` _Defaults to False._
808
811
 
812
+ - `--filename <text>`
813
+ - Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments
809
814
  - `--help <boolean>` _Defaults to False._
810
815
  - Show this message and exit.
811
816
 
@@ -0,0 +1,75 @@
1
+ import click
2
+
3
+ from dbt_platform_helper.domain.codebase import Codebase
4
+ from dbt_platform_helper.platform_exception import PlatformException
5
+ from dbt_platform_helper.utils.click import ClickDocOptGroup
6
+ from dbt_platform_helper.utils.versioning import (
7
+ check_platform_helper_version_needs_update,
8
+ )
9
+
10
+
11
+ @click.group(chain=True, cls=ClickDocOptGroup)
12
+ def codebase():
13
+ """Codebase commands."""
14
+ check_platform_helper_version_needs_update()
15
+
16
+
17
+ @codebase.command()
18
+ def prepare():
19
+ """Sets up an application codebase for use within a DBT platform project."""
20
+ try:
21
+ Codebase().prepare()
22
+ except PlatformException as err:
23
+ click.secho(str(err), fg="red")
24
+ raise click.Abort
25
+
26
+
27
+ @codebase.command()
28
+ @click.option("--app", help="AWS application name", required=True)
29
+ @click.option(
30
+ "--with-images",
31
+ help="List up to the last 10 images tagged for this codebase",
32
+ default=False,
33
+ is_flag=True,
34
+ )
35
+ def list(app, with_images):
36
+ """List available codebases for the application."""
37
+ try:
38
+ Codebase().list(app, with_images)
39
+ except PlatformException as err:
40
+ click.secho(str(err), fg="red")
41
+ raise click.Abort
42
+
43
+
44
+ @codebase.command()
45
+ @click.option("--app", help="AWS application name", required=True)
46
+ @click.option(
47
+ "--codebase",
48
+ help="The codebase name as specified in the platform-config.yml file",
49
+ required=True,
50
+ )
51
+ @click.option("--commit", help="GitHub commit hash", required=True)
52
+ def build(app, codebase, commit):
53
+ """Trigger a CodePipeline pipeline based build."""
54
+ try:
55
+ Codebase().build(app, codebase, commit)
56
+ except PlatformException as err:
57
+ click.secho(str(err), fg="red")
58
+ raise click.Abort
59
+
60
+
61
+ @codebase.command()
62
+ @click.option("--app", help="AWS application name", required=True)
63
+ @click.option("--env", help="AWS Copilot environment", required=True)
64
+ @click.option(
65
+ "--codebase",
66
+ help="The codebase name as specified in the platform-config.yml file",
67
+ required=True,
68
+ )
69
+ @click.option("--commit", help="GitHub commit hash", required=True)
70
+ def deploy(app, env, codebase, commit):
71
+ try:
72
+ Codebase().deploy(app, env, codebase, commit)
73
+ except PlatformException as err:
74
+ click.secho(str(err), fg="red")
75
+ raise click.Abort
@@ -0,0 +1,58 @@
1
+ import click
2
+
3
+ from dbt_platform_helper.domain.conduit import Conduit
4
+ from dbt_platform_helper.platform_exception import PlatformException
5
+ from dbt_platform_helper.providers.cloudformation import CloudFormation
6
+ from dbt_platform_helper.providers.ecs import ECS
7
+ from dbt_platform_helper.providers.secrets import Secrets
8
+ from dbt_platform_helper.utils.application import load_application
9
+ from dbt_platform_helper.utils.click import ClickDocOptCommand
10
+ from dbt_platform_helper.utils.versioning import (
11
+ check_platform_helper_version_needs_update,
12
+ )
13
+
14
+ CONDUIT_ACCESS_OPTIONS = ["read", "write", "admin"]
15
+
16
+
17
+ @click.command(cls=ClickDocOptCommand)
18
+ @click.argument("addon_name", type=str, required=True)
19
+ @click.option("--app", help="Application name", required=True)
20
+ @click.option("--env", help="Environment name", required=True)
21
+ @click.option(
22
+ "--access",
23
+ default="read",
24
+ type=click.Choice(CONDUIT_ACCESS_OPTIONS),
25
+ help="Allow read, write or admin access to the database addons.",
26
+ )
27
+ def conduit(addon_name: str, app: str, env: str, access: str):
28
+ """Opens a shell for a given addon_name create a conduit connection to
29
+ interact with postgres, opensearch or redis."""
30
+ check_platform_helper_version_needs_update()
31
+ application = load_application(app)
32
+
33
+ try:
34
+ secrets_provider: Secrets = Secrets(
35
+ application.environments[env].session.client("ssm"),
36
+ application.environments[env].session.client("secretsmanager"),
37
+ application.name,
38
+ env,
39
+ )
40
+ cloudformation_provider: CloudFormation = CloudFormation(
41
+ application.environments[env].session.client("cloudformation"),
42
+ application.environments[env].session.client("iam"),
43
+ application.environments[env].session.client("ssm"),
44
+ )
45
+
46
+ ecs_provider: ECS = ECS(
47
+ application.environments[env].session.client("ecs"),
48
+ application.environments[env].session.client("ssm"),
49
+ application.name,
50
+ env,
51
+ )
52
+
53
+ Conduit(application, secrets_provider, cloudformation_provider, ecs_provider).start(
54
+ env, addon_name, access
55
+ )
56
+ except PlatformException as err:
57
+ click.secho(str(err), fg="red")
58
+ raise click.Abort
@@ -8,8 +8,8 @@ import botocore
8
8
  import click
9
9
  from prettytable import PrettyTable
10
10
 
11
- from dbt_platform_helper.exceptions import IncompatibleMajorVersion
12
- from dbt_platform_helper.exceptions import ValidationException
11
+ from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
12
+ from dbt_platform_helper.providers.validation import ValidationException
13
13
  from dbt_platform_helper.utils import versioning
14
14
  from dbt_platform_helper.utils.click import ClickDocOptGroup
15
15
  from dbt_platform_helper.utils.validation import config_file_check
@@ -110,7 +110,7 @@ def deployment():
110
110
  str(template_file.resolve())
111
111
  )
112
112
  versioning.validate_template_version(local_version, str(template_file.resolve()))
113
- except IncompatibleMajorVersion:
113
+ except IncompatibleMajorVersionException:
114
114
  local_compatible_symbol = no
115
115
  compatible = False
116
116
  recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
@@ -134,7 +134,7 @@ def deployment():
134
134
  str(template_file.resolve())
135
135
  )
136
136
  versioning.validate_template_version(latest_release, str(template_file.resolve()))
137
- except IncompatibleMajorVersion:
137
+ except IncompatibleMajorVersionException:
138
138
  latest_compatible_symbol = no
139
139
  compatible = False
140
140
  except ValidationException:
@@ -35,6 +35,14 @@ SERVICE_TYPES = [
35
35
  "Worker Service",
36
36
  ]
37
37
 
38
+ ADDON_TEMPLATE_MAP = {
39
+ "s3": ["addons/svc/s3-policy.yml"],
40
+ "s3-policy": ["addons/svc/s3-policy.yml"],
41
+ "appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
42
+ "subscription-filter": ["addons/svc/subscription-filter.yml"],
43
+ "prometheus-policy": ["addons/svc/prometheus-policy.yml"],
44
+ }
45
+
38
46
 
39
47
  def list_copilot_local_environments():
40
48
  return [
@@ -248,9 +256,6 @@ def make_addons():
248
256
 
249
257
  application_name = get_application_name()
250
258
 
251
- with open(PACKAGE_DIR / "addons-template-map.yml") as fd:
252
- addon_template_map = yaml.safe_load(fd)
253
-
254
259
  click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
255
260
 
256
261
  env_path = Path(f"copilot/environments/")
@@ -270,11 +275,6 @@ def make_addons():
270
275
  print(f">>>>>>>>> {addon_name}")
271
276
  addon_type = addon_config.pop("type")
272
277
  environments = addon_config.pop("environments")
273
- if addon_template_map[addon_type].get("requires_addons_parameters", False):
274
- pass
275
- if addon_type in ["postgres"]:
276
- pass
277
-
278
278
  environment_addon_config = {
279
279
  "addon_type": addon_type,
280
280
  "environments": environments,
@@ -309,7 +309,6 @@ def make_addons():
309
309
  _generate_service_addons(
310
310
  addon_config,
311
311
  addon_name,
312
- addon_template_map,
313
312
  addon_type,
314
313
  output_dir,
315
314
  service_addon_config,
@@ -339,7 +338,6 @@ def _generate_env_overrides(output_dir):
339
338
  def _generate_service_addons(
340
339
  addon_config,
341
340
  addon_name,
342
- addon_template_map,
343
341
  addon_type,
344
342
  output_dir,
345
343
  service_addon_config,
@@ -347,8 +345,8 @@ def _generate_service_addons(
347
345
  log_destination_arns,
348
346
  ):
349
347
  # generate svc addons
350
- for addon in addon_template_map[addon_type].get("svc", []):
351
- template = templates.get_template(addon["template"])
348
+ for addon_template in ADDON_TEMPLATE_MAP.get(addon_type, []):
349
+ template = templates.get_template(addon_template)
352
350
 
353
351
  for svc in addon_config.get("services", []):
354
352
  service_path = Path(f"copilot/{svc}/addons/")
@@ -360,10 +358,10 @@ def _generate_service_addons(
360
358
  }
361
359
  )
362
360
 
363
- filename = addon.get("filename", f"{addon_name}.yml")
364
-
365
361
  (output_dir / service_path).mkdir(parents=True, exist_ok=True)
366
- click.echo(mkfile(output_dir, service_path / filename, contents, overwrite=True))
362
+ click.echo(
363
+ mkfile(output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True)
364
+ )
367
365
 
368
366
 
369
367
  def _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path):
@@ -31,10 +31,16 @@ def database():
31
31
  type=str,
32
32
  help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
33
33
  )
34
- def dump(app, from_env, database, from_vpc):
34
+ @click.option(
35
+ "--filename",
36
+ type=str,
37
+ help="Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments",
38
+ )
39
+ def dump(app, from_env, database, from_vpc, filename):
35
40
  """Dump a database into an S3 bucket."""
36
41
  data_copy = DatabaseCopy(app, database)
37
- data_copy.dump(from_env, from_vpc)
42
+ data_copy.dump(from_env, from_vpc, filename)
43
+ # Todo: Catch expected errors and output message
38
44
 
39
45
 
40
46
  @database.command(name="load")
@@ -55,10 +61,16 @@ def dump(app, from_env, database, from_vpc):
55
61
  help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
56
62
  )
57
63
  @click.option("--auto-approve/--no-auto-approve", default=False)
58
- def load(app, to_env, database, to_vpc, auto_approve):
64
+ @click.option(
65
+ "--filename",
66
+ type=str,
67
+ help="Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments",
68
+ )
69
+ def load(app, to_env, database, to_vpc, auto_approve, filename):
59
70
  """Load a database from an S3 bucket."""
60
71
  data_copy = DatabaseCopy(app, database, auto_approve)
61
- data_copy.load(to_env, to_vpc)
72
+ data_copy.load(to_env, to_vpc, filename)
73
+ # Todo: Catch expected errors and output message
62
74
 
63
75
 
64
76
  @database.command(name="copy")
@@ -110,3 +122,4 @@ def copy(
110
122
  """Copy a database between environments."""
111
123
  data_copy = DatabaseCopy(app, database, auto_approve)
112
124
  data_copy.copy(from_env, to_env, from_vpc, to_vpc, svc, template, no_maintenance_page)
125
+ # Todo: Catch expected errors and output message
@@ -5,6 +5,7 @@ from schema import SchemaError
5
5
  from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
6
6
  from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
7
7
  from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
8
+ from dbt_platform_helper.platform_exception import PlatformException
8
9
  from dbt_platform_helper.providers.load_balancers import find_https_listener
9
10
  from dbt_platform_helper.utils.aws import get_aws_session_or_abort
10
11
  from dbt_platform_helper.utils.click import ClickDocOptGroup
@@ -238,10 +239,10 @@ def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
238
239
  try:
239
240
  certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
240
241
  except StopIteration:
241
- raise CertificateNotFoundError()
242
+ raise CertificateNotFoundException()
242
243
 
243
244
  return certificate_arn
244
245
 
245
246
 
246
- class CertificateNotFoundError(Exception):
247
+ class CertificateNotFoundException(PlatformException):
247
248
  pass
@@ -102,7 +102,6 @@ def list(app, env):
102
102
  params = dict(Path=path, Recursive=False, WithDecryption=True, MaxResults=10)
103
103
  secrets = []
104
104
 
105
- # TODO: refactor shared code with get_ssm_secret_names - Check if this is still valid
106
105
  while True:
107
106
  response = client.get_parameters_by_path(**params)
108
107
 
@@ -114,6 +113,7 @@ def list(app, env):
114
113
  else:
115
114
  break
116
115
 
116
+ # Todo: When we refactor this, the above could probably just use dbt_platform_helper.utils.aws.get_ssm_secret_names so we would end up with print("\n".join(get_ssm_secret_names(app, env)))
117
117
  print("\n".join(sorted(secrets)))
118
118
 
119
119
 
@@ -9,11 +9,9 @@ import requests
9
9
  import yaml
10
10
  from boto3 import Session
11
11
 
12
- from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
13
- from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
14
- from dbt_platform_helper.exceptions import NoCopilotCodebasesFoundError
15
- from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
12
+ from dbt_platform_helper.platform_exception import PlatformException
16
13
  from dbt_platform_helper.utils.application import Application
14
+ from dbt_platform_helper.utils.application import ApplicationException
17
15
  from dbt_platform_helper.utils.application import load_application
18
16
  from dbt_platform_helper.utils.aws import check_codebase_exists
19
17
  from dbt_platform_helper.utils.aws import check_image_exists
@@ -29,31 +27,31 @@ from dbt_platform_helper.utils.template import setup_templates
29
27
  class Codebase:
30
28
  def __init__(
31
29
  self,
32
- input_fn: Callable[[str], str] = click.prompt,
33
- echo_fn: Callable[[str], str] = click.secho,
34
- confirm_fn: Callable[[str], bool] = click.confirm,
35
- load_application_fn: Callable[[str], Application] = load_application,
36
- get_aws_session_or_abort_fn: Callable[[str], Session] = get_aws_session_or_abort,
37
- check_codebase_exists_fn: Callable[[str], str] = check_codebase_exists,
38
- check_image_exists_fn: Callable[[str], str] = check_image_exists,
39
- get_build_url_from_arn_fn: Callable[[str], str] = get_build_url_from_arn,
40
- list_latest_images_fn: Callable[[str], str] = list_latest_images,
41
- start_build_extraction_fn: Callable[[str], str] = start_build_extraction,
42
- check_if_commit_exists_fn: Callable[[str], str] = check_if_commit_exists,
43
- subprocess: Callable[[str], str] = subprocess.run,
30
+ input: Callable[[str], str] = click.prompt,
31
+ echo: Callable[[str], str] = click.secho,
32
+ confirm: Callable[[str], bool] = click.confirm,
33
+ load_application: Callable[[str], Application] = load_application,
34
+ get_aws_session_or_abort: Callable[[str], Session] = get_aws_session_or_abort,
35
+ check_codebase_exists: Callable[[str], str] = check_codebase_exists,
36
+ check_image_exists: Callable[[str], str] = check_image_exists,
37
+ get_build_url_from_arn: Callable[[str], str] = get_build_url_from_arn,
38
+ list_latest_images: Callable[[str], str] = list_latest_images,
39
+ start_build_extraction: Callable[[str], str] = start_build_extraction,
40
+ check_if_commit_exists: Callable[[str], str] = check_if_commit_exists,
41
+ run_subprocess: Callable[[str], str] = subprocess.run,
44
42
  ):
45
- self.input_fn = input_fn
46
- self.echo_fn = echo_fn
47
- self.confirm_fn = confirm_fn
48
- self.load_application_fn = load_application_fn
49
- self.get_aws_session_or_abort_fn = get_aws_session_or_abort_fn
50
- self.check_codebase_exists_fn = check_codebase_exists_fn
51
- self.check_image_exists_fn = check_image_exists_fn
52
- self.get_build_url_from_arn_fn = get_build_url_from_arn_fn
53
- self.list_latest_images_fn = list_latest_images_fn
54
- self.start_build_extraction_fn = start_build_extraction_fn
55
- self.check_if_commit_exists_fn = check_if_commit_exists_fn
56
- self.subprocess = subprocess
43
+ self.input = input
44
+ self.echo = echo
45
+ self.confirm = confirm
46
+ self.load_application = load_application
47
+ self.get_aws_session_or_abort = get_aws_session_or_abort
48
+ self.check_codebase_exists = check_codebase_exists
49
+ self.check_image_exists = check_image_exists
50
+ self.get_build_url_from_arn = get_build_url_from_arn
51
+ self.list_latest_images = list_latest_images
52
+ self.start_build_extraction = start_build_extraction
53
+ self.check_if_commit_exists = check_if_commit_exists
54
+ self.run_subprocess = run_subprocess
57
55
 
58
56
  def prepare(self):
59
57
  """Sets up an application codebase for use within a DBT platform
@@ -61,13 +59,15 @@ class Codebase:
61
59
  templates = setup_templates()
62
60
 
63
61
  repository = (
64
- self.subprocess(["git", "remote", "get-url", "origin"], capture_output=True, text=True)
62
+ self.run_subprocess(
63
+ ["git", "remote", "get-url", "origin"], capture_output=True, text=True
64
+ )
65
65
  .stdout.split("/")[-1]
66
66
  .strip()
67
67
  .removesuffix(".git")
68
68
  )
69
69
  if repository.endswith("-deploy") or Path("./copilot").exists():
70
- raise NotInCodeBaseRepositoryError
70
+ raise NotInCodeBaseRepositoryException()
71
71
 
72
72
  builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
73
73
  builder_configuration_response = requests.get(builder_configuration_url)
@@ -91,7 +91,7 @@ class Codebase:
91
91
  config_contents = templates.get_template(f".copilot/config.yml").render(
92
92
  repository=repository, builder_version=builder_version
93
93
  )
94
- self.echo_fn(
94
+ self.echo(
95
95
  mkfile(
96
96
  Path("."), ".copilot/image_build_run.sh", image_build_run_contents, overwrite=True
97
97
  )
@@ -100,27 +100,27 @@ class Codebase:
100
100
  image_build_run_file = Path(".copilot/image_build_run.sh")
101
101
  image_build_run_file.chmod(image_build_run_file.stat().st_mode | stat.S_IEXEC)
102
102
 
103
- self.echo_fn(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
103
+ self.echo(mkfile(Path("."), ".copilot/config.yml", config_contents, overwrite=True))
104
104
 
105
105
  for phase in ["build", "install", "post_build", "pre_build"]:
106
106
  phase_contents = templates.get_template(f".copilot/phases/{phase}.sh").render()
107
107
 
108
- self.echo_fn(
108
+ self.echo(
109
109
  mkfile(Path("./.copilot"), f"phases/{phase}.sh", phase_contents, overwrite=True)
110
110
  )
111
111
 
112
112
  def build(self, app: str, codebase: str, commit: str):
113
113
  """Trigger a CodePipeline pipeline based build."""
114
- session = self.get_aws_session_or_abort_fn()
115
- self.load_application_fn(app, default_session=session)
114
+ session = self.get_aws_session_or_abort()
115
+ self.load_application(app, default_session=session)
116
116
 
117
- self.check_if_commit_exists_fn(commit)
117
+ self.check_if_commit_exists(commit)
118
118
 
119
119
  codebuild_client = session.client("codebuild")
120
120
  build_url = self.__start_build_with_confirmation(
121
- self.confirm_fn,
121
+ self.confirm,
122
122
  codebuild_client,
123
- self.get_build_url_from_arn_fn,
123
+ self.get_build_url_from_arn,
124
124
  f'You are about to build "{app}" for "{codebase}" with commit "{commit}". Do you want to continue?',
125
125
  {
126
126
  "projectName": f"codebuild-{app}-{codebase}",
@@ -130,29 +130,29 @@ class Codebase:
130
130
  )
131
131
 
132
132
  if build_url:
133
- return self.echo_fn(
133
+ return self.echo(
134
134
  f"Your build has been triggered. Check your build progress in the AWS Console: {build_url}"
135
135
  )
136
136
 
137
- raise ApplicationDeploymentNotTriggered()
137
+ raise ApplicationDeploymentNotTriggered(codebase)
138
138
 
139
139
  def deploy(self, app, env, codebase, commit):
140
140
  """Trigger a CodePipeline pipeline based deployment."""
141
- session = self.get_aws_session_or_abort_fn()
141
+ session = self.get_aws_session_or_abort()
142
142
 
143
- application = self.load_application_fn(app, default_session=session)
143
+ application = self.load_application(app, default_session=session)
144
144
  if not application.environments.get(env):
145
- raise ApplicationEnvironmentNotFoundError()
145
+ raise ApplicationEnvironmentNotFoundException(env)
146
146
 
147
- self.check_codebase_exists_fn(session, application, codebase)
147
+ self.check_codebase_exists(session, application, codebase)
148
148
 
149
- self.check_image_exists_fn(session, application, codebase, commit)
149
+ self.check_image_exists(session, application, codebase, commit)
150
150
 
151
151
  codebuild_client = session.client("codebuild")
152
152
  build_url = self.__start_build_with_confirmation(
153
- self.confirm_fn,
153
+ self.confirm,
154
154
  codebuild_client,
155
- self.get_build_url_from_arn_fn,
155
+ self.get_build_url_from_arn,
156
156
  f'You are about to deploy "{app}" for "{codebase}" with commit "{commit}" to the "{env}" environment. Do you want to continue?',
157
157
  {
158
158
  "projectName": f"pipeline-{application.name}-{codebase}-BuildProject",
@@ -166,34 +166,34 @@ class Codebase:
166
166
  )
167
167
 
168
168
  if build_url:
169
- return self.echo_fn(
169
+ return self.echo(
170
170
  "Your deployment has been triggered. Check your build progress in the AWS Console: "
171
171
  f"{build_url}",
172
172
  )
173
173
 
174
- raise ApplicationDeploymentNotTriggered()
174
+ raise ApplicationDeploymentNotTriggered(codebase)
175
175
 
176
176
  def list(self, app: str, with_images: bool):
177
177
  """List available codebases for the application."""
178
- session = self.get_aws_session_or_abort_fn()
179
- application = self.load_application_fn(app, session)
178
+ session = self.get_aws_session_or_abort()
179
+ application = self.load_application(app, session)
180
180
  ssm_client = session.client("ssm")
181
181
  ecr_client = session.client("ecr")
182
182
  codebases = self.__get_codebases(application, ssm_client)
183
183
 
184
- self.echo_fn("The following codebases are available:")
184
+ self.echo("The following codebases are available:")
185
185
 
186
186
  for codebase in codebases:
187
- self.echo_fn(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
187
+ self.echo(f"- {codebase['name']} (https://github.com/{codebase['repository']})")
188
188
  if with_images:
189
- self.list_latest_images_fn(
189
+ self.list_latest_images(
190
190
  ecr_client,
191
191
  f"{application.name}/{codebase['name']}",
192
192
  codebase["repository"],
193
- self.echo_fn,
193
+ self.echo,
194
194
  )
195
195
 
196
- self.echo_fn("")
196
+ self.echo("")
197
197
 
198
198
  def __get_codebases(self, application, ssm_client):
199
199
  parameters = ssm_client.get_parameters_by_path(
@@ -204,19 +204,37 @@ class Codebase:
204
204
  codebases = [json.loads(p["Value"]) for p in parameters]
205
205
 
206
206
  if not codebases:
207
- # TODO Is this really an error? Or just no codebases so we could return an empty list?
208
- raise NoCopilotCodebasesFoundError
207
+ return []
209
208
  return codebases
210
209
 
211
210
  def __start_build_with_confirmation(
212
211
  self,
213
- confirm_fn,
212
+ confirm,
214
213
  codebuild_client,
215
- get_build_url_from_arn_fn,
214
+ get_build_url_from_arn,
216
215
  confirmation_message,
217
216
  build_options,
218
217
  ):
219
- if confirm_fn(confirmation_message):
220
- build_arn = self.start_build_extraction_fn(codebuild_client, build_options)
221
- return get_build_url_from_arn_fn(build_arn)
218
+ if confirm(confirmation_message):
219
+ build_arn = self.start_build_extraction(codebuild_client, build_options)
220
+ return get_build_url_from_arn(build_arn)
222
221
  return None
222
+
223
+
224
+ class ApplicationDeploymentNotTriggered(PlatformException):
225
+ def __init__(self, codebase: str):
226
+ super().__init__(f"""Your deployment for {codebase} was not triggered.""")
227
+
228
+
229
+ class ApplicationEnvironmentNotFoundException(ApplicationException):
230
+ def __init__(self, environment: str):
231
+ super().__init__(
232
+ f"""The environment "{environment}" either does not exist or has not been deployed."""
233
+ )
234
+
235
+
236
+ class NotInCodeBaseRepositoryException(PlatformException):
237
+ def __init__(self):
238
+ super().__init__(
239
+ "You are in the deploy repository; make sure you are in the application codebase repository.",
240
+ )