dbt-platform-helper 12.3.0__tar.gz → 12.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbt-platform-helper might be problematic. Click here for more details.

Files changed (94) hide show
  1. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/PKG-INFO +1 -1
  2. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/COMMANDS.md +6 -1
  3. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/codebase.py +1 -1
  4. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/conduit.py +2 -2
  5. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/config.py +4 -4
  6. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/copilot.py +13 -15
  7. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/database.py +17 -4
  8. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/environment.py +3 -2
  9. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/codebase.py +23 -5
  10. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/conduit.py +0 -6
  11. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/database_copy.py +14 -13
  12. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/maintenance_page.py +8 -8
  13. dbt_platform_helper-12.4.0/dbt_platform_helper/platform_exception.py +5 -0
  14. dbt_platform_helper-12.4.0/dbt_platform_helper/providers/aws.py +32 -0
  15. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/cloudformation.py +8 -1
  16. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/copilot.py +2 -5
  17. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/ecs.py +19 -4
  18. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/load_balancers.py +11 -5
  19. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/secrets.py +51 -10
  20. dbt_platform_helper-12.4.0/dbt_platform_helper/providers/validation.py +19 -0
  21. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/application.py +14 -2
  22. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/arn_parser.py +1 -1
  23. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/aws.py +13 -10
  24. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/git.py +2 -2
  25. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/validation.py +18 -4
  26. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/versioning.py +8 -8
  27. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/pyproject.toml +1 -1
  28. dbt_platform_helper-12.3.0/dbt_platform_helper/addons-template-map.yml +0 -29
  29. dbt_platform_helper-12.3.0/dbt_platform_helper/exceptions.py +0 -147
  30. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/LICENSE +0 -0
  31. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/README.md +0 -0
  32. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/__init__.py +0 -0
  33. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/addon-plans.yml +0 -0
  34. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/__init__.py +0 -0
  35. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/application.py +0 -0
  36. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/generate.py +0 -0
  37. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/notify.py +0 -0
  38. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/pipeline.py +0 -0
  39. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/secrets.py +0 -0
  40. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/commands/version.py +0 -0
  41. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/constants.py +0 -0
  42. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/default-extensions.yml +0 -0
  43. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/domain/__init__.py +0 -0
  44. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/jinja2_tags.py +0 -0
  45. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/providers/__init__.py +0 -0
  46. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/config.yml +0 -0
  47. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/image_build_run.sh +0 -0
  48. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/phases/build.sh +0 -0
  49. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/phases/install.sh +0 -0
  50. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/phases/post_build.sh +0 -0
  51. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/.copilot/phases/pre_build.sh +0 -0
  52. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/COMMANDS.md.jinja +0 -0
  53. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addon-instructions.txt +0 -0
  54. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/README.md +0 -0
  55. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/svc/appconfig-ipfilter.yml +0 -0
  56. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/svc/prometheus-policy.yml +0 -0
  57. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/svc/s3-policy.yml +0 -0
  58. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/addons/svc/subscription-filter.yml +0 -0
  59. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/ci-codebuild-role-policy.json +0 -0
  60. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/create-codebuild-role.json +0 -0
  61. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/custom-codebuild-role-policy.json +0 -0
  62. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/env/manifest.yml +0 -0
  63. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/env/terraform-overrides/cfn.patches.yml +0 -0
  64. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/environment-pipelines/main.tf +0 -0
  65. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/environments/main.tf +0 -0
  66. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/manifest.yml +0 -0
  67. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/.gitignore +0 -0
  68. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/bin/override.ts +0 -0
  69. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.deploy.yml +0 -0
  70. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.image.yml +0 -0
  71. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/cdk.json +0 -0
  72. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json +0 -0
  73. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/package.json +0 -0
  74. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/stack.ts +0 -0
  75. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/tsconfig.json +0 -0
  76. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/types.ts +0 -0
  77. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/environments/buildspec.yml +0 -0
  78. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/environments/manifest.yml +0 -0
  79. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/pipelines/environments/overrides/cfn.patches.yml +0 -0
  80. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/maintenance_pages/default.html +0 -0
  81. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/maintenance_pages/dmas-migration.html +0 -0
  82. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/maintenance_pages/migration.html +0 -0
  83. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/manifest-backend.yml +0 -0
  84. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/manifest-public.yml +0 -0
  85. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +0 -0
  86. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/__init__.py +0 -0
  87. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/click.py +0 -0
  88. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/cloudfoundry.py +0 -0
  89. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/files.py +0 -0
  90. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/manifests.py +0 -0
  91. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/messages.py +0 -0
  92. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/platform_config.py +0 -0
  93. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/dbt_platform_helper/utils/template.py +0 -0
  94. {dbt_platform_helper-12.3.0 → dbt_platform_helper-12.4.0}/platform_helper.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dbt-platform-helper
3
- Version: 12.3.0
3
+ Version: 12.4.0
4
4
  Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
5
5
  License: MIT
6
6
  Author: Department for Business and Trade Platform Team
@@ -765,6 +765,7 @@ platform-helper database (dump|load|copy)
765
765
  ```
766
766
  platform-helper database dump --from <from_env> --database <database>
767
767
  [--app <application>] [--from-vpc <from_vpc>]
768
+ [--filename <filename>]
768
769
  ```
769
770
 
770
771
  ## Options
@@ -777,6 +778,8 @@ platform-helper database dump --from <from_env> --database <database>
777
778
  - The name of the database you are dumping data from
778
779
  - `--from-vpc <text>`
779
780
  - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
781
+ - `--filename <text>`
782
+ - Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments
780
783
  - `--help <boolean>` _Defaults to False._
781
784
  - Show this message and exit.
782
785
 
@@ -791,7 +794,7 @@ platform-helper database dump --from <from_env> --database <database>
791
794
  ```
792
795
  platform-helper database load --to <to_env> --database <database>
793
796
  [--app <application>] [--to-vpc <to_vpc>]
794
- [--auto-approve]
797
+ [--filename <filename>] [--auto-approve]
795
798
  ```
796
799
 
797
800
  ## Options
@@ -806,6 +809,8 @@ platform-helper database load --to <to_env> --database <database>
806
809
  - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
807
810
  - `--auto-approve <boolean>` _Defaults to False._
808
811
 
812
+ - `--filename <text>`
813
+ - Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments
809
814
  - `--help <boolean>` _Defaults to False._
810
815
  - Show this message and exit.
811
816
 
@@ -1,7 +1,7 @@
1
1
  import click
2
2
 
3
3
  from dbt_platform_helper.domain.codebase import Codebase
4
- from dbt_platform_helper.exceptions import PlatformException
4
+ from dbt_platform_helper.platform_exception import PlatformException
5
5
  from dbt_platform_helper.utils.click import ClickDocOptGroup
6
6
  from dbt_platform_helper.utils.versioning import (
7
7
  check_platform_helper_version_needs_update,
@@ -1,7 +1,7 @@
1
1
  import click
2
2
 
3
3
  from dbt_platform_helper.domain.conduit import Conduit
4
- from dbt_platform_helper.exceptions import AWSException
4
+ from dbt_platform_helper.platform_exception import PlatformException
5
5
  from dbt_platform_helper.providers.cloudformation import CloudFormation
6
6
  from dbt_platform_helper.providers.ecs import ECS
7
7
  from dbt_platform_helper.providers.secrets import Secrets
@@ -53,6 +53,6 @@ def conduit(addon_name: str, app: str, env: str, access: str):
53
53
  Conduit(application, secrets_provider, cloudformation_provider, ecs_provider).start(
54
54
  env, addon_name, access
55
55
  )
56
- except AWSException as err:
56
+ except PlatformException as err:
57
57
  click.secho(str(err), fg="red")
58
58
  raise click.Abort
@@ -8,8 +8,8 @@ import botocore
8
8
  import click
9
9
  from prettytable import PrettyTable
10
10
 
11
- from dbt_platform_helper.exceptions import IncompatibleMajorVersion
12
- from dbt_platform_helper.exceptions import ValidationException
11
+ from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
12
+ from dbt_platform_helper.providers.validation import ValidationException
13
13
  from dbt_platform_helper.utils import versioning
14
14
  from dbt_platform_helper.utils.click import ClickDocOptGroup
15
15
  from dbt_platform_helper.utils.validation import config_file_check
@@ -110,7 +110,7 @@ def deployment():
110
110
  str(template_file.resolve())
111
111
  )
112
112
  versioning.validate_template_version(local_version, str(template_file.resolve()))
113
- except IncompatibleMajorVersion:
113
+ except IncompatibleMajorVersionException:
114
114
  local_compatible_symbol = no
115
115
  compatible = False
116
116
  recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
@@ -134,7 +134,7 @@ def deployment():
134
134
  str(template_file.resolve())
135
135
  )
136
136
  versioning.validate_template_version(latest_release, str(template_file.resolve()))
137
- except IncompatibleMajorVersion:
137
+ except IncompatibleMajorVersionException:
138
138
  latest_compatible_symbol = no
139
139
  compatible = False
140
140
  except ValidationException:
@@ -35,6 +35,14 @@ SERVICE_TYPES = [
35
35
  "Worker Service",
36
36
  ]
37
37
 
38
+ ADDON_TEMPLATE_MAP = {
39
+ "s3": ["addons/svc/s3-policy.yml"],
40
+ "s3-policy": ["addons/svc/s3-policy.yml"],
41
+ "appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
42
+ "subscription-filter": ["addons/svc/subscription-filter.yml"],
43
+ "prometheus-policy": ["addons/svc/prometheus-policy.yml"],
44
+ }
45
+
38
46
 
39
47
  def list_copilot_local_environments():
40
48
  return [
@@ -248,9 +256,6 @@ def make_addons():
248
256
 
249
257
  application_name = get_application_name()
250
258
 
251
- with open(PACKAGE_DIR / "addons-template-map.yml") as fd:
252
- addon_template_map = yaml.safe_load(fd)
253
-
254
259
  click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
255
260
 
256
261
  env_path = Path(f"copilot/environments/")
@@ -270,11 +275,6 @@ def make_addons():
270
275
  print(f">>>>>>>>> {addon_name}")
271
276
  addon_type = addon_config.pop("type")
272
277
  environments = addon_config.pop("environments")
273
- if addon_template_map[addon_type].get("requires_addons_parameters", False):
274
- pass
275
- if addon_type in ["postgres"]:
276
- pass
277
-
278
278
  environment_addon_config = {
279
279
  "addon_type": addon_type,
280
280
  "environments": environments,
@@ -309,7 +309,6 @@ def make_addons():
309
309
  _generate_service_addons(
310
310
  addon_config,
311
311
  addon_name,
312
- addon_template_map,
313
312
  addon_type,
314
313
  output_dir,
315
314
  service_addon_config,
@@ -339,7 +338,6 @@ def _generate_env_overrides(output_dir):
339
338
  def _generate_service_addons(
340
339
  addon_config,
341
340
  addon_name,
342
- addon_template_map,
343
341
  addon_type,
344
342
  output_dir,
345
343
  service_addon_config,
@@ -347,8 +345,8 @@ def _generate_service_addons(
347
345
  log_destination_arns,
348
346
  ):
349
347
  # generate svc addons
350
- for addon in addon_template_map[addon_type].get("svc", []):
351
- template = templates.get_template(addon["template"])
348
+ for addon_template in ADDON_TEMPLATE_MAP.get(addon_type, []):
349
+ template = templates.get_template(addon_template)
352
350
 
353
351
  for svc in addon_config.get("services", []):
354
352
  service_path = Path(f"copilot/{svc}/addons/")
@@ -360,10 +358,10 @@ def _generate_service_addons(
360
358
  }
361
359
  )
362
360
 
363
- filename = addon.get("filename", f"{addon_name}.yml")
364
-
365
361
  (output_dir / service_path).mkdir(parents=True, exist_ok=True)
366
- click.echo(mkfile(output_dir, service_path / filename, contents, overwrite=True))
362
+ click.echo(
363
+ mkfile(output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True)
364
+ )
367
365
 
368
366
 
369
367
  def _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path):
@@ -31,10 +31,16 @@ def database():
31
31
  type=str,
32
32
  help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
33
33
  )
34
- def dump(app, from_env, database, from_vpc):
34
+ @click.option(
35
+ "--filename",
36
+ type=str,
37
+ help="Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments",
38
+ )
39
+ def dump(app, from_env, database, from_vpc, filename):
35
40
  """Dump a database into an S3 bucket."""
36
41
  data_copy = DatabaseCopy(app, database)
37
- data_copy.dump(from_env, from_vpc)
42
+ data_copy.dump(from_env, from_vpc, filename)
43
+ # Todo: Catch expected errors and output message
38
44
 
39
45
 
40
46
  @database.command(name="load")
@@ -55,10 +61,16 @@ def dump(app, from_env, database, from_vpc):
55
61
  help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
56
62
  )
57
63
  @click.option("--auto-approve/--no-auto-approve", default=False)
58
- def load(app, to_env, database, to_vpc, auto_approve):
64
+ @click.option(
65
+ "--filename",
66
+ type=str,
67
+ help="Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments",
68
+ )
69
+ def load(app, to_env, database, to_vpc, auto_approve, filename):
59
70
  """Load a database from an S3 bucket."""
60
71
  data_copy = DatabaseCopy(app, database, auto_approve)
61
- data_copy.load(to_env, to_vpc)
72
+ data_copy.load(to_env, to_vpc, filename)
73
+ # Todo: Catch expected errors and output message
62
74
 
63
75
 
64
76
  @database.command(name="copy")
@@ -110,3 +122,4 @@ def copy(
110
122
  """Copy a database between environments."""
111
123
  data_copy = DatabaseCopy(app, database, auto_approve)
112
124
  data_copy.copy(from_env, to_env, from_vpc, to_vpc, svc, template, no_maintenance_page)
125
+ # Todo: Catch expected errors and output message
@@ -5,6 +5,7 @@ from schema import SchemaError
5
5
  from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
6
6
  from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
7
7
  from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
8
+ from dbt_platform_helper.platform_exception import PlatformException
8
9
  from dbt_platform_helper.providers.load_balancers import find_https_listener
9
10
  from dbt_platform_helper.utils.aws import get_aws_session_or_abort
10
11
  from dbt_platform_helper.utils.click import ClickDocOptGroup
@@ -238,10 +239,10 @@ def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
238
239
  try:
239
240
  certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
240
241
  except StopIteration:
241
- raise CertificateNotFoundError()
242
+ raise CertificateNotFoundException()
242
243
 
243
244
  return certificate_arn
244
245
 
245
246
 
246
- class CertificateNotFoundError(Exception):
247
+ class CertificateNotFoundException(PlatformException):
247
248
  pass
@@ -9,10 +9,9 @@ import requests
9
9
  import yaml
10
10
  from boto3 import Session
11
11
 
12
- from dbt_platform_helper.exceptions import ApplicationDeploymentNotTriggered
13
- from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
14
- from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
12
+ from dbt_platform_helper.platform_exception import PlatformException
15
13
  from dbt_platform_helper.utils.application import Application
14
+ from dbt_platform_helper.utils.application import ApplicationException
16
15
  from dbt_platform_helper.utils.application import load_application
17
16
  from dbt_platform_helper.utils.aws import check_codebase_exists
18
17
  from dbt_platform_helper.utils.aws import check_image_exists
@@ -68,7 +67,7 @@ class Codebase:
68
67
  .removesuffix(".git")
69
68
  )
70
69
  if repository.endswith("-deploy") or Path("./copilot").exists():
71
- raise NotInCodeBaseRepositoryError()
70
+ raise NotInCodeBaseRepositoryException()
72
71
 
73
72
  builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
74
73
  builder_configuration_response = requests.get(builder_configuration_url)
@@ -143,7 +142,7 @@ class Codebase:
143
142
 
144
143
  application = self.load_application(app, default_session=session)
145
144
  if not application.environments.get(env):
146
- raise ApplicationEnvironmentNotFoundError(env)
145
+ raise ApplicationEnvironmentNotFoundException(env)
147
146
 
148
147
  self.check_codebase_exists(session, application, codebase)
149
148
 
@@ -220,3 +219,22 @@ class Codebase:
220
219
  build_arn = self.start_build_extraction(codebuild_client, build_options)
221
220
  return get_build_url_from_arn(build_arn)
222
221
  return None
222
+
223
+
224
+ class ApplicationDeploymentNotTriggered(PlatformException):
225
+ def __init__(self, codebase: str):
226
+ super().__init__(f"""Your deployment for {codebase} was not triggered.""")
227
+
228
+
229
+ class ApplicationEnvironmentNotFoundException(ApplicationException):
230
+ def __init__(self, environment: str):
231
+ super().__init__(
232
+ f"""The environment "{environment}" either does not exist or has not been deployed."""
233
+ )
234
+
235
+
236
+ class NotInCodeBaseRepositoryException(PlatformException):
237
+ def __init__(self):
238
+ super().__init__(
239
+ "You are in the deploy repository; make sure you are in the application codebase repository.",
240
+ )
@@ -6,7 +6,6 @@ import click
6
6
  from dbt_platform_helper.providers.cloudformation import CloudFormation
7
7
  from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
8
8
  from dbt_platform_helper.providers.copilot import create_addon_client_task
9
- from dbt_platform_helper.providers.copilot import create_postgres_admin_task
10
9
  from dbt_platform_helper.providers.ecs import ECS
11
10
  from dbt_platform_helper.providers.secrets import Secrets
12
11
  from dbt_platform_helper.utils.application import Application
@@ -23,7 +22,6 @@ class Conduit:
23
22
  subprocess: subprocess = subprocess,
24
23
  connect_to_addon_client_task=connect_to_addon_client_task,
25
24
  create_addon_client_task=create_addon_client_task,
26
- create_postgres_admin_task=create_postgres_admin_task,
27
25
  ):
28
26
 
29
27
  self.application = application
@@ -34,7 +32,6 @@ class Conduit:
34
32
  self.echo = echo
35
33
  self.connect_to_addon_client_task = connect_to_addon_client_task
36
34
  self.create_addon_client_task = create_addon_client_task
37
- self.create_postgres_admin_task = create_postgres_admin_task
38
35
 
39
36
  def start(self, env: str, addon_name: str, access: str = "read"):
40
37
  clients = self._initialise_clients(env)
@@ -49,7 +46,6 @@ class Conduit:
49
46
  self.create_addon_client_task(
50
47
  clients["iam"],
51
48
  clients["ssm"],
52
- clients["secrets_manager"],
53
49
  self.subprocess,
54
50
  self.application,
55
51
  env,
@@ -89,8 +85,6 @@ class Conduit:
89
85
  "ecs": self.application.environments[env].session.client("ecs"),
90
86
  "iam": self.application.environments[env].session.client("iam"),
91
87
  "ssm": self.application.environments[env].session.client("ssm"),
92
- "cloudformation": self.application.environments[env].session.client("cloudformation"),
93
- "secrets_manager": self.application.environments[env].session.client("secretsmanager"),
94
88
  }
95
89
 
96
90
  def _get_addon_details(self, addon_name, access):
@@ -8,9 +8,9 @@ from boto3 import Session
8
8
 
9
9
  from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
10
10
  from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
11
- from dbt_platform_helper.exceptions import ApplicationNotFoundError
12
- from dbt_platform_helper.exceptions import AWSException
11
+ from dbt_platform_helper.providers.aws import AWSException
13
12
  from dbt_platform_helper.utils.application import Application
13
+ from dbt_platform_helper.utils.application import ApplicationNotFoundException
14
14
  from dbt_platform_helper.utils.application import load_application
15
15
  from dbt_platform_helper.utils.aws import Vpc
16
16
  from dbt_platform_helper.utils.aws import get_connection_string
@@ -58,10 +58,10 @@ class DatabaseCopy:
58
58
 
59
59
  try:
60
60
  self.application = load_application(self.app)
61
- except ApplicationNotFoundError:
61
+ except ApplicationNotFoundException:
62
62
  abort(f"No such application '{app}'.")
63
63
 
64
- def _execute_operation(self, is_dump: bool, env: str, vpc_name: str, to_env: str):
64
+ def _execute_operation(self, is_dump: bool, env: str, vpc_name: str, filename: str):
65
65
  vpc_name = self.enrich_vpc_name(env, vpc_name)
66
66
 
67
67
  environments = self.application.environments
@@ -89,7 +89,7 @@ class DatabaseCopy:
89
89
 
90
90
  try:
91
91
  task_arn = self.run_database_copy_task(
92
- env_session, env, vpc_config, is_dump, db_connection_string, to_env
92
+ env_session, env, vpc_config, is_dump, db_connection_string, filename
93
93
  )
94
94
  except Exception as exc:
95
95
  self.abort(f"{exc} (Account id: {self.account_id(env)})")
@@ -122,14 +122,15 @@ class DatabaseCopy:
122
122
  vpc_config: Vpc,
123
123
  is_dump: bool,
124
124
  db_connection_string: str,
125
- to_env: str,
125
+ filename: str,
126
126
  ) -> str:
127
127
  client = session.client("ecs")
128
128
  action = "dump" if is_dump else "load"
129
+ dump_file_name = filename if filename else "data_dump"
129
130
  env_vars = [
130
131
  {"name": "DATA_COPY_OPERATION", "value": action.upper()},
131
132
  {"name": "DB_CONNECTION_STRING", "value": db_connection_string},
132
- {"name": "TO_ENVIRONMENT", "value": to_env},
133
+ {"name": "DUMP_FILE_NAME", "value": dump_file_name},
133
134
  ]
134
135
  if not is_dump:
135
136
  env_vars.append({"name": "ECS_CLUSTER", "value": f"{self.app}-{env}"})
@@ -159,12 +160,12 @@ class DatabaseCopy:
159
160
 
160
161
  return response.get("tasks", [{}])[0].get("taskArn")
161
162
 
162
- def dump(self, env: str, vpc_name: str, to_env: str):
163
- self._execute_operation(True, env, vpc_name, to_env)
163
+ def dump(self, env: str, vpc_name: str, filename: str = None):
164
+ self._execute_operation(True, env, vpc_name, filename)
164
165
 
165
- def load(self, env: str, vpc_name: str):
166
+ def load(self, env: str, vpc_name: str, filename: str = None):
166
167
  if self.is_confirmed_ready_to_load(env):
167
- self._execute_operation(False, env, vpc_name, to_env=env)
168
+ self._execute_operation(False, env, vpc_name, filename)
168
169
 
169
170
  def copy(
170
171
  self,
@@ -179,8 +180,8 @@ class DatabaseCopy:
179
180
  to_vpc = self.enrich_vpc_name(to_env, to_vpc)
180
181
  if not no_maintenance_page:
181
182
  self.maintenance_page_provider.activate(self.app, to_env, services, template, to_vpc)
182
- self.dump(from_env, from_vpc, to_env)
183
- self.load(to_env, to_vpc)
183
+ self.dump(from_env, from_vpc, f"data_dump_{to_env}")
184
+ self.load(to_env, to_vpc, f"data_dump_{to_env}")
184
185
  if not no_maintenance_page:
185
186
  self.maintenance_page_provider.deactivate(self.app, to_env)
186
187
 
@@ -9,9 +9,9 @@ from typing import Union
9
9
  import boto3
10
10
  import click
11
11
 
12
- from dbt_platform_helper.providers.load_balancers import ListenerNotFoundError
13
- from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundError
14
- from dbt_platform_helper.providers.load_balancers import LoadBalancerNotFoundError
12
+ from dbt_platform_helper.providers.load_balancers import ListenerNotFoundException
13
+ from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundException
14
+ from dbt_platform_helper.providers.load_balancers import LoadBalancerNotFoundException
15
15
  from dbt_platform_helper.providers.load_balancers import find_https_listener
16
16
  from dbt_platform_helper.utils.application import Environment
17
17
  from dbt_platform_helper.utils.application import Service
@@ -75,13 +75,13 @@ class MaintenancePageProvider:
75
75
  else:
76
76
  raise click.Abort
77
77
 
78
- except LoadBalancerNotFoundError:
78
+ except LoadBalancerNotFoundException:
79
79
  click.secho(
80
80
  f"No load balancer found for environment {env} in the application {app}.", fg="red"
81
81
  )
82
82
  raise click.Abort
83
83
 
84
- except ListenerNotFoundError:
84
+ except ListenerNotFoundException:
85
85
  click.secho(
86
86
  f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
87
87
  )
@@ -110,13 +110,13 @@ class MaintenancePageProvider:
110
110
  f"Maintenance page removed from environment {env} in application {app}", fg="green"
111
111
  )
112
112
 
113
- except LoadBalancerNotFoundError:
113
+ except LoadBalancerNotFoundException:
114
114
  click.secho(
115
115
  f"No load balancer found for environment {env} in the application {app}.", fg="red"
116
116
  )
117
117
  raise click.Abort
118
118
 
119
- except ListenerNotFoundError:
119
+ except ListenerNotFoundException:
120
120
  click.secho(
121
121
  f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
122
122
  )
@@ -180,7 +180,7 @@ def remove_maintenance_page(session: boto3.Session, listener_arn: str):
180
180
  deleted = delete_listener_rule(tag_descriptions, name, lb_client)
181
181
 
182
182
  if name == "MaintenancePage" and not deleted:
183
- raise ListenerRuleNotFoundError()
183
+ raise ListenerRuleNotFoundException()
184
184
 
185
185
 
186
186
  def get_rules_tag_descriptions(rules: list, lb_client):
@@ -0,0 +1,5 @@
1
+ # This exception exists so that we can easily catch exceptions
2
+ # at the command level where we know we can just output the
3
+ # error and abort.
4
+ class PlatformException(Exception):
5
+ pass
@@ -0,0 +1,32 @@
1
+ from dbt_platform_helper.platform_exception import PlatformException
2
+
3
+
4
+ class AWSException(PlatformException):
5
+ pass
6
+
7
+
8
+ class CreateTaskTimeoutException(AWSException):
9
+ def __init__(self, addon_name: str, application_name: str, environment: str):
10
+ super().__init__(
11
+ f"""Client ({addon_name}) ECS task has failed to start for "{application_name}" in "{environment}" environment."""
12
+ )
13
+
14
+
15
+ class ImageNotFoundException(AWSException):
16
+ def __init__(self, commit: str):
17
+ super().__init__(
18
+ f"""The commit hash "{commit}" has not been built into an image, try the `platform-helper codebase build` command first."""
19
+ )
20
+
21
+
22
+ class LogGroupNotFoundException(AWSException):
23
+ def __init__(self, log_group_name: str):
24
+ super().__init__(f"""No log group called "{log_group_name}".""")
25
+
26
+
27
+ # Todo: This should probably be in the AWS Copilot provider, but was causing circular import when we tried it pre refactoring the utils/aws.py
28
+ class CopilotCodebaseNotFoundException(PlatformException):
29
+ def __init__(self, codebase: str):
30
+ super().__init__(
31
+ f"""The codebase "{codebase}" either does not exist or has not been deployed."""
32
+ )
@@ -4,7 +4,7 @@ import botocore
4
4
  from cfn_tools import dump_yaml
5
5
  from cfn_tools import load_yaml
6
6
 
7
- from dbt_platform_helper.exceptions import CloudFormationException
7
+ from dbt_platform_helper.platform_exception import PlatformException
8
8
 
9
9
 
10
10
  class CloudFormation:
@@ -125,3 +125,10 @@ class CloudFormation:
125
125
  raise CloudFormationException(
126
126
  stack_name, f"Error while waiting for stack status: {str(err)}"
127
127
  )
128
+
129
+
130
+ class CloudFormationException(PlatformException):
131
+ def __init__(self, stack_name: str, current_status: str):
132
+ super().__init__(
133
+ f"The CloudFormation stack '{stack_name}' is not in a good state: {current_status}"
134
+ )
@@ -4,7 +4,7 @@ import time
4
4
  from botocore.exceptions import ClientError
5
5
 
6
6
  from dbt_platform_helper.constants import CONDUIT_DOCKER_IMAGE_LOCATION
7
- from dbt_platform_helper.exceptions import CreateTaskTimeoutError
7
+ from dbt_platform_helper.providers.aws import CreateTaskTimeoutException
8
8
  from dbt_platform_helper.providers.secrets import Secrets
9
9
  from dbt_platform_helper.utils.application import Application
10
10
  from dbt_platform_helper.utils.messages import abort_with_error
@@ -13,7 +13,6 @@ from dbt_platform_helper.utils.messages import abort_with_error
13
13
  def create_addon_client_task(
14
14
  iam_client,
15
15
  ssm_client,
16
- secrets_manager_client,
17
16
  subprocess,
18
17
  application: Application,
19
18
  env: str,
@@ -32,7 +31,6 @@ def create_addon_client_task(
32
31
  elif access == "admin":
33
32
  create_postgres_admin_task(
34
33
  ssm_client,
35
- secrets_manager_client,
36
34
  subprocess,
37
35
  application,
38
36
  addon_name,
@@ -74,7 +72,6 @@ def create_addon_client_task(
74
72
 
75
73
  def create_postgres_admin_task(
76
74
  ssm_client,
77
- secrets_manager_client,
78
75
  subprocess,
79
76
  app: Application,
80
77
  addon_name: str,
@@ -147,7 +144,7 @@ def connect_to_addon_client_task(
147
144
  time.sleep(1)
148
145
 
149
146
  if not running:
150
- raise CreateTaskTimeoutError(task_name, application_name, env)
147
+ raise CreateTaskTimeoutException(task_name, application_name, env)
151
148
 
152
149
 
153
150
  def _normalise_secret_name(addon_name: str) -> str:
@@ -3,8 +3,7 @@ import string
3
3
  import time
4
4
  from typing import List
5
5
 
6
- from dbt_platform_helper.exceptions import ECSAgentNotRunning
7
- from dbt_platform_helper.exceptions import NoClusterError
6
+ from dbt_platform_helper.platform_exception import PlatformException
8
7
 
9
8
 
10
9
  class ECS:
@@ -36,7 +35,7 @@ class ECS:
36
35
  if app_key_found and env_key_found and cluster_key_found:
37
36
  return cluster_arn
38
37
 
39
- raise NoClusterError(self.application_name, self.env)
38
+ raise NoClusterException(self.application_name, self.env)
40
39
 
41
40
  def get_or_create_task_name(self, addon_name: str, parameter_name: str) -> str:
42
41
  """Fetches the task name from SSM or creates a new one if not found."""
@@ -84,4 +83,20 @@ class ECS:
84
83
  time.sleep(1)
85
84
 
86
85
  if execute_command_agent_status != "RUNNING":
87
- raise ECSAgentNotRunning
86
+ raise ECSAgentNotRunningException
87
+
88
+
89
+ class ECSException(PlatformException):
90
+ pass
91
+
92
+
93
+ class ECSAgentNotRunningException(ECSException):
94
+ def __init__(self):
95
+ super().__init__("""ECS exec agent never reached "RUNNING" status""")
96
+
97
+
98
+ class NoClusterException(ECSException):
99
+ def __init__(self, application_name: str, environment: str):
100
+ super().__init__(
101
+ f"""No ECS cluster found for "{application_name}" in "{environment}" environment."""
102
+ )
@@ -1,5 +1,7 @@
1
1
  import boto3
2
2
 
3
+ from dbt_platform_helper.platform_exception import PlatformException
4
+
3
5
 
4
6
  def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
5
7
  lb_client = session.client("elbv2")
@@ -16,7 +18,7 @@ def find_load_balancer(session: boto3.Session, app: str, env: str) -> str:
16
18
  load_balancer_arn = lb["ResourceArn"]
17
19
 
18
20
  if not load_balancer_arn:
19
- raise LoadBalancerNotFoundError()
21
+ raise LoadBalancerNotFoundException()
20
22
 
21
23
  return load_balancer_arn
22
24
 
@@ -34,18 +36,22 @@ def find_https_listener(session: boto3.Session, app: str, env: str) -> str:
34
36
  pass
35
37
 
36
38
  if not listener_arn:
37
- raise ListenerNotFoundError()
39
+ raise ListenerNotFoundException()
38
40
 
39
41
  return listener_arn
40
42
 
41
43
 
42
- class LoadBalancerNotFoundError(Exception):
44
+ class LoadBalancerException(PlatformException):
45
+ pass
46
+
47
+
48
+ class LoadBalancerNotFoundException(LoadBalancerException):
43
49
  pass
44
50
 
45
51
 
46
- class ListenerNotFoundError(Exception):
52
+ class ListenerNotFoundException(LoadBalancerException):
47
53
  pass
48
54
 
49
55
 
50
- class ListenerRuleNotFoundError(Exception):
56
+ class ListenerRuleNotFoundException(LoadBalancerException):
51
57
  pass