dbt-platform-helper 11.0.1__tar.gz → 11.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dbt-platform-helper might be problematic. Click here for more details.

Files changed (109) hide show
  1. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/PKG-INFO +2 -1
  2. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/COMMANDS.md +83 -20
  3. dbt_platform_helper-11.2.0/dbt_platform_helper/commands/database.py +112 -0
  4. dbt_platform_helper-11.2.0/dbt_platform_helper/commands/environment.py +218 -0
  5. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/pipeline.py +103 -12
  6. dbt_platform_helper-11.2.0/dbt_platform_helper/domain/database_copy.py +220 -0
  7. dbt_platform_helper-11.2.0/dbt_platform_helper/domain/maintenance_page.py +459 -0
  8. dbt_platform_helper-11.2.0/dbt_platform_helper/providers/load_balancers.py +51 -0
  9. dbt_platform_helper-11.2.0/dbt_platform_helper/templates/environment-pipelines/main.tf +52 -0
  10. dbt_platform_helper-11.2.0/dbt_platform_helper/utils/__init__.py +0 -0
  11. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/aws.py +47 -37
  12. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/validation.py +29 -0
  13. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/pyproject.toml +2 -1
  14. dbt_platform_helper-11.0.1/dbt_platform_helper/commands/database.py +0 -33
  15. dbt_platform_helper-11.0.1/dbt_platform_helper/commands/database_helpers.py +0 -145
  16. dbt_platform_helper-11.0.1/dbt_platform_helper/commands/environment.py +0 -730
  17. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/LICENSE +0 -0
  18. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/README.md +0 -0
  19. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/__init__.py +0 -0
  20. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/addon-plans.yml +0 -0
  21. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/addons-template-map.yml +0 -0
  22. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/__init__.py +0 -0
  23. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/application.py +0 -0
  24. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/check_cloudformation.py +0 -0
  25. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/codebase.py +0 -0
  26. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/conduit.py +0 -0
  27. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/config.py +0 -0
  28. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/copilot.py +0 -0
  29. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/dns.py +0 -0
  30. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/generate.py +0 -0
  31. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/notify.py +0 -0
  32. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/secrets.py +0 -0
  33. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/commands/version.py +0 -0
  34. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/constants.py +0 -0
  35. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/custom_resources/__init__.py +0 -0
  36. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/custom_resources/s3_object.py +0 -0
  37. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/default-extensions.yml +0 -0
  38. {dbt_platform_helper-11.0.1/dbt_platform_helper/utils → dbt_platform_helper-11.2.0/dbt_platform_helper/domain}/__init__.py +0 -0
  39. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/exceptions.py +0 -0
  40. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/jinja2_tags.py +0 -0
  41. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/.copilot/config.yml +0 -0
  42. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/.copilot/image_build_run.sh +0 -0
  43. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/.copilot/phases/build.sh +0 -0
  44. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/.copilot/phases/install.sh +0 -0
  45. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/.copilot/phases/post_build.sh +0 -0
  46. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/.copilot/phases/pre_build.sh +0 -0
  47. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/COMMANDS.md.jinja +0 -0
  48. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addon-instructions.txt +0 -0
  49. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/README.md +0 -0
  50. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/env/addons.parameters.yml +0 -0
  51. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/env/aurora-postgres.yml +0 -0
  52. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/env/monitoring.yml +0 -0
  53. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/env/opensearch.yml +0 -0
  54. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/env/rds-postgres.yml +0 -0
  55. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/env/redis-cluster.yml +0 -0
  56. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/env/s3.yml +0 -0
  57. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/env/vpc.yml +0 -0
  58. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/svc/appconfig-ipfilter.yml +0 -0
  59. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/svc/prometheus-policy.yml +0 -0
  60. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/svc/s3-policy.yml +0 -0
  61. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/addons/svc/subscription-filter.yml +0 -0
  62. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/ci-codebuild-role-policy.json +0 -0
  63. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/create-codebuild-role.json +0 -0
  64. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/custom-codebuild-role-policy.json +0 -0
  65. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/manifest.yml +0 -0
  66. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/.gitignore +0 -0
  67. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/README.md +0 -0
  68. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/bin/override.ts +0 -0
  69. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/cdk.json +0 -0
  70. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/log_resource_policy.json +0 -0
  71. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/package-lock.json +0 -0
  72. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/package.json +0 -0
  73. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/stack.ts +0 -0
  74. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/overrides/tsconfig.json +0 -0
  75. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/env/terraform-overrides/cfn.patches.yml +0 -0
  76. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/environments/main.tf +0 -0
  77. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/manifest.yml +0 -0
  78. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/.gitignore +0 -0
  79. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/bin/override.ts +0 -0
  80. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.deploy.yml +0 -0
  81. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/buildspec.image.yml +0 -0
  82. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/cdk.json +0 -0
  83. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/package-lock.json +0 -0
  84. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/package.json +0 -0
  85. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/stack.ts +0 -0
  86. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/tsconfig.json +0 -0
  87. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/codebase/overrides/types.ts +0 -0
  88. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/environments/buildspec.yml +0 -0
  89. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/environments/manifest.yml +0 -0
  90. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/pipelines/environments/overrides/cfn.patches.yml +0 -0
  91. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/svc/maintenance_pages/default.html +0 -0
  92. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/svc/maintenance_pages/dmas-migration.html +0 -0
  93. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/svc/maintenance_pages/migration.html +0 -0
  94. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/svc/manifest-backend.yml +0 -0
  95. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/svc/manifest-public.yml +0 -0
  96. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +0 -0
  97. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/application.py +0 -0
  98. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/arn_parser.py +0 -0
  99. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/click.py +0 -0
  100. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/cloudformation.py +0 -0
  101. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/cloudfoundry.py +0 -0
  102. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/files.py +0 -0
  103. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/git.py +0 -0
  104. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/manifests.py +0 -0
  105. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/messages.py +0 -0
  106. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/platform_config.py +0 -0
  107. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/template.py +0 -0
  108. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/dbt_platform_helper/utils/versioning.py +0 -0
  109. {dbt_platform_helper-11.0.1 → dbt_platform_helper-11.2.0}/platform_helper.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dbt-platform-helper
3
- Version: 11.0.1
3
+ Version: 11.2.0
4
4
  Summary: Set of tools to help transfer applications/services from GOV.UK PaaS to DBT PaaS augmenting AWS Copilot.
5
5
  License: MIT
6
6
  Author: Department for Business and Trade Platform Team
@@ -34,6 +34,7 @@ Requires-Dist: schema (==0.7.5)
34
34
  Requires-Dist: semver (>=3.0.2,<4.0.0)
35
35
  Requires-Dist: slack-sdk (>=3.27.1,<4.0.0)
36
36
  Requires-Dist: tomlkit (>=0.12.2,<0.13.0)
37
+ Requires-Dist: yamllint (>=1.35.1,<2.0.0)
37
38
  Description-Content-Type: text/markdown
38
39
 
39
40
  # DBT Platform Helper
@@ -42,6 +42,7 @@
42
42
  - [platform-helper database](#platform-helper-database)
43
43
  - [platform-helper database dump](#platform-helper-database-dump)
44
44
  - [platform-helper database load](#platform-helper-database-load)
45
+ - [platform-helper database copy](#platform-helper-database-copy)
45
46
  - [platform-helper version](#platform-helper-version)
46
47
  - [platform-helper version get-platform-helper-for-project](#platform-helper-version-get-platform-helper-for-project)
47
48
 
@@ -792,14 +793,36 @@ platform-helper pipeline generate
792
793
  Given a platform-config.yml file, generate environment and service
793
794
  deployment pipelines.
794
795
 
796
+ This command does the following in relation to the environment pipelines:
797
+ - Reads contents of `platform-config.yml/environment-pipelines` configuration.
798
+ The `terraform/environment-pipelines/<aws_account>/main.tf` file is generated using this configuration.
799
+ The `main.tf` file is then used to generate Terraform for creating an environment pipeline resource.
800
+
801
+ This command does the following in relation to the codebase pipelines:
802
+ - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
803
+
804
+ (Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
805
+ - Generates the copilot manifest.yml for copilot/environments/<environment>
806
+
795
807
  ## Usage
796
808
 
797
809
  ```
798
- platform-helper pipeline generate
810
+ platform-helper pipeline generate [--terraform-platform-modules-version <terraform_platform_modules_version>]
811
+ [--deploy-branch <deploy_branch>]
799
812
  ```
800
813
 
801
814
  ## Options
802
815
 
816
+ - `--terraform-platform-modules-version <text>`
817
+ - Override the default version of terraform-platform-modules with a specific version or branch.
818
+ Precedence of version used is version supplied via CLI, then the version found in
819
+ platform-config.yml/default_versions/terraform-platform-modules.
820
+ In absence of these inputs, defaults to version '5'.
821
+ - `--deploy-branch <text>`
822
+ - Specify the branch of <application>-deploy used to configure the source stage in the environment-pipeline resource.
823
+ This is generated from the terraform/environments-pipeline/<aws_account>/main.tf file.
824
+ (Default <application>-deploy branch is specified in
825
+ <application>-deploy/platform-config.yml/environment_pipelines/<environment-pipeline>/branch).
803
826
  - `--help <boolean>` _Defaults to False._
804
827
  - Show this message and exit.
805
828
 
@@ -962,10 +985,12 @@ platform-helper notify add-comment <slack_channel_id> <slack_token>
962
985
 
963
986
  [↩ Parent](#platform-helper)
964
987
 
988
+ Commands to copy data between databases.
989
+
965
990
  ## Usage
966
991
 
967
992
  ```
968
- platform-helper database (dump|load)
993
+ platform-helper database (dump|load|copy)
969
994
  ```
970
995
 
971
996
  ## Options
@@ -975,6 +1000,7 @@ platform-helper database (dump|load)
975
1000
 
976
1001
  ## Commands
977
1002
 
1003
+ - [`copy` ↪](#platform-helper-database-copy)
978
1004
  - [`dump` ↪](#platform-helper-database-dump)
979
1005
  - [`load` ↪](#platform-helper-database-load)
980
1006
 
@@ -987,23 +1013,20 @@ platform-helper database (dump|load)
987
1013
  ## Usage
988
1014
 
989
1015
  ```
990
- platform-helper database dump --account-id <account_id> --app <application>
991
- --env <environment> --database <database>
992
- --vpc-name <vpc_name>
1016
+ platform-helper database dump --from <from_env> --database <database>
1017
+ [--app <application>] [--from-vpc <from_vpc>]
993
1018
  ```
994
1019
 
995
1020
  ## Options
996
1021
 
997
- - `--account-id <text>`
998
-
999
1022
  - `--app <text>`
1000
-
1001
- - `--env <text>`
1002
-
1023
+ - The application name. Required unless you are running the command from your deploy repo
1024
+ - `--from <text>`
1025
+ - The environment you are dumping data from
1003
1026
  - `--database <text>`
1004
-
1005
- - `--vpc-name <text>`
1006
-
1027
+ - The name of the database you are dumping data from
1028
+ - `--from-vpc <text>`
1029
+ - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
1007
1030
  - `--help <boolean>` _Defaults to False._
1008
1031
  - Show this message and exit.
1009
1032
 
@@ -1016,22 +1039,62 @@ platform-helper database dump --account-id <account_id> --app <application>
1016
1039
  ## Usage
1017
1040
 
1018
1041
  ```
1019
- platform-helper database load --account-id <account_id> --app <application>
1020
- --env <environment> --database <database>
1021
- --vpc-name <vpc_name>
1042
+ platform-helper database load --to <to_env> --database <database>
1043
+ [--app <application>] [--to-vpc <to_vpc>]
1044
+ [--auto-approve]
1022
1045
  ```
1023
1046
 
1024
1047
  ## Options
1025
1048
 
1026
- - `--account-id <text>`
1027
-
1028
1049
  - `--app <text>`
1050
+ - The application name. Required unless you are running the command from your deploy repo
1051
+ - `--to <text>`
1052
+ - The environment you are loading data into
1053
+ - `--database <text>`
1054
+ - The name of the database you are loading data into
1055
+ - `--to-vpc <text>`
1056
+ - The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
1057
+ - `--auto-approve <boolean>` _Defaults to False._
1029
1058
 
1030
- - `--env <text>`
1059
+ - `--help <boolean>` _Defaults to False._
1060
+ - Show this message and exit.
1031
1061
 
1062
+ # platform-helper database copy
1063
+
1064
+ [↩ Parent](#platform-helper-database)
1065
+
1066
+ Copy a database between environments.
1067
+
1068
+ ## Usage
1069
+
1070
+ ```
1071
+ platform-helper database copy --from <from_env> --to <to_env> --database <database>
1072
+ --svc <service> [--app <application>] [--from-vpc <from_vpc>]
1073
+ [--to-vpc <to_vpc>] [--template (default|migration|dmas-migration)]
1074
+ [--auto-approve] [--no-maintenance-page]
1075
+ ```
1076
+
1077
+ ## Options
1078
+
1079
+ - `--app <text>`
1080
+ - The application name. Required unless you are running the command from your deploy repo
1081
+ - `--from <text>`
1082
+ - The environment you are copying data from
1083
+ - `--to <text>`
1084
+ - The environment you are copying data into
1032
1085
  - `--database <text>`
1086
+ - The name of the database you are copying
1087
+ - `--from-vpc <text>`
1088
+ - The vpc the environment you are copying from is running in. Required unless you are running the command from your deploy repo
1089
+ - `--to-vpc <text>`
1090
+ - The vpc the environment you are copying into is running in. Required unless you are running the command from your deploy repo
1091
+ - `--auto-approve <boolean>` _Defaults to False._
1033
1092
 
1034
- - `--vpc-name <text>`
1093
+ - `--svc <text>` _Defaults to ['web']._
1094
+
1095
+ - `--template <choice>` _Defaults to default._
1096
+ - The maintenance page you wish to put up.
1097
+ - `--no-maintenance-page <boolean>` _Defaults to False._
1035
1098
 
1036
1099
  - `--help <boolean>` _Defaults to False._
1037
1100
  - Show this message and exit.
@@ -0,0 +1,112 @@
1
+ import click
2
+
3
+ from dbt_platform_helper.commands.environment import AVAILABLE_TEMPLATES
4
+ from dbt_platform_helper.domain.database_copy import DatabaseCopy
5
+ from dbt_platform_helper.utils.click import ClickDocOptGroup
6
+
7
+
8
+ @click.group(chain=True, cls=ClickDocOptGroup)
9
+ def database():
10
+ """Commands to copy data between databases."""
11
+
12
+
13
+ @database.command(name="dump")
14
+ @click.option(
15
+ "--app",
16
+ type=str,
17
+ help="The application name. Required unless you are running the command from your deploy repo",
18
+ )
19
+ @click.option(
20
+ "--from",
21
+ "from_env",
22
+ type=str,
23
+ required=True,
24
+ help="The environment you are dumping data from",
25
+ )
26
+ @click.option(
27
+ "--database", type=str, required=True, help="The name of the database you are dumping data from"
28
+ )
29
+ @click.option(
30
+ "--from-vpc",
31
+ type=str,
32
+ help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
33
+ )
34
+ def dump(app, from_env, database, from_vpc):
35
+ """Dump a database into an S3 bucket."""
36
+ data_copy = DatabaseCopy(app, database)
37
+ data_copy.dump(from_env, from_vpc)
38
+
39
+
40
+ @database.command(name="load")
41
+ @click.option(
42
+ "--app",
43
+ type=str,
44
+ help="The application name. Required unless you are running the command from your deploy repo",
45
+ )
46
+ @click.option(
47
+ "--to", "to_env", type=str, required=True, help="The environment you are loading data into"
48
+ )
49
+ @click.option(
50
+ "--database", type=str, required=True, help="The name of the database you are loading data into"
51
+ )
52
+ @click.option(
53
+ "--to-vpc",
54
+ type=str,
55
+ help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
56
+ )
57
+ @click.option("--auto-approve/--no-auto-approve", default=False)
58
+ def load(app, to_env, database, to_vpc, auto_approve):
59
+ """Load a database from an S3 bucket."""
60
+ data_copy = DatabaseCopy(app, database, auto_approve)
61
+ data_copy.load(to_env, to_vpc)
62
+
63
+
64
+ @database.command(name="copy")
65
+ @click.option(
66
+ "--app",
67
+ type=str,
68
+ help="The application name. Required unless you are running the command from your deploy repo",
69
+ )
70
+ @click.option(
71
+ "--from", "from_env", type=str, required=True, help="The environment you are copying data from"
72
+ )
73
+ @click.option(
74
+ "--to", "to_env", type=str, required=True, help="The environment you are copying data into"
75
+ )
76
+ @click.option(
77
+ "--database", type=str, required=True, help="The name of the database you are copying"
78
+ )
79
+ @click.option(
80
+ "--from-vpc",
81
+ type=str,
82
+ help="The vpc the environment you are copying from is running in. Required unless you are running the command from your deploy repo",
83
+ )
84
+ @click.option(
85
+ "--to-vpc",
86
+ type=str,
87
+ help="The vpc the environment you are copying into is running in. Required unless you are running the command from your deploy repo",
88
+ )
89
+ @click.option("--auto-approve/--no-auto-approve", default=False)
90
+ @click.option("--svc", type=str, required=True, multiple=True, default=["web"])
91
+ @click.option(
92
+ "--template",
93
+ type=click.Choice(AVAILABLE_TEMPLATES),
94
+ default="default",
95
+ help="The maintenance page you wish to put up.",
96
+ )
97
+ @click.option("--no-maintenance-page", flag_value=True)
98
+ def copy(
99
+ app,
100
+ from_env,
101
+ to_env,
102
+ database,
103
+ from_vpc,
104
+ to_vpc,
105
+ auto_approve,
106
+ svc,
107
+ template,
108
+ no_maintenance_page,
109
+ ):
110
+ """Copy a database between environments."""
111
+ data_copy = DatabaseCopy(app, database, auto_approve)
112
+ data_copy.copy(from_env, to_env, from_vpc, to_vpc, svc, template, no_maintenance_page)
@@ -0,0 +1,218 @@
1
+ import boto3
2
+ import click
3
+ from schema import SchemaError
4
+
5
+ from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
6
+ from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
7
+ from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
8
+ from dbt_platform_helper.providers.load_balancers import find_https_listener
9
+ from dbt_platform_helper.utils.aws import get_aws_session_or_abort
10
+ from dbt_platform_helper.utils.click import ClickDocOptGroup
11
+ from dbt_platform_helper.utils.files import apply_environment_defaults
12
+ from dbt_platform_helper.utils.files import mkfile
13
+ from dbt_platform_helper.utils.platform_config import is_terraform_project
14
+ from dbt_platform_helper.utils.template import setup_templates
15
+ from dbt_platform_helper.utils.validation import load_and_validate_platform_config
16
+ from dbt_platform_helper.utils.versioning import (
17
+ check_platform_helper_version_needs_update,
18
+ )
19
+
20
+ AVAILABLE_TEMPLATES = ["default", "migration", "dmas-migration"]
21
+
22
+
23
+ @click.group(cls=ClickDocOptGroup)
24
+ def environment():
25
+ """Commands affecting environments."""
26
+ check_platform_helper_version_needs_update()
27
+
28
+
29
+ @environment.command()
30
+ @click.option("--app", type=str, required=True)
31
+ @click.option("--env", type=str, required=True)
32
+ @click.option("--svc", type=str, required=True, multiple=True, default=["web"])
33
+ @click.option(
34
+ "--template",
35
+ type=click.Choice(AVAILABLE_TEMPLATES),
36
+ default="default",
37
+ help="The maintenance page you wish to put up.",
38
+ )
39
+ @click.option("--vpc", type=str)
40
+ def offline(app, env, svc, template, vpc):
41
+ """Take load-balanced web services offline with a maintenance page."""
42
+ maintenance_page = MaintenancePageProvider()
43
+ maintenance_page.activate(app, env, svc, template, vpc)
44
+
45
+
46
+ @environment.command()
47
+ @click.option("--app", type=str, required=True)
48
+ @click.option("--env", type=str, required=True)
49
+ def online(app, env):
50
+ """Remove a maintenance page from an environment."""
51
+ maintenance_page = MaintenancePageProvider()
52
+ maintenance_page.deactivate(app, env)
53
+
54
+
55
+ def get_vpc_id(session, env_name, vpc_name=None):
56
+ if not vpc_name:
57
+ vpc_name = f"{session.profile_name}-{env_name}"
58
+
59
+ filters = [{"Name": "tag:Name", "Values": [vpc_name]}]
60
+ vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
61
+
62
+ if not vpcs:
63
+ filters[0]["Values"] = [session.profile_name]
64
+ vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
65
+
66
+ if not vpcs:
67
+ click.secho(
68
+ f"No VPC found with name {vpc_name} in AWS account {session.profile_name}.", fg="red"
69
+ )
70
+ raise click.Abort
71
+
72
+ return vpcs[0]["VpcId"]
73
+
74
+
75
+ def get_subnet_ids(session, vpc_id):
76
+ subnets = session.client("ec2").describe_subnets(
77
+ Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
78
+ )["Subnets"]
79
+
80
+ if not subnets:
81
+ click.secho(f"No subnets found for VPC with id: {vpc_id}.", fg="red")
82
+ raise click.Abort
83
+
84
+ public_tag = {"Key": "subnet_type", "Value": "public"}
85
+ public = [subnet["SubnetId"] for subnet in subnets if public_tag in subnet["Tags"]]
86
+ private_tag = {"Key": "subnet_type", "Value": "private"}
87
+ private = [subnet["SubnetId"] for subnet in subnets if private_tag in subnet["Tags"]]
88
+
89
+ return public, private
90
+
91
+
92
+ def get_cert_arn(session, application, env_name):
93
+ try:
94
+ arn = find_https_certificate(session, application, env_name)
95
+ except:
96
+ click.secho(
97
+ f"No certificate found with domain name matching environment {env_name}.", fg="red"
98
+ )
99
+ raise click.Abort
100
+
101
+ return arn
102
+
103
+
104
+ @environment.command()
105
+ @click.option("--vpc-name", hidden=True)
106
+ @click.option("--name", "-n", required=True)
107
+ def generate(name, vpc_name):
108
+ if vpc_name:
109
+ click.secho(
110
+ f"This option is deprecated. Please add the VPC name for your envs to {PLATFORM_CONFIG_FILE}",
111
+ fg="red",
112
+ )
113
+ raise click.Abort
114
+
115
+ try:
116
+ conf = load_and_validate_platform_config()
117
+ except SchemaError as ex:
118
+ click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
119
+ raise click.Abort
120
+
121
+ env_config = apply_environment_defaults(conf)["environments"][name]
122
+ profile_for_environment = env_config.get("accounts", {}).get("deploy", {}).get("name")
123
+ click.secho(f"Using {profile_for_environment} for this AWS session")
124
+ session = get_aws_session_or_abort(profile_for_environment)
125
+
126
+ _generate_copilot_environment_manifests(name, conf["application"], env_config, session)
127
+
128
+
129
+ @environment.command(help="Generate terraform manifest for the specified environment.")
130
+ @click.option(
131
+ "--name", "-n", required=True, help="The name of the environment to generate a manifest for."
132
+ )
133
+ @click.option(
134
+ "--terraform-platform-modules-version",
135
+ help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
136
+ )
137
+ def generate_terraform(name, terraform_platform_modules_version):
138
+ if not is_terraform_project():
139
+ click.secho("This is not a terraform project. Exiting.", fg="red")
140
+ exit(1)
141
+
142
+ conf = load_and_validate_platform_config()
143
+
144
+ env_config = apply_environment_defaults(conf)["environments"][name]
145
+ _generate_terraform_environment_manifests(
146
+ conf["application"], name, env_config, terraform_platform_modules_version
147
+ )
148
+
149
+
150
+ def _generate_copilot_environment_manifests(name, application, env_config, session):
151
+ env_template = setup_templates().get_template("env/manifest.yml")
152
+ vpc_name = env_config.get("vpc", None)
153
+ vpc_id = get_vpc_id(session, name, vpc_name)
154
+ pub_subnet_ids, priv_subnet_ids = get_subnet_ids(session, vpc_id)
155
+ cert_arn = get_cert_arn(session, application, name)
156
+ contents = env_template.render(
157
+ {
158
+ "name": name,
159
+ "vpc_id": vpc_id,
160
+ "pub_subnet_ids": pub_subnet_ids,
161
+ "priv_subnet_ids": priv_subnet_ids,
162
+ "certificate_arn": cert_arn,
163
+ }
164
+ )
165
+ click.echo(mkfile(".", f"copilot/environments/{name}/manifest.yml", contents, overwrite=True))
166
+
167
+
168
+ def _generate_terraform_environment_manifests(
169
+ application, env, env_config, cli_terraform_platform_modules_version
170
+ ):
171
+ env_template = setup_templates().get_template("environments/main.tf")
172
+
173
+ terraform_platform_modules_version = _determine_terraform_platform_modules_version(
174
+ env_config, cli_terraform_platform_modules_version
175
+ )
176
+
177
+ contents = env_template.render(
178
+ {
179
+ "application": application,
180
+ "environment": env,
181
+ "config": env_config,
182
+ "terraform_platform_modules_version": terraform_platform_modules_version,
183
+ }
184
+ )
185
+
186
+ click.echo(mkfile(".", f"terraform/environments/{env}/main.tf", contents, overwrite=True))
187
+
188
+
189
+ def _determine_terraform_platform_modules_version(env_conf, cli_terraform_platform_modules_version):
190
+ cli_terraform_platform_modules_version = cli_terraform_platform_modules_version
191
+ env_conf_terraform_platform_modules_version = env_conf.get("versions", {}).get(
192
+ "terraform-platform-modules"
193
+ )
194
+ version_preference_order = [
195
+ cli_terraform_platform_modules_version,
196
+ env_conf_terraform_platform_modules_version,
197
+ DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
198
+ ]
199
+ return [version for version in version_preference_order if version][0]
200
+
201
+
202
+ def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
203
+ listener_arn = find_https_listener(session, app, env)
204
+ cert_client = session.client("elbv2")
205
+ certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
206
+ "Certificates"
207
+ ]
208
+
209
+ try:
210
+ certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
211
+ except StopIteration:
212
+ raise CertificateNotFoundError()
213
+
214
+ return certificate_arn
215
+
216
+
217
+ class CertificateNotFoundError(Exception):
218
+ pass
@@ -5,6 +5,7 @@ from shutil import rmtree
5
5
 
6
6
  import click
7
7
 
8
+ from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
8
9
  from dbt_platform_helper.utils.application import get_application_name
9
10
  from dbt_platform_helper.utils.aws import get_account_details
10
11
  from dbt_platform_helper.utils.aws import get_codestar_connection_arn
@@ -24,6 +25,7 @@ from dbt_platform_helper.utils.versioning import (
24
25
 
25
26
  CODEBASE_PIPELINES_KEY = "codebase_pipelines"
26
27
  ENVIRONMENTS_KEY = "environments"
28
+ ENVIRONMENT_PIPELINES_KEY = "environment_pipelines"
27
29
 
28
30
 
29
31
  @click.group(chain=True, cls=ClickDocOptGroup)
@@ -33,18 +35,55 @@ def pipeline():
33
35
 
34
36
 
35
37
  @pipeline.command()
36
- def generate():
37
- """Given a platform-config.yml file, generate environment and service
38
- deployment pipelines."""
38
+ @click.option(
39
+ "--terraform-platform-modules-version",
40
+ help=f"""Override the default version of terraform-platform-modules with a specific version or branch.
41
+ Precedence of version used is version supplied via CLI, then the version found in
42
+ platform-config.yml/default_versions/terraform-platform-modules.
43
+ In absence of these inputs, defaults to version '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}'.""",
44
+ )
45
+ @click.option(
46
+ "--deploy-branch",
47
+ help="""Specify the branch of <application>-deploy used to configure the source stage in the environment-pipeline resource.
48
+ This is generated from the terraform/environments-pipeline/<aws_account>/main.tf file.
49
+ (Default <application>-deploy branch is specified in
50
+ <application>-deploy/platform-config.yml/environment_pipelines/<environment-pipeline>/branch).""",
51
+ default=None,
52
+ )
53
+ def generate(terraform_platform_modules_version, deploy_branch):
54
+ """
55
+ Given a platform-config.yml file, generate environment and service
56
+ deployment pipelines.
57
+
58
+ This command does the following in relation to the environment pipelines:
59
+ - Reads contents of `platform-config.yml/environment-pipelines` configuration.
60
+ The `terraform/environment-pipelines/<aws_account>/main.tf` file is generated using this configuration.
61
+ The `main.tf` file is then used to generate Terraform for creating an environment pipeline resource.
62
+
63
+ This command does the following in relation to the codebase pipelines:
64
+ - Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
65
+
66
+ (Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
67
+ - Generates the copilot manifest.yml for copilot/environments/<environment>
68
+ """
39
69
  pipeline_config = load_and_validate_platform_config()
40
70
 
41
- no_codebase_pipelines = CODEBASE_PIPELINES_KEY not in pipeline_config
42
- no_environment_pipelines = ENVIRONMENTS_KEY not in pipeline_config
71
+ has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
72
+ has_legacy_environment_pipelines = ENVIRONMENTS_KEY in pipeline_config
73
+ has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
43
74
 
44
- if no_codebase_pipelines and no_environment_pipelines:
75
+ if (
76
+ not has_codebase_pipelines
77
+ and not has_legacy_environment_pipelines
78
+ and not has_environment_pipelines
79
+ ):
45
80
  click.secho("No pipelines defined: nothing to do.", err=True, fg="yellow")
46
81
  return
47
82
 
83
+ platform_config_terraform_modules_default_version = pipeline_config.get(
84
+ "default_versions", {}
85
+ ).get("terraform-platform-modules", "")
86
+
48
87
  templates = setup_templates()
49
88
  app_name = get_application_name()
50
89
 
@@ -57,22 +96,34 @@ def generate():
57
96
  abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
58
97
 
59
98
  base_path = Path(".")
60
- pipelines_dir = base_path / f"copilot/pipelines"
99
+ copilot_pipelines_dir = base_path / f"copilot/pipelines"
61
100
 
62
- _clean_pipeline_config(pipelines_dir)
101
+ _clean_pipeline_config(copilot_pipelines_dir)
63
102
 
64
- if not is_terraform_project() and ENVIRONMENTS_KEY in pipeline_config:
103
+ if is_terraform_project() and has_environment_pipelines:
104
+ environment_pipelines = pipeline_config[ENVIRONMENT_PIPELINES_KEY]
105
+
106
+ for config in environment_pipelines.values():
107
+ aws_account = config.get("account")
108
+ _generate_terraform_environment_pipeline_manifest(
109
+ pipeline_config["application"],
110
+ aws_account,
111
+ terraform_platform_modules_version,
112
+ platform_config_terraform_modules_default_version,
113
+ deploy_branch,
114
+ )
115
+ if not is_terraform_project() and has_legacy_environment_pipelines:
65
116
  _generate_copilot_environments_pipeline(
66
117
  app_name,
67
118
  codestar_connection_arn,
68
119
  git_repo,
69
120
  apply_environment_defaults(pipeline_config)[ENVIRONMENTS_KEY],
70
121
  base_path,
71
- pipelines_dir,
122
+ copilot_pipelines_dir,
72
123
  templates,
73
124
  )
74
125
 
75
- if CODEBASE_PIPELINES_KEY in pipeline_config:
126
+ if has_codebase_pipelines:
76
127
  account_id, _ = get_account_details()
77
128
 
78
129
  for codebase in pipeline_config[CODEBASE_PIPELINES_KEY]:
@@ -83,7 +134,7 @@ def generate():
83
134
  git_repo,
84
135
  codebase,
85
136
  base_path,
86
- pipelines_dir,
137
+ copilot_pipelines_dir,
87
138
  templates,
88
139
  )
89
140
 
@@ -170,3 +221,43 @@ def _create_file_from_template(
170
221
  ).render(template_data)
171
222
  message = mkfile(base_path, pipelines_dir / file_name, contents, overwrite=True)
172
223
  click.echo(message)
224
+
225
+
226
+ def _generate_terraform_environment_pipeline_manifest(
227
+ application,
228
+ aws_account,
229
+ cli_terraform_platform_modules_version,
230
+ platform_config_terraform_modules_default_version,
231
+ deploy_branch,
232
+ ):
233
+ env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
234
+
235
+ terraform_platform_modules_version = _determine_terraform_platform_modules_version(
236
+ cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
237
+ )
238
+
239
+ contents = env_pipeline_template.render(
240
+ {
241
+ "application": application,
242
+ "aws_account": aws_account,
243
+ "terraform_platform_modules_version": terraform_platform_modules_version,
244
+ "deploy_branch": deploy_branch,
245
+ }
246
+ )
247
+
248
+ dir_path = f"terraform/environment-pipelines/{aws_account}"
249
+ makedirs(dir_path, exist_ok=True)
250
+
251
+ click.echo(mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
252
+
253
+
254
+ def _determine_terraform_platform_modules_version(
255
+ cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
256
+ ):
257
+
258
+ version_preference_order = [
259
+ cli_terraform_platform_modules_version,
260
+ platform_config_terraform_modules_default_version,
261
+ DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
262
+ ]
263
+ return [version for version in version_preference_order if version][0]