dbt-platform-helper 12.3.0__py3-none-any.whl → 12.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/COMMANDS.md +6 -1
- dbt_platform_helper/commands/codebase.py +1 -1
- dbt_platform_helper/commands/conduit.py +2 -2
- dbt_platform_helper/commands/config.py +4 -4
- dbt_platform_helper/commands/copilot.py +13 -15
- dbt_platform_helper/commands/database.py +17 -4
- dbt_platform_helper/commands/environment.py +3 -2
- dbt_platform_helper/commands/pipeline.py +1 -29
- dbt_platform_helper/constants.py +3 -1
- dbt_platform_helper/domain/codebase.py +23 -5
- dbt_platform_helper/domain/conduit.py +0 -6
- dbt_platform_helper/domain/database_copy.py +14 -13
- dbt_platform_helper/domain/maintenance_page.py +9 -9
- dbt_platform_helper/platform_exception.py +5 -0
- dbt_platform_helper/providers/aws.py +32 -0
- dbt_platform_helper/providers/cache.py +83 -0
- dbt_platform_helper/providers/cloudformation.py +8 -1
- dbt_platform_helper/providers/copilot.py +2 -5
- dbt_platform_helper/providers/ecs.py +19 -4
- dbt_platform_helper/providers/load_balancers.py +11 -5
- dbt_platform_helper/providers/platform_config_schema.py +605 -0
- dbt_platform_helper/providers/secrets.py +51 -10
- dbt_platform_helper/providers/validation.py +19 -0
- dbt_platform_helper/utils/application.py +14 -2
- dbt_platform_helper/utils/arn_parser.py +1 -1
- dbt_platform_helper/utils/aws.py +22 -21
- dbt_platform_helper/utils/files.py +0 -70
- dbt_platform_helper/utils/git.py +2 -2
- dbt_platform_helper/utils/validation.py +3 -551
- dbt_platform_helper/utils/versioning.py +8 -8
- {dbt_platform_helper-12.3.0.dist-info → dbt_platform_helper-12.4.1.dist-info}/METADATA +1 -1
- {dbt_platform_helper-12.3.0.dist-info → dbt_platform_helper-12.4.1.dist-info}/RECORD +35 -35
- dbt_platform_helper/addons-template-map.yml +0 -29
- dbt_platform_helper/exceptions.py +0 -147
- dbt_platform_helper/templates/pipelines/environments/buildspec.yml +0 -80
- dbt_platform_helper/templates/pipelines/environments/manifest.yml +0 -48
- dbt_platform_helper/templates/pipelines/environments/overrides/cfn.patches.yml +0 -21
- {dbt_platform_helper-12.3.0.dist-info → dbt_platform_helper-12.4.1.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-12.3.0.dist-info → dbt_platform_helper-12.4.1.dist-info}/WHEEL +0 -0
- {dbt_platform_helper-12.3.0.dist-info → dbt_platform_helper-12.4.1.dist-info}/entry_points.txt +0 -0
dbt_platform_helper/COMMANDS.md
CHANGED
|
@@ -765,6 +765,7 @@ platform-helper database (dump|load|copy)
|
|
|
765
765
|
```
|
|
766
766
|
platform-helper database dump --from <from_env> --database <database>
|
|
767
767
|
[--app <application>] [--from-vpc <from_vpc>]
|
|
768
|
+
[--filename <filename>]
|
|
768
769
|
```
|
|
769
770
|
|
|
770
771
|
## Options
|
|
@@ -777,6 +778,8 @@ platform-helper database dump --from <from_env> --database <database>
|
|
|
777
778
|
- The name of the database you are dumping data from
|
|
778
779
|
- `--from-vpc <text>`
|
|
779
780
|
- The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
|
|
781
|
+
- `--filename <text>`
|
|
782
|
+
- Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments
|
|
780
783
|
- `--help <boolean>` _Defaults to False._
|
|
781
784
|
- Show this message and exit.
|
|
782
785
|
|
|
@@ -791,7 +794,7 @@ platform-helper database dump --from <from_env> --database <database>
|
|
|
791
794
|
```
|
|
792
795
|
platform-helper database load --to <to_env> --database <database>
|
|
793
796
|
[--app <application>] [--to-vpc <to_vpc>]
|
|
794
|
-
[--auto-approve]
|
|
797
|
+
[--filename <filename>] [--auto-approve]
|
|
795
798
|
```
|
|
796
799
|
|
|
797
800
|
## Options
|
|
@@ -806,6 +809,8 @@ platform-helper database load --to <to_env> --database <database>
|
|
|
806
809
|
- The vpc the specified environment is running in. Required unless you are running the command from your deploy repo
|
|
807
810
|
- `--auto-approve <boolean>` _Defaults to False._
|
|
808
811
|
|
|
812
|
+
- `--filename <text>`
|
|
813
|
+
- Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments
|
|
809
814
|
- `--help <boolean>` _Defaults to False._
|
|
810
815
|
- Show this message and exit.
|
|
811
816
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
3
|
from dbt_platform_helper.domain.codebase import Codebase
|
|
4
|
-
from dbt_platform_helper.
|
|
4
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
5
5
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
6
6
|
from dbt_platform_helper.utils.versioning import (
|
|
7
7
|
check_platform_helper_version_needs_update,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
3
|
from dbt_platform_helper.domain.conduit import Conduit
|
|
4
|
-
from dbt_platform_helper.
|
|
4
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
5
5
|
from dbt_platform_helper.providers.cloudformation import CloudFormation
|
|
6
6
|
from dbt_platform_helper.providers.ecs import ECS
|
|
7
7
|
from dbt_platform_helper.providers.secrets import Secrets
|
|
@@ -53,6 +53,6 @@ def conduit(addon_name: str, app: str, env: str, access: str):
|
|
|
53
53
|
Conduit(application, secrets_provider, cloudformation_provider, ecs_provider).start(
|
|
54
54
|
env, addon_name, access
|
|
55
55
|
)
|
|
56
|
-
except
|
|
56
|
+
except PlatformException as err:
|
|
57
57
|
click.secho(str(err), fg="red")
|
|
58
58
|
raise click.Abort
|
|
@@ -8,8 +8,8 @@ import botocore
|
|
|
8
8
|
import click
|
|
9
9
|
from prettytable import PrettyTable
|
|
10
10
|
|
|
11
|
-
from dbt_platform_helper.
|
|
12
|
-
from dbt_platform_helper.
|
|
11
|
+
from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
|
|
12
|
+
from dbt_platform_helper.providers.validation import ValidationException
|
|
13
13
|
from dbt_platform_helper.utils import versioning
|
|
14
14
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
15
15
|
from dbt_platform_helper.utils.validation import config_file_check
|
|
@@ -110,7 +110,7 @@ def deployment():
|
|
|
110
110
|
str(template_file.resolve())
|
|
111
111
|
)
|
|
112
112
|
versioning.validate_template_version(local_version, str(template_file.resolve()))
|
|
113
|
-
except
|
|
113
|
+
except IncompatibleMajorVersionException:
|
|
114
114
|
local_compatible_symbol = no
|
|
115
115
|
compatible = False
|
|
116
116
|
recommendations["dbt-platform-helper-upgrade"] = RECOMMENDATIONS[
|
|
@@ -134,7 +134,7 @@ def deployment():
|
|
|
134
134
|
str(template_file.resolve())
|
|
135
135
|
)
|
|
136
136
|
versioning.validate_template_version(latest_release, str(template_file.resolve()))
|
|
137
|
-
except
|
|
137
|
+
except IncompatibleMajorVersionException:
|
|
138
138
|
latest_compatible_symbol = no
|
|
139
139
|
compatible = False
|
|
140
140
|
except ValidationException:
|
|
@@ -35,6 +35,14 @@ SERVICE_TYPES = [
|
|
|
35
35
|
"Worker Service",
|
|
36
36
|
]
|
|
37
37
|
|
|
38
|
+
ADDON_TEMPLATE_MAP = {
|
|
39
|
+
"s3": ["addons/svc/s3-policy.yml"],
|
|
40
|
+
"s3-policy": ["addons/svc/s3-policy.yml"],
|
|
41
|
+
"appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
|
|
42
|
+
"subscription-filter": ["addons/svc/subscription-filter.yml"],
|
|
43
|
+
"prometheus-policy": ["addons/svc/prometheus-policy.yml"],
|
|
44
|
+
}
|
|
45
|
+
|
|
38
46
|
|
|
39
47
|
def list_copilot_local_environments():
|
|
40
48
|
return [
|
|
@@ -248,9 +256,6 @@ def make_addons():
|
|
|
248
256
|
|
|
249
257
|
application_name = get_application_name()
|
|
250
258
|
|
|
251
|
-
with open(PACKAGE_DIR / "addons-template-map.yml") as fd:
|
|
252
|
-
addon_template_map = yaml.safe_load(fd)
|
|
253
|
-
|
|
254
259
|
click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
|
|
255
260
|
|
|
256
261
|
env_path = Path(f"copilot/environments/")
|
|
@@ -270,11 +275,6 @@ def make_addons():
|
|
|
270
275
|
print(f">>>>>>>>> {addon_name}")
|
|
271
276
|
addon_type = addon_config.pop("type")
|
|
272
277
|
environments = addon_config.pop("environments")
|
|
273
|
-
if addon_template_map[addon_type].get("requires_addons_parameters", False):
|
|
274
|
-
pass
|
|
275
|
-
if addon_type in ["postgres"]:
|
|
276
|
-
pass
|
|
277
|
-
|
|
278
278
|
environment_addon_config = {
|
|
279
279
|
"addon_type": addon_type,
|
|
280
280
|
"environments": environments,
|
|
@@ -309,7 +309,6 @@ def make_addons():
|
|
|
309
309
|
_generate_service_addons(
|
|
310
310
|
addon_config,
|
|
311
311
|
addon_name,
|
|
312
|
-
addon_template_map,
|
|
313
312
|
addon_type,
|
|
314
313
|
output_dir,
|
|
315
314
|
service_addon_config,
|
|
@@ -339,7 +338,6 @@ def _generate_env_overrides(output_dir):
|
|
|
339
338
|
def _generate_service_addons(
|
|
340
339
|
addon_config,
|
|
341
340
|
addon_name,
|
|
342
|
-
addon_template_map,
|
|
343
341
|
addon_type,
|
|
344
342
|
output_dir,
|
|
345
343
|
service_addon_config,
|
|
@@ -347,8 +345,8 @@ def _generate_service_addons(
|
|
|
347
345
|
log_destination_arns,
|
|
348
346
|
):
|
|
349
347
|
# generate svc addons
|
|
350
|
-
for
|
|
351
|
-
template = templates.get_template(
|
|
348
|
+
for addon_template in ADDON_TEMPLATE_MAP.get(addon_type, []):
|
|
349
|
+
template = templates.get_template(addon_template)
|
|
352
350
|
|
|
353
351
|
for svc in addon_config.get("services", []):
|
|
354
352
|
service_path = Path(f"copilot/{svc}/addons/")
|
|
@@ -360,10 +358,10 @@ def _generate_service_addons(
|
|
|
360
358
|
}
|
|
361
359
|
)
|
|
362
360
|
|
|
363
|
-
filename = addon.get("filename", f"{addon_name}.yml")
|
|
364
|
-
|
|
365
361
|
(output_dir / service_path).mkdir(parents=True, exist_ok=True)
|
|
366
|
-
click.echo(
|
|
362
|
+
click.echo(
|
|
363
|
+
mkfile(output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True)
|
|
364
|
+
)
|
|
367
365
|
|
|
368
366
|
|
|
369
367
|
def _cleanup_old_files(config, output_dir, env_addons_path, env_overrides_path):
|
|
@@ -31,10 +31,16 @@ def database():
|
|
|
31
31
|
type=str,
|
|
32
32
|
help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
|
|
33
33
|
)
|
|
34
|
-
|
|
34
|
+
@click.option(
|
|
35
|
+
"--filename",
|
|
36
|
+
type=str,
|
|
37
|
+
help="Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments",
|
|
38
|
+
)
|
|
39
|
+
def dump(app, from_env, database, from_vpc, filename):
|
|
35
40
|
"""Dump a database into an S3 bucket."""
|
|
36
41
|
data_copy = DatabaseCopy(app, database)
|
|
37
|
-
data_copy.dump(from_env, from_vpc)
|
|
42
|
+
data_copy.dump(from_env, from_vpc, filename)
|
|
43
|
+
# Todo: Catch expected errors and output message
|
|
38
44
|
|
|
39
45
|
|
|
40
46
|
@database.command(name="load")
|
|
@@ -55,10 +61,16 @@ def dump(app, from_env, database, from_vpc):
|
|
|
55
61
|
help="The vpc the specified environment is running in. Required unless you are running the command from your deploy repo",
|
|
56
62
|
)
|
|
57
63
|
@click.option("--auto-approve/--no-auto-approve", default=False)
|
|
58
|
-
|
|
64
|
+
@click.option(
|
|
65
|
+
"--filename",
|
|
66
|
+
type=str,
|
|
67
|
+
help="Specify a name for the database dump file. Recommended if the same dump database is being used for multiple load environments",
|
|
68
|
+
)
|
|
69
|
+
def load(app, to_env, database, to_vpc, auto_approve, filename):
|
|
59
70
|
"""Load a database from an S3 bucket."""
|
|
60
71
|
data_copy = DatabaseCopy(app, database, auto_approve)
|
|
61
|
-
data_copy.load(to_env, to_vpc)
|
|
72
|
+
data_copy.load(to_env, to_vpc, filename)
|
|
73
|
+
# Todo: Catch expected errors and output message
|
|
62
74
|
|
|
63
75
|
|
|
64
76
|
@database.command(name="copy")
|
|
@@ -110,3 +122,4 @@ def copy(
|
|
|
110
122
|
"""Copy a database between environments."""
|
|
111
123
|
data_copy = DatabaseCopy(app, database, auto_approve)
|
|
112
124
|
data_copy.copy(from_env, to_env, from_vpc, to_vpc, svc, template, no_maintenance_page)
|
|
125
|
+
# Todo: Catch expected errors and output message
|
|
@@ -5,6 +5,7 @@ from schema import SchemaError
|
|
|
5
5
|
from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
|
|
6
6
|
from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
|
|
7
7
|
from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
|
|
8
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
8
9
|
from dbt_platform_helper.providers.load_balancers import find_https_listener
|
|
9
10
|
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
10
11
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
@@ -238,10 +239,10 @@ def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
|
|
|
238
239
|
try:
|
|
239
240
|
certificate_arn = next(c["CertificateArn"] for c in certificates if c["IsDefault"])
|
|
240
241
|
except StopIteration:
|
|
241
|
-
raise
|
|
242
|
+
raise CertificateNotFoundException()
|
|
242
243
|
|
|
243
244
|
return certificate_arn
|
|
244
245
|
|
|
245
246
|
|
|
246
|
-
class
|
|
247
|
+
class CertificateNotFoundException(PlatformException):
|
|
247
248
|
pass
|
|
@@ -67,14 +67,9 @@ def generate(terraform_platform_modules_version, deploy_branch):
|
|
|
67
67
|
pipeline_config = load_and_validate_platform_config()
|
|
68
68
|
|
|
69
69
|
has_codebase_pipelines = CODEBASE_PIPELINES_KEY in pipeline_config
|
|
70
|
-
has_legacy_environment_pipelines = ENVIRONMENTS_KEY in pipeline_config
|
|
71
70
|
has_environment_pipelines = ENVIRONMENT_PIPELINES_KEY in pipeline_config
|
|
72
71
|
|
|
73
|
-
if (
|
|
74
|
-
not has_codebase_pipelines
|
|
75
|
-
and not has_legacy_environment_pipelines
|
|
76
|
-
and not has_environment_pipelines
|
|
77
|
-
):
|
|
72
|
+
if not (has_codebase_pipelines or has_environment_pipelines):
|
|
78
73
|
click.secho("No pipelines defined: nothing to do.", err=True, fg="yellow")
|
|
79
74
|
return
|
|
80
75
|
|
|
@@ -178,29 +173,6 @@ def _generate_codebase_pipeline(
|
|
|
178
173
|
)
|
|
179
174
|
|
|
180
175
|
|
|
181
|
-
def _generate_copilot_environments_pipeline(
|
|
182
|
-
app_name, codestar_connection_arn, git_repo, configuration, base_path, pipelines_dir, templates
|
|
183
|
-
):
|
|
184
|
-
makedirs(pipelines_dir / "environments/overrides", exist_ok=True)
|
|
185
|
-
|
|
186
|
-
template_data = {
|
|
187
|
-
"app_name": app_name,
|
|
188
|
-
"git_repo": git_repo,
|
|
189
|
-
"codestar_connection_arn": codestar_connection_arn,
|
|
190
|
-
"pipeline_environments": configuration,
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
_create_file_from_template(
|
|
194
|
-
base_path, "environments/buildspec.yml", pipelines_dir, template_data, templates
|
|
195
|
-
)
|
|
196
|
-
_create_file_from_template(
|
|
197
|
-
base_path, "environments/manifest.yml", pipelines_dir, template_data, templates
|
|
198
|
-
)
|
|
199
|
-
_create_file_from_template(
|
|
200
|
-
base_path, "environments/overrides/cfn.patches.yml", pipelines_dir, template_data, templates
|
|
201
|
-
)
|
|
202
|
-
|
|
203
|
-
|
|
204
176
|
def _create_file_from_template(
|
|
205
177
|
base_path, file_name, pipelines_dir, template_data, templates, template_name=None
|
|
206
178
|
):
|
dbt_platform_helper/constants.py
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
# Todo: Move to Config provider
|
|
1
2
|
PLATFORM_CONFIG_FILE = "platform-config.yml"
|
|
3
|
+
# Todo: Can we get rid of this yet?
|
|
2
4
|
PLATFORM_HELPER_VERSION_FILE = ".platform-helper-version"
|
|
5
|
+
# Todo: Move to ???
|
|
3
6
|
DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION = "5"
|
|
4
|
-
PLATFORM_HELPER_CACHE_FILE = ".platform-helper-config-cache.yml"
|
|
5
7
|
|
|
6
8
|
# Keys
|
|
7
9
|
CODEBASE_PIPELINES_KEY = "codebase_pipelines"
|
|
@@ -9,10 +9,9 @@ import requests
|
|
|
9
9
|
import yaml
|
|
10
10
|
from boto3 import Session
|
|
11
11
|
|
|
12
|
-
from dbt_platform_helper.
|
|
13
|
-
from dbt_platform_helper.exceptions import ApplicationEnvironmentNotFoundError
|
|
14
|
-
from dbt_platform_helper.exceptions import NotInCodeBaseRepositoryError
|
|
12
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
15
13
|
from dbt_platform_helper.utils.application import Application
|
|
14
|
+
from dbt_platform_helper.utils.application import ApplicationException
|
|
16
15
|
from dbt_platform_helper.utils.application import load_application
|
|
17
16
|
from dbt_platform_helper.utils.aws import check_codebase_exists
|
|
18
17
|
from dbt_platform_helper.utils.aws import check_image_exists
|
|
@@ -68,7 +67,7 @@ class Codebase:
|
|
|
68
67
|
.removesuffix(".git")
|
|
69
68
|
)
|
|
70
69
|
if repository.endswith("-deploy") or Path("./copilot").exists():
|
|
71
|
-
raise
|
|
70
|
+
raise NotInCodeBaseRepositoryException()
|
|
72
71
|
|
|
73
72
|
builder_configuration_url = "https://raw.githubusercontent.com/uktrade/ci-image-builder/main/image_builder/configuration/builder_configuration.yml"
|
|
74
73
|
builder_configuration_response = requests.get(builder_configuration_url)
|
|
@@ -143,7 +142,7 @@ class Codebase:
|
|
|
143
142
|
|
|
144
143
|
application = self.load_application(app, default_session=session)
|
|
145
144
|
if not application.environments.get(env):
|
|
146
|
-
raise
|
|
145
|
+
raise ApplicationEnvironmentNotFoundException(env)
|
|
147
146
|
|
|
148
147
|
self.check_codebase_exists(session, application, codebase)
|
|
149
148
|
|
|
@@ -220,3 +219,22 @@ class Codebase:
|
|
|
220
219
|
build_arn = self.start_build_extraction(codebuild_client, build_options)
|
|
221
220
|
return get_build_url_from_arn(build_arn)
|
|
222
221
|
return None
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class ApplicationDeploymentNotTriggered(PlatformException):
|
|
225
|
+
def __init__(self, codebase: str):
|
|
226
|
+
super().__init__(f"""Your deployment for {codebase} was not triggered.""")
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class ApplicationEnvironmentNotFoundException(ApplicationException):
|
|
230
|
+
def __init__(self, environment: str):
|
|
231
|
+
super().__init__(
|
|
232
|
+
f"""The environment "{environment}" either does not exist or has not been deployed."""
|
|
233
|
+
)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class NotInCodeBaseRepositoryException(PlatformException):
|
|
237
|
+
def __init__(self):
|
|
238
|
+
super().__init__(
|
|
239
|
+
"You are in the deploy repository; make sure you are in the application codebase repository.",
|
|
240
|
+
)
|
|
@@ -6,7 +6,6 @@ import click
|
|
|
6
6
|
from dbt_platform_helper.providers.cloudformation import CloudFormation
|
|
7
7
|
from dbt_platform_helper.providers.copilot import connect_to_addon_client_task
|
|
8
8
|
from dbt_platform_helper.providers.copilot import create_addon_client_task
|
|
9
|
-
from dbt_platform_helper.providers.copilot import create_postgres_admin_task
|
|
10
9
|
from dbt_platform_helper.providers.ecs import ECS
|
|
11
10
|
from dbt_platform_helper.providers.secrets import Secrets
|
|
12
11
|
from dbt_platform_helper.utils.application import Application
|
|
@@ -23,7 +22,6 @@ class Conduit:
|
|
|
23
22
|
subprocess: subprocess = subprocess,
|
|
24
23
|
connect_to_addon_client_task=connect_to_addon_client_task,
|
|
25
24
|
create_addon_client_task=create_addon_client_task,
|
|
26
|
-
create_postgres_admin_task=create_postgres_admin_task,
|
|
27
25
|
):
|
|
28
26
|
|
|
29
27
|
self.application = application
|
|
@@ -34,7 +32,6 @@ class Conduit:
|
|
|
34
32
|
self.echo = echo
|
|
35
33
|
self.connect_to_addon_client_task = connect_to_addon_client_task
|
|
36
34
|
self.create_addon_client_task = create_addon_client_task
|
|
37
|
-
self.create_postgres_admin_task = create_postgres_admin_task
|
|
38
35
|
|
|
39
36
|
def start(self, env: str, addon_name: str, access: str = "read"):
|
|
40
37
|
clients = self._initialise_clients(env)
|
|
@@ -49,7 +46,6 @@ class Conduit:
|
|
|
49
46
|
self.create_addon_client_task(
|
|
50
47
|
clients["iam"],
|
|
51
48
|
clients["ssm"],
|
|
52
|
-
clients["secrets_manager"],
|
|
53
49
|
self.subprocess,
|
|
54
50
|
self.application,
|
|
55
51
|
env,
|
|
@@ -89,8 +85,6 @@ class Conduit:
|
|
|
89
85
|
"ecs": self.application.environments[env].session.client("ecs"),
|
|
90
86
|
"iam": self.application.environments[env].session.client("iam"),
|
|
91
87
|
"ssm": self.application.environments[env].session.client("ssm"),
|
|
92
|
-
"cloudformation": self.application.environments[env].session.client("cloudformation"),
|
|
93
|
-
"secrets_manager": self.application.environments[env].session.client("secretsmanager"),
|
|
94
88
|
}
|
|
95
89
|
|
|
96
90
|
def _get_addon_details(self, addon_name, access):
|
|
@@ -8,9 +8,9 @@ from boto3 import Session
|
|
|
8
8
|
|
|
9
9
|
from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
|
|
10
10
|
from dbt_platform_helper.domain.maintenance_page import MaintenancePageProvider
|
|
11
|
-
from dbt_platform_helper.
|
|
12
|
-
from dbt_platform_helper.exceptions import AWSException
|
|
11
|
+
from dbt_platform_helper.providers.aws import AWSException
|
|
13
12
|
from dbt_platform_helper.utils.application import Application
|
|
13
|
+
from dbt_platform_helper.utils.application import ApplicationNotFoundException
|
|
14
14
|
from dbt_platform_helper.utils.application import load_application
|
|
15
15
|
from dbt_platform_helper.utils.aws import Vpc
|
|
16
16
|
from dbt_platform_helper.utils.aws import get_connection_string
|
|
@@ -58,10 +58,10 @@ class DatabaseCopy:
|
|
|
58
58
|
|
|
59
59
|
try:
|
|
60
60
|
self.application = load_application(self.app)
|
|
61
|
-
except
|
|
61
|
+
except ApplicationNotFoundException:
|
|
62
62
|
abort(f"No such application '{app}'.")
|
|
63
63
|
|
|
64
|
-
def _execute_operation(self, is_dump: bool, env: str, vpc_name: str,
|
|
64
|
+
def _execute_operation(self, is_dump: bool, env: str, vpc_name: str, filename: str):
|
|
65
65
|
vpc_name = self.enrich_vpc_name(env, vpc_name)
|
|
66
66
|
|
|
67
67
|
environments = self.application.environments
|
|
@@ -89,7 +89,7 @@ class DatabaseCopy:
|
|
|
89
89
|
|
|
90
90
|
try:
|
|
91
91
|
task_arn = self.run_database_copy_task(
|
|
92
|
-
env_session, env, vpc_config, is_dump, db_connection_string,
|
|
92
|
+
env_session, env, vpc_config, is_dump, db_connection_string, filename
|
|
93
93
|
)
|
|
94
94
|
except Exception as exc:
|
|
95
95
|
self.abort(f"{exc} (Account id: {self.account_id(env)})")
|
|
@@ -122,14 +122,15 @@ class DatabaseCopy:
|
|
|
122
122
|
vpc_config: Vpc,
|
|
123
123
|
is_dump: bool,
|
|
124
124
|
db_connection_string: str,
|
|
125
|
-
|
|
125
|
+
filename: str,
|
|
126
126
|
) -> str:
|
|
127
127
|
client = session.client("ecs")
|
|
128
128
|
action = "dump" if is_dump else "load"
|
|
129
|
+
dump_file_name = filename if filename else "data_dump"
|
|
129
130
|
env_vars = [
|
|
130
131
|
{"name": "DATA_COPY_OPERATION", "value": action.upper()},
|
|
131
132
|
{"name": "DB_CONNECTION_STRING", "value": db_connection_string},
|
|
132
|
-
{"name": "
|
|
133
|
+
{"name": "DUMP_FILE_NAME", "value": dump_file_name},
|
|
133
134
|
]
|
|
134
135
|
if not is_dump:
|
|
135
136
|
env_vars.append({"name": "ECS_CLUSTER", "value": f"{self.app}-{env}"})
|
|
@@ -159,12 +160,12 @@ class DatabaseCopy:
|
|
|
159
160
|
|
|
160
161
|
return response.get("tasks", [{}])[0].get("taskArn")
|
|
161
162
|
|
|
162
|
-
def dump(self, env: str, vpc_name: str,
|
|
163
|
-
self._execute_operation(True, env, vpc_name,
|
|
163
|
+
def dump(self, env: str, vpc_name: str, filename: str = None):
|
|
164
|
+
self._execute_operation(True, env, vpc_name, filename)
|
|
164
165
|
|
|
165
|
-
def load(self, env: str, vpc_name: str):
|
|
166
|
+
def load(self, env: str, vpc_name: str, filename: str = None):
|
|
166
167
|
if self.is_confirmed_ready_to_load(env):
|
|
167
|
-
self._execute_operation(False, env, vpc_name,
|
|
168
|
+
self._execute_operation(False, env, vpc_name, filename)
|
|
168
169
|
|
|
169
170
|
def copy(
|
|
170
171
|
self,
|
|
@@ -179,8 +180,8 @@ class DatabaseCopy:
|
|
|
179
180
|
to_vpc = self.enrich_vpc_name(to_env, to_vpc)
|
|
180
181
|
if not no_maintenance_page:
|
|
181
182
|
self.maintenance_page_provider.activate(self.app, to_env, services, template, to_vpc)
|
|
182
|
-
self.dump(from_env, from_vpc, to_env)
|
|
183
|
-
self.load(to_env, to_vpc)
|
|
183
|
+
self.dump(from_env, from_vpc, f"data_dump_{to_env}")
|
|
184
|
+
self.load(to_env, to_vpc, f"data_dump_{to_env}")
|
|
184
185
|
if not no_maintenance_page:
|
|
185
186
|
self.maintenance_page_provider.deactivate(self.app, to_env)
|
|
186
187
|
|
|
@@ -9,9 +9,9 @@ from typing import Union
|
|
|
9
9
|
import boto3
|
|
10
10
|
import click
|
|
11
11
|
|
|
12
|
-
from dbt_platform_helper.providers.load_balancers import
|
|
13
|
-
from dbt_platform_helper.providers.load_balancers import
|
|
14
|
-
from dbt_platform_helper.providers.load_balancers import
|
|
12
|
+
from dbt_platform_helper.providers.load_balancers import ListenerNotFoundException
|
|
13
|
+
from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundException
|
|
14
|
+
from dbt_platform_helper.providers.load_balancers import LoadBalancerNotFoundException
|
|
15
15
|
from dbt_platform_helper.providers.load_balancers import find_https_listener
|
|
16
16
|
from dbt_platform_helper.utils.application import Environment
|
|
17
17
|
from dbt_platform_helper.utils.application import Service
|
|
@@ -75,13 +75,13 @@ class MaintenancePageProvider:
|
|
|
75
75
|
else:
|
|
76
76
|
raise click.Abort
|
|
77
77
|
|
|
78
|
-
except
|
|
78
|
+
except LoadBalancerNotFoundException:
|
|
79
79
|
click.secho(
|
|
80
80
|
f"No load balancer found for environment {env} in the application {app}.", fg="red"
|
|
81
81
|
)
|
|
82
82
|
raise click.Abort
|
|
83
83
|
|
|
84
|
-
except
|
|
84
|
+
except ListenerNotFoundException:
|
|
85
85
|
click.secho(
|
|
86
86
|
f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
|
|
87
87
|
)
|
|
@@ -110,13 +110,13 @@ class MaintenancePageProvider:
|
|
|
110
110
|
f"Maintenance page removed from environment {env} in application {app}", fg="green"
|
|
111
111
|
)
|
|
112
112
|
|
|
113
|
-
except
|
|
113
|
+
except LoadBalancerNotFoundException:
|
|
114
114
|
click.secho(
|
|
115
115
|
f"No load balancer found for environment {env} in the application {app}.", fg="red"
|
|
116
116
|
)
|
|
117
117
|
raise click.Abort
|
|
118
118
|
|
|
119
|
-
except
|
|
119
|
+
except ListenerNotFoundException:
|
|
120
120
|
click.secho(
|
|
121
121
|
f"No HTTPS listener found for environment {env} in the application {app}.", fg="red"
|
|
122
122
|
)
|
|
@@ -180,7 +180,7 @@ def remove_maintenance_page(session: boto3.Session, listener_arn: str):
|
|
|
180
180
|
deleted = delete_listener_rule(tag_descriptions, name, lb_client)
|
|
181
181
|
|
|
182
182
|
if name == "MaintenancePage" and not deleted:
|
|
183
|
-
raise
|
|
183
|
+
raise ListenerRuleNotFoundException()
|
|
184
184
|
|
|
185
185
|
|
|
186
186
|
def get_rules_tag_descriptions(rules: list, lb_client):
|
|
@@ -261,7 +261,7 @@ def add_maintenance_page(
|
|
|
261
261
|
)
|
|
262
262
|
|
|
263
263
|
click.secho(
|
|
264
|
-
f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/
|
|
264
|
+
f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
|
|
265
265
|
fg="green",
|
|
266
266
|
)
|
|
267
267
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
from dbt_platform_helper.platform_exception import PlatformException
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class AWSException(PlatformException):
|
|
5
|
+
pass
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CreateTaskTimeoutException(AWSException):
|
|
9
|
+
def __init__(self, addon_name: str, application_name: str, environment: str):
|
|
10
|
+
super().__init__(
|
|
11
|
+
f"""Client ({addon_name}) ECS task has failed to start for "{application_name}" in "{environment}" environment."""
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ImageNotFoundException(AWSException):
|
|
16
|
+
def __init__(self, commit: str):
|
|
17
|
+
super().__init__(
|
|
18
|
+
f"""The commit hash "{commit}" has not been built into an image, try the `platform-helper codebase build` command first."""
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class LogGroupNotFoundException(AWSException):
|
|
23
|
+
def __init__(self, log_group_name: str):
|
|
24
|
+
super().__init__(f"""No log group called "{log_group_name}".""")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
# Todo: This should probably be in the AWS Copilot provider, but was causing circular import when we tried it pre refactoring the utils/aws.py
|
|
28
|
+
class CopilotCodebaseNotFoundException(PlatformException):
|
|
29
|
+
def __init__(self, codebase: str):
|
|
30
|
+
super().__init__(
|
|
31
|
+
f"""The codebase "{codebase}" either does not exist or has not been deployed."""
|
|
32
|
+
)
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
import yaml
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class CacheProvider:
|
|
9
|
+
def __init__(self):
|
|
10
|
+
self._cache_file = ".platform-helper-config-cache.yml"
|
|
11
|
+
|
|
12
|
+
def read_supported_versions_from_cache(self, resource_name):
|
|
13
|
+
|
|
14
|
+
platform_helper_config = self.__read_file_as_yaml(self._cache_file)
|
|
15
|
+
|
|
16
|
+
return platform_helper_config.get(resource_name).get("versions")
|
|
17
|
+
|
|
18
|
+
def update_cache(self, resource_name, supported_versions):
|
|
19
|
+
|
|
20
|
+
platform_helper_config = {}
|
|
21
|
+
|
|
22
|
+
if self.__cache_exists():
|
|
23
|
+
platform_helper_config = self.__read_file_as_yaml(self._cache_file)
|
|
24
|
+
|
|
25
|
+
cache_dict = {
|
|
26
|
+
resource_name: {
|
|
27
|
+
"versions": supported_versions,
|
|
28
|
+
"date-retrieved": datetime.now().strftime("%d-%m-%y %H:%M:%S"),
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
platform_helper_config.update(cache_dict)
|
|
33
|
+
|
|
34
|
+
self.__write_cache(platform_helper_config)
|
|
35
|
+
|
|
36
|
+
def cache_refresh_required(self, resource_name) -> bool:
|
|
37
|
+
"""
|
|
38
|
+
Checks if the platform-helper should reach out to AWS to 'refresh' its
|
|
39
|
+
cached values.
|
|
40
|
+
|
|
41
|
+
An API call is needed if any of the following conditions are met:
|
|
42
|
+
1. No cache file (.platform-helper-config.yml) exists.
|
|
43
|
+
2. The resource name (e.g. redis, opensearch) does not exist within the cache file.
|
|
44
|
+
3. The date-retrieved value of the cached data is > than a time interval. In this case 1 day.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
if not self.__cache_exists():
|
|
48
|
+
return True
|
|
49
|
+
|
|
50
|
+
platform_helper_config = self.__read_file_as_yaml(self._cache_file)
|
|
51
|
+
|
|
52
|
+
if platform_helper_config.get(resource_name):
|
|
53
|
+
return self.__check_if_cached_datetime_is_greater_than_interval(
|
|
54
|
+
platform_helper_config[resource_name].get("date-retrieved"), 1
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
return True
|
|
58
|
+
|
|
59
|
+
@staticmethod
|
|
60
|
+
def __check_if_cached_datetime_is_greater_than_interval(date_retrieved, interval_in_days):
|
|
61
|
+
|
|
62
|
+
current_datetime = datetime.now()
|
|
63
|
+
cached_datetime = datetime.strptime(date_retrieved, "%d-%m-%y %H:%M:%S")
|
|
64
|
+
delta = current_datetime - cached_datetime
|
|
65
|
+
|
|
66
|
+
return delta.days > interval_in_days
|
|
67
|
+
|
|
68
|
+
# TODO - same applies here as below
|
|
69
|
+
@staticmethod
|
|
70
|
+
def __read_file_as_yaml(file_name):
|
|
71
|
+
|
|
72
|
+
return yaml.safe_load(Path(file_name).read_text())
|
|
73
|
+
|
|
74
|
+
# TODO - temp fix to the unit test coverage issue, plan is to seperate out any yaml interaction methods into a seperate 'yaml' provider
|
|
75
|
+
# should be done under a different sub-task which will need to loop back to this provider as part of that work to use the yaml provider instead
|
|
76
|
+
def __write_cache(self, contents):
|
|
77
|
+
|
|
78
|
+
with open(self._cache_file, "w") as file:
|
|
79
|
+
file.write("# [!] This file is autogenerated via the platform-helper. Do not edit.\n")
|
|
80
|
+
yaml.dump(contents, file)
|
|
81
|
+
|
|
82
|
+
def __cache_exists(self):
|
|
83
|
+
return os.path.exists(self._cache_file)
|