dbt-platform-helper 12.4.1__py3-none-any.whl → 12.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dbt-platform-helper might be problematic. Click here for more details.
- dbt_platform_helper/COMMANDS.md +1 -6
- dbt_platform_helper/commands/config.py +2 -2
- dbt_platform_helper/commands/copilot.py +51 -30
- dbt_platform_helper/commands/environment.py +25 -185
- dbt_platform_helper/commands/pipeline.py +10 -173
- dbt_platform_helper/constants.py +10 -0
- dbt_platform_helper/domain/codebase.py +8 -4
- dbt_platform_helper/domain/config_validator.py +242 -0
- dbt_platform_helper/domain/copilot_environment.py +204 -0
- dbt_platform_helper/domain/database_copy.py +16 -12
- dbt_platform_helper/domain/maintenance_page.py +44 -20
- dbt_platform_helper/domain/pipelines.py +213 -0
- dbt_platform_helper/domain/terraform_environment.py +86 -0
- dbt_platform_helper/domain/test_platform_terraform_manifest_generator.py +100 -0
- dbt_platform_helper/jinja2_tags.py +1 -1
- dbt_platform_helper/providers/cache.py +14 -21
- dbt_platform_helper/providers/cloudformation.py +0 -1
- dbt_platform_helper/providers/config.py +100 -0
- dbt_platform_helper/providers/copilot.py +2 -0
- dbt_platform_helper/providers/files.py +26 -0
- dbt_platform_helper/providers/opensearch.py +36 -0
- dbt_platform_helper/providers/platform_config_schema.py +589 -527
- dbt_platform_helper/providers/redis.py +34 -0
- dbt_platform_helper/providers/vpc.py +57 -0
- dbt_platform_helper/providers/yaml_file.py +72 -0
- dbt_platform_helper/templates/addons/svc/s3-cross-account-policy.yml +67 -0
- dbt_platform_helper/utils/application.py +32 -34
- dbt_platform_helper/utils/aws.py +1 -107
- dbt_platform_helper/utils/files.py +8 -59
- dbt_platform_helper/utils/platform_config.py +0 -7
- dbt_platform_helper/utils/template.py +10 -0
- dbt_platform_helper/utils/validation.py +5 -327
- dbt_platform_helper/utils/versioning.py +12 -0
- {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/METADATA +2 -2
- {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/RECORD +38 -26
- {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/WHEEL +1 -1
- {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/LICENSE +0 -0
- {dbt_platform_helper-12.4.1.dist-info → dbt_platform_helper-12.5.1.dist-info}/entry_points.txt +0 -0
dbt_platform_helper/COMMANDS.md
CHANGED
|
@@ -460,13 +460,11 @@ platform-helper environment online --app <application> --env <environment>
|
|
|
460
460
|
## Usage
|
|
461
461
|
|
|
462
462
|
```
|
|
463
|
-
platform-helper environment generate --name <name>
|
|
463
|
+
platform-helper environment generate --name <name>
|
|
464
464
|
```
|
|
465
465
|
|
|
466
466
|
## Options
|
|
467
467
|
|
|
468
|
-
- `--vpc-name <text>`
|
|
469
|
-
|
|
470
468
|
- `--name
|
|
471
469
|
-n <text>`
|
|
472
470
|
|
|
@@ -551,9 +549,6 @@ platform-helper pipeline generate
|
|
|
551
549
|
This command does the following in relation to the codebase pipelines:
|
|
552
550
|
- Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
|
|
553
551
|
|
|
554
|
-
(Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
|
|
555
|
-
- Generates the copilot manifest.yml for copilot/environments/<environment>
|
|
556
|
-
|
|
557
552
|
## Usage
|
|
558
553
|
|
|
559
554
|
```
|
|
@@ -8,11 +8,11 @@ import botocore
|
|
|
8
8
|
import click
|
|
9
9
|
from prettytable import PrettyTable
|
|
10
10
|
|
|
11
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
11
12
|
from dbt_platform_helper.providers.validation import IncompatibleMajorVersionException
|
|
12
13
|
from dbt_platform_helper.providers.validation import ValidationException
|
|
13
14
|
from dbt_platform_helper.utils import versioning
|
|
14
15
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
15
|
-
from dbt_platform_helper.utils.validation import config_file_check
|
|
16
16
|
from dbt_platform_helper.utils.versioning import get_platform_helper_versions
|
|
17
17
|
|
|
18
18
|
yes = "\033[92m✔\033[0m"
|
|
@@ -98,7 +98,7 @@ def deployment():
|
|
|
98
98
|
|
|
99
99
|
recommendations = {}
|
|
100
100
|
|
|
101
|
-
config_file_check()
|
|
101
|
+
ConfigProvider.config_file_check()
|
|
102
102
|
|
|
103
103
|
for template_file in addons_templates:
|
|
104
104
|
generated_with_version = maybe
|
|
@@ -7,17 +7,22 @@ from pathlib import PosixPath
|
|
|
7
7
|
|
|
8
8
|
import click
|
|
9
9
|
import yaml
|
|
10
|
+
from schema import SchemaError
|
|
10
11
|
|
|
11
12
|
from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
|
|
13
|
+
from dbt_platform_helper.domain.config_validator import ConfigValidator
|
|
14
|
+
from dbt_platform_helper.domain.copilot_environment import CopilotTemplating
|
|
15
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
16
|
+
from dbt_platform_helper.providers.files import FileProvider
|
|
12
17
|
from dbt_platform_helper.utils.application import get_application_name
|
|
13
18
|
from dbt_platform_helper.utils.application import load_application
|
|
14
19
|
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
15
20
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
16
21
|
from dbt_platform_helper.utils.files import generate_override_files
|
|
17
|
-
from dbt_platform_helper.utils.
|
|
22
|
+
from dbt_platform_helper.utils.messages import abort_with_error
|
|
23
|
+
from dbt_platform_helper.utils.template import ADDON_TEMPLATE_MAP
|
|
18
24
|
from dbt_platform_helper.utils.template import camel_case
|
|
19
25
|
from dbt_platform_helper.utils.template import setup_templates
|
|
20
|
-
from dbt_platform_helper.utils.validation import config_file_check
|
|
21
26
|
from dbt_platform_helper.utils.validation import validate_addons
|
|
22
27
|
from dbt_platform_helper.utils.versioning import (
|
|
23
28
|
check_platform_helper_version_needs_update,
|
|
@@ -35,14 +40,6 @@ SERVICE_TYPES = [
|
|
|
35
40
|
"Worker Service",
|
|
36
41
|
]
|
|
37
42
|
|
|
38
|
-
ADDON_TEMPLATE_MAP = {
|
|
39
|
-
"s3": ["addons/svc/s3-policy.yml"],
|
|
40
|
-
"s3-policy": ["addons/svc/s3-policy.yml"],
|
|
41
|
-
"appconfig-ipfilter": ["addons/svc/appconfig-ipfilter.yml"],
|
|
42
|
-
"subscription-filter": ["addons/svc/subscription-filter.yml"],
|
|
43
|
-
"prometheus-policy": ["addons/svc/prometheus-policy.yml"],
|
|
44
|
-
}
|
|
45
|
-
|
|
46
43
|
|
|
47
44
|
def list_copilot_local_environments():
|
|
48
45
|
return [
|
|
@@ -244,25 +241,42 @@ def _get_s3_kms_alias_arns(session, application_name, config):
|
|
|
244
241
|
return arns
|
|
245
242
|
|
|
246
243
|
|
|
244
|
+
def copilot_provider():
|
|
245
|
+
return CopilotTemplating()
|
|
246
|
+
|
|
247
|
+
|
|
247
248
|
@copilot.command()
|
|
248
249
|
def make_addons():
|
|
249
250
|
"""Generate addons CloudFormation for each environment."""
|
|
250
|
-
|
|
251
|
-
|
|
251
|
+
try:
|
|
252
|
+
config_provider = ConfigProvider(ConfigValidator())
|
|
253
|
+
_make_addons(config_provider)
|
|
254
|
+
except Exception as exc:
|
|
255
|
+
abort_with_error(exc)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _make_addons(config_provider: ConfigProvider):
|
|
259
|
+
config_provider.config_file_check()
|
|
260
|
+
try:
|
|
261
|
+
config = config_provider.load_and_validate_platform_config()
|
|
262
|
+
except SchemaError as ex:
|
|
263
|
+
click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
|
|
264
|
+
raise click.Abort
|
|
252
265
|
|
|
253
266
|
templates = setup_templates()
|
|
254
|
-
|
|
267
|
+
extensions = _get_extensions()
|
|
255
268
|
session = get_aws_session_or_abort()
|
|
256
269
|
|
|
257
270
|
application_name = get_application_name()
|
|
258
271
|
|
|
259
272
|
click.echo("\n>>> Generating Terraform compatible addons CloudFormation\n")
|
|
260
273
|
|
|
274
|
+
output_dir = Path(".").absolute()
|
|
261
275
|
env_path = Path(f"copilot/environments/")
|
|
262
276
|
env_addons_path = env_path / "addons"
|
|
263
277
|
env_overrides_path = env_path / "overrides"
|
|
264
278
|
|
|
265
|
-
_cleanup_old_files(
|
|
279
|
+
_cleanup_old_files(extensions, output_dir, env_addons_path, env_overrides_path)
|
|
266
280
|
_generate_env_overrides(output_dir)
|
|
267
281
|
|
|
268
282
|
svc_names = list_copilot_local_services()
|
|
@@ -271,33 +285,33 @@ def make_addons():
|
|
|
271
285
|
_generate_svc_overrides(base_path, templates, svc_name)
|
|
272
286
|
|
|
273
287
|
services = []
|
|
274
|
-
for
|
|
275
|
-
|
|
276
|
-
addon_type =
|
|
277
|
-
environments =
|
|
288
|
+
for ext_name, ext_data in extensions.items():
|
|
289
|
+
extension = {**ext_data}
|
|
290
|
+
addon_type = extension.pop("type")
|
|
291
|
+
environments = extension.pop("environments")
|
|
278
292
|
environment_addon_config = {
|
|
279
293
|
"addon_type": addon_type,
|
|
280
294
|
"environments": environments,
|
|
281
|
-
"name":
|
|
282
|
-
"prefix": camel_case(
|
|
283
|
-
"secret_name":
|
|
284
|
-
**
|
|
295
|
+
"name": extension.get("name", None) or ext_name,
|
|
296
|
+
"prefix": camel_case(ext_name),
|
|
297
|
+
"secret_name": ext_name.upper().replace("-", "_"),
|
|
298
|
+
**extension,
|
|
285
299
|
}
|
|
286
300
|
|
|
287
301
|
services.append(environment_addon_config)
|
|
288
302
|
|
|
289
303
|
service_addon_config = {
|
|
290
304
|
"application_name": application_name,
|
|
291
|
-
"name":
|
|
292
|
-
"prefix": camel_case(
|
|
305
|
+
"name": extension.get("name", None) or ext_name,
|
|
306
|
+
"prefix": camel_case(ext_name),
|
|
293
307
|
"environments": environments,
|
|
294
|
-
**
|
|
308
|
+
**extension,
|
|
295
309
|
}
|
|
296
310
|
|
|
297
311
|
log_destination_arns = get_log_destination_arn()
|
|
298
312
|
|
|
299
313
|
if addon_type in ["s3", "s3-policy"]:
|
|
300
|
-
if
|
|
314
|
+
if extensions[ext_name].get("serve_static_content"):
|
|
301
315
|
continue
|
|
302
316
|
|
|
303
317
|
s3_kms_arns = _get_s3_kms_alias_arns(session, application_name, environments)
|
|
@@ -307,8 +321,8 @@ def make_addons():
|
|
|
307
321
|
)
|
|
308
322
|
|
|
309
323
|
_generate_service_addons(
|
|
310
|
-
|
|
311
|
-
|
|
324
|
+
extension,
|
|
325
|
+
ext_name,
|
|
312
326
|
addon_type,
|
|
313
327
|
output_dir,
|
|
314
328
|
service_addon_config,
|
|
@@ -316,10 +330,15 @@ def make_addons():
|
|
|
316
330
|
log_destination_arns,
|
|
317
331
|
)
|
|
318
332
|
|
|
333
|
+
environments = config_provider.apply_environment_defaults(config)["environments"]
|
|
334
|
+
|
|
335
|
+
provider = copilot_provider()
|
|
336
|
+
provider.generate_cross_account_s3_policies(environments, extensions)
|
|
337
|
+
|
|
319
338
|
click.echo(templates.get_template("addon-instructions.txt").render(services=services))
|
|
320
339
|
|
|
321
340
|
|
|
322
|
-
def
|
|
341
|
+
def _get_extensions():
|
|
323
342
|
config = _validate_and_normalise_extensions_config(PACKAGE_DIR / "default-extensions.yml")
|
|
324
343
|
project_config = _validate_and_normalise_extensions_config(PLATFORM_CONFIG_FILE, "extensions")
|
|
325
344
|
config.update(project_config)
|
|
@@ -360,7 +379,9 @@ def _generate_service_addons(
|
|
|
360
379
|
|
|
361
380
|
(output_dir / service_path).mkdir(parents=True, exist_ok=True)
|
|
362
381
|
click.echo(
|
|
363
|
-
mkfile(
|
|
382
|
+
FileProvider.mkfile(
|
|
383
|
+
output_dir, service_path / f"{addon_name}.yml", contents, overwrite=True
|
|
384
|
+
)
|
|
364
385
|
)
|
|
365
386
|
|
|
366
387
|
|
|
@@ -1,18 +1,15 @@
|
|
|
1
|
-
import boto3
|
|
2
1
|
import click
|
|
3
2
|
from schema import SchemaError
|
|
4
3
|
|
|
5
4
|
from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
|
|
6
5
|
from dbt_platform_helper.constants import PLATFORM_CONFIG_FILE
|
|
7
|
-
from dbt_platform_helper.domain.
|
|
6
|
+
from dbt_platform_helper.domain.config_validator import ConfigValidator
|
|
7
|
+
from dbt_platform_helper.domain.copilot_environment import CopilotEnvironment
|
|
8
|
+
from dbt_platform_helper.domain.maintenance_page import MaintenancePage
|
|
9
|
+
from dbt_platform_helper.domain.terraform_environment import TerraformEnvironment
|
|
8
10
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
9
|
-
from dbt_platform_helper.providers.
|
|
10
|
-
from dbt_platform_helper.utils.aws import get_aws_session_or_abort
|
|
11
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
11
12
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
12
|
-
from dbt_platform_helper.utils.files import apply_environment_defaults
|
|
13
|
-
from dbt_platform_helper.utils.files import mkfile
|
|
14
|
-
from dbt_platform_helper.utils.template import setup_templates
|
|
15
|
-
from dbt_platform_helper.utils.validation import load_and_validate_platform_config
|
|
16
13
|
from dbt_platform_helper.utils.versioning import (
|
|
17
14
|
check_platform_helper_version_needs_update,
|
|
18
15
|
)
|
|
@@ -39,8 +36,11 @@ def environment():
|
|
|
39
36
|
@click.option("--vpc", type=str)
|
|
40
37
|
def offline(app, env, svc, template, vpc):
|
|
41
38
|
"""Take load-balanced web services offline with a maintenance page."""
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
try:
|
|
40
|
+
MaintenancePage().activate(app, env, svc, template, vpc)
|
|
41
|
+
except PlatformException as err:
|
|
42
|
+
click.secho(str(err), fg="red")
|
|
43
|
+
raise click.Abort
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
@environment.command()
|
|
@@ -48,112 +48,26 @@ def offline(app, env, svc, template, vpc):
|
|
|
48
48
|
@click.option("--env", type=str, required=True)
|
|
49
49
|
def online(app, env):
|
|
50
50
|
"""Remove a maintenance page from an environment."""
|
|
51
|
-
maintenance_page = MaintenancePageProvider()
|
|
52
|
-
maintenance_page.deactivate(app, env)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
def get_vpc_id(session, env_name, vpc_name=None):
|
|
56
|
-
if not vpc_name:
|
|
57
|
-
vpc_name = f"{session.profile_name}-{env_name}"
|
|
58
|
-
|
|
59
|
-
filters = [{"Name": "tag:Name", "Values": [vpc_name]}]
|
|
60
|
-
vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
|
|
61
|
-
|
|
62
|
-
if not vpcs:
|
|
63
|
-
filters[0]["Values"] = [session.profile_name]
|
|
64
|
-
vpcs = session.client("ec2").describe_vpcs(Filters=filters)["Vpcs"]
|
|
65
|
-
|
|
66
|
-
if not vpcs:
|
|
67
|
-
click.secho(
|
|
68
|
-
f"No VPC found with name {vpc_name} in AWS account {session.profile_name}.", fg="red"
|
|
69
|
-
)
|
|
70
|
-
raise click.Abort
|
|
71
|
-
|
|
72
|
-
return vpcs[0]["VpcId"]
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
def get_subnet_ids(session, vpc_id, environment_name):
|
|
76
|
-
subnets = session.client("ec2").describe_subnets(
|
|
77
|
-
Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
|
|
78
|
-
)["Subnets"]
|
|
79
|
-
|
|
80
|
-
if not subnets:
|
|
81
|
-
click.secho(f"No subnets found for VPC with id: {vpc_id}.", fg="red")
|
|
82
|
-
raise click.Abort
|
|
83
|
-
|
|
84
|
-
public_tag = {"Key": "subnet_type", "Value": "public"}
|
|
85
|
-
public_subnets = [subnet["SubnetId"] for subnet in subnets if public_tag in subnet["Tags"]]
|
|
86
|
-
private_tag = {"Key": "subnet_type", "Value": "private"}
|
|
87
|
-
private_subnets = [subnet["SubnetId"] for subnet in subnets if private_tag in subnet["Tags"]]
|
|
88
|
-
|
|
89
|
-
# This call and the method declaration can be removed when we stop using AWS Copilot to deploy the services
|
|
90
|
-
public_subnets, private_subnets = _match_subnet_id_order_to_cloudformation_exports(
|
|
91
|
-
session,
|
|
92
|
-
environment_name,
|
|
93
|
-
public_subnets,
|
|
94
|
-
private_subnets,
|
|
95
|
-
)
|
|
96
|
-
|
|
97
|
-
return public_subnets, private_subnets
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
def _match_subnet_id_order_to_cloudformation_exports(
|
|
101
|
-
session, environment_name, public_subnets, private_subnets
|
|
102
|
-
):
|
|
103
|
-
public_subnet_exports = []
|
|
104
|
-
private_subnet_exports = []
|
|
105
|
-
for page in session.client("cloudformation").get_paginator("list_exports").paginate():
|
|
106
|
-
for export in page["Exports"]:
|
|
107
|
-
if f"-{environment_name}-" in export["Name"]:
|
|
108
|
-
if export["Name"].endswith("-PublicSubnets"):
|
|
109
|
-
public_subnet_exports = export["Value"].split(",")
|
|
110
|
-
if export["Name"].endswith("-PrivateSubnets"):
|
|
111
|
-
private_subnet_exports = export["Value"].split(",")
|
|
112
|
-
|
|
113
|
-
# If the elements match, regardless of order, use the list from the CloudFormation exports
|
|
114
|
-
if set(public_subnets) == set(public_subnet_exports):
|
|
115
|
-
public_subnets = public_subnet_exports
|
|
116
|
-
if set(private_subnets) == set(private_subnet_exports):
|
|
117
|
-
private_subnets = private_subnet_exports
|
|
118
|
-
|
|
119
|
-
return public_subnets, private_subnets
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
def get_cert_arn(session, application, env_name):
|
|
123
51
|
try:
|
|
124
|
-
|
|
125
|
-
except:
|
|
126
|
-
click.secho(
|
|
127
|
-
f"No certificate found with domain name matching environment {env_name}.", fg="red"
|
|
128
|
-
)
|
|
52
|
+
MaintenancePage().deactivate(app, env)
|
|
53
|
+
except PlatformException as err:
|
|
54
|
+
click.secho(str(err), fg="red")
|
|
129
55
|
raise click.Abort
|
|
130
56
|
|
|
131
|
-
return arn
|
|
132
|
-
|
|
133
57
|
|
|
134
58
|
@environment.command()
|
|
135
|
-
@click.option("--vpc-name", hidden=True)
|
|
136
59
|
@click.option("--name", "-n", required=True)
|
|
137
|
-
def generate(name
|
|
138
|
-
if vpc_name:
|
|
139
|
-
click.secho(
|
|
140
|
-
f"This option is deprecated. Please add the VPC name for your envs to {PLATFORM_CONFIG_FILE}",
|
|
141
|
-
fg="red",
|
|
142
|
-
)
|
|
143
|
-
raise click.Abort
|
|
144
|
-
|
|
60
|
+
def generate(name):
|
|
145
61
|
try:
|
|
146
|
-
|
|
62
|
+
config_provider = ConfigProvider(ConfigValidator())
|
|
63
|
+
CopilotEnvironment(config_provider).generate(name)
|
|
64
|
+
# TODO this exception will never be caught as the config provider catches schema errors and aborts
|
|
147
65
|
except SchemaError as ex:
|
|
148
66
|
click.secho(f"Invalid `{PLATFORM_CONFIG_FILE}` file: {str(ex)}", fg="red")
|
|
149
67
|
raise click.Abort
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
click.secho(f"Using {profile_for_environment} for this AWS session")
|
|
154
|
-
session = get_aws_session_or_abort(profile_for_environment)
|
|
155
|
-
|
|
156
|
-
_generate_copilot_environment_manifests(name, conf["application"], env_config, session)
|
|
68
|
+
except PlatformException as err:
|
|
69
|
+
click.secho(str(err), fg="red")
|
|
70
|
+
raise click.Abort
|
|
157
71
|
|
|
158
72
|
|
|
159
73
|
@environment.command(help="Generate terraform manifest for the specified environment.")
|
|
@@ -165,84 +79,10 @@ def generate(name, vpc_name):
|
|
|
165
79
|
help=f"Override the default version of terraform-platform-modules. (Default version is '{DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION}').",
|
|
166
80
|
)
|
|
167
81
|
def generate_terraform(name, terraform_platform_modules_version):
|
|
168
|
-
conf = load_and_validate_platform_config()
|
|
169
|
-
|
|
170
|
-
env_config = apply_environment_defaults(conf)["environments"][name]
|
|
171
|
-
_generate_terraform_environment_manifests(
|
|
172
|
-
conf["application"], name, env_config, terraform_platform_modules_version
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
def _generate_copilot_environment_manifests(environment_name, application, env_config, session):
|
|
177
|
-
env_template = setup_templates().get_template("env/manifest.yml")
|
|
178
|
-
vpc_name = env_config.get("vpc", None)
|
|
179
|
-
vpc_id = get_vpc_id(session, environment_name, vpc_name)
|
|
180
|
-
pub_subnet_ids, priv_subnet_ids = get_subnet_ids(session, vpc_id, environment_name)
|
|
181
|
-
cert_arn = get_cert_arn(session, application, environment_name)
|
|
182
|
-
contents = env_template.render(
|
|
183
|
-
{
|
|
184
|
-
"name": environment_name,
|
|
185
|
-
"vpc_id": vpc_id,
|
|
186
|
-
"pub_subnet_ids": pub_subnet_ids,
|
|
187
|
-
"priv_subnet_ids": priv_subnet_ids,
|
|
188
|
-
"certificate_arn": cert_arn,
|
|
189
|
-
}
|
|
190
|
-
)
|
|
191
|
-
click.echo(
|
|
192
|
-
mkfile(
|
|
193
|
-
".", f"copilot/environments/{environment_name}/manifest.yml", contents, overwrite=True
|
|
194
|
-
)
|
|
195
|
-
)
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
def _generate_terraform_environment_manifests(
|
|
199
|
-
application, env, env_config, cli_terraform_platform_modules_version
|
|
200
|
-
):
|
|
201
|
-
env_template = setup_templates().get_template("environments/main.tf")
|
|
202
|
-
|
|
203
|
-
terraform_platform_modules_version = _determine_terraform_platform_modules_version(
|
|
204
|
-
env_config, cli_terraform_platform_modules_version
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
contents = env_template.render(
|
|
208
|
-
{
|
|
209
|
-
"application": application,
|
|
210
|
-
"environment": env,
|
|
211
|
-
"config": env_config,
|
|
212
|
-
"terraform_platform_modules_version": terraform_platform_modules_version,
|
|
213
|
-
}
|
|
214
|
-
)
|
|
215
|
-
|
|
216
|
-
click.echo(mkfile(".", f"terraform/environments/{env}/main.tf", contents, overwrite=True))
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
def _determine_terraform_platform_modules_version(env_conf, cli_terraform_platform_modules_version):
|
|
220
|
-
cli_terraform_platform_modules_version = cli_terraform_platform_modules_version
|
|
221
|
-
env_conf_terraform_platform_modules_version = env_conf.get("versions", {}).get(
|
|
222
|
-
"terraform-platform-modules"
|
|
223
|
-
)
|
|
224
|
-
version_preference_order = [
|
|
225
|
-
cli_terraform_platform_modules_version,
|
|
226
|
-
env_conf_terraform_platform_modules_version,
|
|
227
|
-
DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
|
|
228
|
-
]
|
|
229
|
-
return [version for version in version_preference_order if version][0]
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
def find_https_certificate(session: boto3.Session, app: str, env: str) -> str:
|
|
233
|
-
listener_arn = find_https_listener(session, app, env)
|
|
234
|
-
cert_client = session.client("elbv2")
|
|
235
|
-
certificates = cert_client.describe_listener_certificates(ListenerArn=listener_arn)[
|
|
236
|
-
"Certificates"
|
|
237
|
-
]
|
|
238
82
|
|
|
239
83
|
try:
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
class CertificateNotFoundException(PlatformException):
|
|
248
|
-
pass
|
|
84
|
+
config_provider = ConfigProvider(ConfigValidator())
|
|
85
|
+
TerraformEnvironment(config_provider).generate(name, terraform_platform_modules_version)
|
|
86
|
+
except PlatformException as err:
|
|
87
|
+
click.secho(str(err), fg="red")
|
|
88
|
+
raise click.Abort
|
|
@@ -1,30 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
|
-
from os import makedirs
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
from shutil import rmtree
|
|
5
2
|
|
|
6
3
|
import click
|
|
7
4
|
|
|
8
5
|
from dbt_platform_helper.constants import DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION
|
|
9
|
-
from dbt_platform_helper.
|
|
10
|
-
from dbt_platform_helper.
|
|
6
|
+
from dbt_platform_helper.domain.config_validator import ConfigValidator
|
|
7
|
+
from dbt_platform_helper.domain.pipelines import Pipelines
|
|
8
|
+
from dbt_platform_helper.providers.config import ConfigProvider
|
|
11
9
|
from dbt_platform_helper.utils.aws import get_codestar_connection_arn
|
|
12
|
-
from dbt_platform_helper.utils.aws import get_public_repository_arn
|
|
13
10
|
from dbt_platform_helper.utils.click import ClickDocOptGroup
|
|
14
|
-
from dbt_platform_helper.utils.files import generate_override_files_from_template
|
|
15
|
-
from dbt_platform_helper.utils.files import mkfile
|
|
16
11
|
from dbt_platform_helper.utils.git import git_remote
|
|
17
12
|
from dbt_platform_helper.utils.messages import abort_with_error
|
|
18
|
-
from dbt_platform_helper.utils.template import setup_templates
|
|
19
|
-
from dbt_platform_helper.utils.validation import load_and_validate_platform_config
|
|
20
13
|
from dbt_platform_helper.utils.versioning import (
|
|
21
14
|
check_platform_helper_version_needs_update,
|
|
22
15
|
)
|
|
23
16
|
|
|
24
|
-
CODEBASE_PIPELINES_KEY = "codebase_pipelines"
|
|
25
|
-
ENVIRONMENTS_KEY = "environments"
|
|
26
|
-
ENVIRONMENT_PIPELINES_KEY = "environment_pipelines"
|
|
27
|
-
|
|
28
17
|
|
|
29
18
|
@click.group(chain=True, cls=ClickDocOptGroup)
|
|
30
19
|
def pipeline():
|
|
@@ -60,164 +49,12 @@ def generate(terraform_platform_modules_version, deploy_branch):
|
|
|
60
49
|
|
|
61
50
|
This command does the following in relation to the codebase pipelines:
|
|
62
51
|
- Generates the copilot pipeline manifest.yml for copilot/pipelines/<codebase_pipeline_name>
|
|
63
|
-
|
|
64
|
-
(Deprecated) This command does the following for non terraform projects (legacy AWS Copilot):
|
|
65
|
-
- Generates the copilot manifest.yml for copilot/environments/<environment>
|
|
66
52
|
"""
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
click.secho("No pipelines defined: nothing to do.", err=True, fg="yellow")
|
|
74
|
-
return
|
|
75
|
-
|
|
76
|
-
platform_config_terraform_modules_default_version = pipeline_config.get(
|
|
77
|
-
"default_versions", {}
|
|
78
|
-
).get("terraform-platform-modules", "")
|
|
79
|
-
|
|
80
|
-
templates = setup_templates()
|
|
81
|
-
app_name = get_application_name()
|
|
82
|
-
|
|
83
|
-
git_repo = git_remote()
|
|
84
|
-
if not git_repo:
|
|
85
|
-
abort_with_error("The current directory is not a git repository")
|
|
86
|
-
|
|
87
|
-
codestar_connection_arn = get_codestar_connection_arn(app_name)
|
|
88
|
-
if codestar_connection_arn is None:
|
|
89
|
-
abort_with_error(f'There is no CodeStar Connection named "{app_name}" to use')
|
|
90
|
-
|
|
91
|
-
base_path = Path(".")
|
|
92
|
-
copilot_pipelines_dir = base_path / f"copilot/pipelines"
|
|
93
|
-
|
|
94
|
-
_clean_pipeline_config(copilot_pipelines_dir)
|
|
95
|
-
|
|
96
|
-
if has_environment_pipelines:
|
|
97
|
-
environment_pipelines = pipeline_config[ENVIRONMENT_PIPELINES_KEY]
|
|
98
|
-
|
|
99
|
-
for config in environment_pipelines.values():
|
|
100
|
-
aws_account = config.get("account")
|
|
101
|
-
_generate_terraform_environment_pipeline_manifest(
|
|
102
|
-
pipeline_config["application"],
|
|
103
|
-
aws_account,
|
|
104
|
-
terraform_platform_modules_version,
|
|
105
|
-
platform_config_terraform_modules_default_version,
|
|
106
|
-
deploy_branch,
|
|
107
|
-
)
|
|
108
|
-
|
|
109
|
-
if has_codebase_pipelines:
|
|
110
|
-
account_id, _ = get_account_details()
|
|
111
|
-
|
|
112
|
-
for codebase in pipeline_config[CODEBASE_PIPELINES_KEY]:
|
|
113
|
-
_generate_codebase_pipeline(
|
|
114
|
-
account_id,
|
|
115
|
-
app_name,
|
|
116
|
-
codestar_connection_arn,
|
|
117
|
-
git_repo,
|
|
118
|
-
codebase,
|
|
119
|
-
base_path,
|
|
120
|
-
copilot_pipelines_dir,
|
|
121
|
-
templates,
|
|
122
|
-
)
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
def _clean_pipeline_config(pipelines_dir):
|
|
126
|
-
if pipelines_dir.exists():
|
|
127
|
-
click.echo("Deleting copilot/pipelines directory.")
|
|
128
|
-
rmtree(pipelines_dir)
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
def _generate_codebase_pipeline(
|
|
132
|
-
account_id,
|
|
133
|
-
app_name,
|
|
134
|
-
codestar_connection_arn,
|
|
135
|
-
git_repo,
|
|
136
|
-
codebase,
|
|
137
|
-
base_path,
|
|
138
|
-
pipelines_dir,
|
|
139
|
-
templates,
|
|
140
|
-
):
|
|
141
|
-
makedirs(pipelines_dir / codebase["name"] / "overrides", exist_ok=True)
|
|
142
|
-
environments = []
|
|
143
|
-
for pipelines in codebase["pipelines"]:
|
|
144
|
-
environments += pipelines[ENVIRONMENTS_KEY]
|
|
145
|
-
|
|
146
|
-
additional_ecr = codebase.get("additional_ecr_repository", None)
|
|
147
|
-
add_public_perms = additional_ecr and additional_ecr.startswith("public.ecr.aws")
|
|
148
|
-
additional_ecr_arn = get_public_repository_arn(additional_ecr) if add_public_perms else None
|
|
149
|
-
|
|
150
|
-
template_data = {
|
|
151
|
-
"account_id": account_id,
|
|
152
|
-
"app_name": app_name,
|
|
153
|
-
"deploy_repo": git_repo,
|
|
154
|
-
"codebase": codebase,
|
|
155
|
-
ENVIRONMENTS_KEY: environments,
|
|
156
|
-
"codestar_connection_arn": codestar_connection_arn,
|
|
157
|
-
"codestar_connection_id": codestar_connection_arn.split("/")[-1],
|
|
158
|
-
"additional_ecr_arn": additional_ecr_arn,
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
_create_file_from_template(
|
|
162
|
-
base_path,
|
|
163
|
-
f"{codebase['name']}/manifest.yml",
|
|
164
|
-
pipelines_dir,
|
|
165
|
-
template_data,
|
|
166
|
-
templates,
|
|
167
|
-
"codebase/manifest.yml",
|
|
168
|
-
)
|
|
169
|
-
|
|
170
|
-
overrides_path = Path(__file__).parent.parent.joinpath("templates/pipelines/codebase/overrides")
|
|
171
|
-
generate_override_files_from_template(
|
|
172
|
-
base_path, overrides_path, pipelines_dir / codebase["name"] / "overrides", template_data
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
def _create_file_from_template(
|
|
177
|
-
base_path, file_name, pipelines_dir, template_data, templates, template_name=None
|
|
178
|
-
):
|
|
179
|
-
contents = templates.get_template(
|
|
180
|
-
f"pipelines/{file_name if template_name is None else template_name}"
|
|
181
|
-
).render(template_data)
|
|
182
|
-
message = mkfile(base_path, pipelines_dir / file_name, contents, overwrite=True)
|
|
183
|
-
click.echo(message)
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
def _generate_terraform_environment_pipeline_manifest(
|
|
187
|
-
application,
|
|
188
|
-
aws_account,
|
|
189
|
-
cli_terraform_platform_modules_version,
|
|
190
|
-
platform_config_terraform_modules_default_version,
|
|
191
|
-
deploy_branch,
|
|
192
|
-
):
|
|
193
|
-
env_pipeline_template = setup_templates().get_template("environment-pipelines/main.tf")
|
|
194
|
-
|
|
195
|
-
terraform_platform_modules_version = _determine_terraform_platform_modules_version(
|
|
196
|
-
cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
|
|
53
|
+
pipelines = Pipelines(
|
|
54
|
+
ConfigProvider(ConfigValidator()),
|
|
55
|
+
click.secho,
|
|
56
|
+
abort_with_error,
|
|
57
|
+
git_remote,
|
|
58
|
+
get_codestar_connection_arn,
|
|
197
59
|
)
|
|
198
|
-
|
|
199
|
-
contents = env_pipeline_template.render(
|
|
200
|
-
{
|
|
201
|
-
"application": application,
|
|
202
|
-
"aws_account": aws_account,
|
|
203
|
-
"terraform_platform_modules_version": terraform_platform_modules_version,
|
|
204
|
-
"deploy_branch": deploy_branch,
|
|
205
|
-
}
|
|
206
|
-
)
|
|
207
|
-
|
|
208
|
-
dir_path = f"terraform/environment-pipelines/{aws_account}"
|
|
209
|
-
makedirs(dir_path, exist_ok=True)
|
|
210
|
-
|
|
211
|
-
click.echo(mkfile(".", f"{dir_path}/main.tf", contents, overwrite=True))
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
def _determine_terraform_platform_modules_version(
|
|
215
|
-
cli_terraform_platform_modules_version, platform_config_terraform_modules_default_version
|
|
216
|
-
):
|
|
217
|
-
|
|
218
|
-
version_preference_order = [
|
|
219
|
-
cli_terraform_platform_modules_version,
|
|
220
|
-
platform_config_terraform_modules_default_version,
|
|
221
|
-
DEFAULT_TERRAFORM_PLATFORM_MODULES_VERSION,
|
|
222
|
-
]
|
|
223
|
-
return [version for version in version_preference_order if version][0]
|
|
60
|
+
pipelines.generate(terraform_platform_modules_version, deploy_branch)
|