dbt-platform-helper 13.1.0__py3-none-any.whl → 15.16.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbt_platform_helper/COMMANDS.md +107 -27
- dbt_platform_helper/commands/application.py +5 -6
- dbt_platform_helper/commands/codebase.py +31 -10
- dbt_platform_helper/commands/conduit.py +3 -5
- dbt_platform_helper/commands/config.py +20 -311
- dbt_platform_helper/commands/copilot.py +18 -391
- dbt_platform_helper/commands/database.py +17 -9
- dbt_platform_helper/commands/environment.py +20 -14
- dbt_platform_helper/commands/generate.py +0 -3
- dbt_platform_helper/commands/internal.py +140 -0
- dbt_platform_helper/commands/notify.py +58 -78
- dbt_platform_helper/commands/pipeline.py +23 -19
- dbt_platform_helper/commands/secrets.py +39 -93
- dbt_platform_helper/commands/version.py +7 -12
- dbt_platform_helper/constants.py +52 -7
- dbt_platform_helper/domain/codebase.py +89 -39
- dbt_platform_helper/domain/conduit.py +335 -76
- dbt_platform_helper/domain/config.py +381 -0
- dbt_platform_helper/domain/copilot.py +398 -0
- dbt_platform_helper/domain/copilot_environment.py +8 -8
- dbt_platform_helper/domain/database_copy.py +2 -2
- dbt_platform_helper/domain/maintenance_page.py +254 -430
- dbt_platform_helper/domain/notify.py +64 -0
- dbt_platform_helper/domain/pipelines.py +43 -35
- dbt_platform_helper/domain/plans.py +41 -0
- dbt_platform_helper/domain/secrets.py +279 -0
- dbt_platform_helper/domain/service.py +570 -0
- dbt_platform_helper/domain/terraform_environment.py +14 -13
- dbt_platform_helper/domain/update_alb_rules.py +412 -0
- dbt_platform_helper/domain/versioning.py +249 -0
- dbt_platform_helper/{providers → entities}/platform_config_schema.py +75 -82
- dbt_platform_helper/entities/semantic_version.py +83 -0
- dbt_platform_helper/entities/service.py +339 -0
- dbt_platform_helper/platform_exception.py +4 -0
- dbt_platform_helper/providers/autoscaling.py +24 -0
- dbt_platform_helper/providers/aws/__init__.py +0 -0
- dbt_platform_helper/providers/aws/exceptions.py +70 -0
- dbt_platform_helper/providers/aws/interfaces.py +13 -0
- dbt_platform_helper/providers/aws/opensearch.py +23 -0
- dbt_platform_helper/providers/aws/redis.py +21 -0
- dbt_platform_helper/providers/aws/sso_auth.py +75 -0
- dbt_platform_helper/providers/cache.py +40 -4
- dbt_platform_helper/providers/cloudformation.py +1 -1
- dbt_platform_helper/providers/config.py +137 -19
- dbt_platform_helper/providers/config_validator.py +112 -51
- dbt_platform_helper/providers/copilot.py +24 -16
- dbt_platform_helper/providers/ecr.py +89 -7
- dbt_platform_helper/providers/ecs.py +228 -36
- dbt_platform_helper/providers/environment_variable.py +24 -0
- dbt_platform_helper/providers/files.py +1 -1
- dbt_platform_helper/providers/io.py +36 -4
- dbt_platform_helper/providers/kms.py +22 -0
- dbt_platform_helper/providers/load_balancers.py +402 -42
- dbt_platform_helper/providers/logs.py +72 -0
- dbt_platform_helper/providers/parameter_store.py +134 -0
- dbt_platform_helper/providers/s3.py +21 -0
- dbt_platform_helper/providers/schema_migrations/__init__.py +0 -0
- dbt_platform_helper/providers/schema_migrations/schema_v0_to_v1_migration.py +43 -0
- dbt_platform_helper/providers/schema_migrator.py +77 -0
- dbt_platform_helper/providers/secrets.py +5 -5
- dbt_platform_helper/providers/slack_channel_notifier.py +62 -0
- dbt_platform_helper/providers/terraform_manifest.py +121 -19
- dbt_platform_helper/providers/version.py +106 -23
- dbt_platform_helper/providers/version_status.py +27 -0
- dbt_platform_helper/providers/vpc.py +36 -5
- dbt_platform_helper/providers/yaml_file.py +58 -2
- dbt_platform_helper/templates/environment-pipelines/main.tf +4 -3
- dbt_platform_helper/templates/svc/overrides/cfn.patches.yml +5 -0
- dbt_platform_helper/utilities/decorators.py +103 -0
- dbt_platform_helper/utils/application.py +119 -22
- dbt_platform_helper/utils/aws.py +39 -150
- dbt_platform_helper/utils/deep_merge.py +10 -0
- dbt_platform_helper/utils/git.py +1 -14
- dbt_platform_helper/utils/validation.py +1 -1
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/METADATA +11 -20
- dbt_platform_helper-15.16.0.dist-info/RECORD +118 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/WHEEL +1 -1
- platform_helper.py +3 -1
- terraform/elasticache-redis/plans.yml +85 -0
- terraform/opensearch/plans.yml +71 -0
- terraform/postgres/plans.yml +128 -0
- dbt_platform_helper/addon-plans.yml +0 -224
- dbt_platform_helper/providers/aws.py +0 -37
- dbt_platform_helper/providers/opensearch.py +0 -36
- dbt_platform_helper/providers/redis.py +0 -34
- dbt_platform_helper/providers/semantic_version.py +0 -126
- dbt_platform_helper/templates/svc/manifest-backend.yml +0 -69
- dbt_platform_helper/templates/svc/manifest-public.yml +0 -109
- dbt_platform_helper/utils/cloudfoundry.py +0 -14
- dbt_platform_helper/utils/files.py +0 -53
- dbt_platform_helper/utils/manifests.py +0 -18
- dbt_platform_helper/utils/versioning.py +0 -238
- dbt_platform_helper-13.1.0.dist-info/RECORD +0 -96
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info}/entry_points.txt +0 -0
- {dbt_platform_helper-13.1.0.dist-info → dbt_platform_helper-15.16.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -8,18 +8,19 @@ from typing import Callable
|
|
|
8
8
|
from typing import List
|
|
9
9
|
from typing import Union
|
|
10
10
|
|
|
11
|
-
import boto3
|
|
12
11
|
import click
|
|
13
12
|
|
|
13
|
+
from dbt_platform_helper.constants import MAINTENANCE_PAGE_REASON
|
|
14
|
+
from dbt_platform_helper.constants import MANAGED_BY_PLATFORM
|
|
14
15
|
from dbt_platform_helper.platform_exception import PlatformException
|
|
15
16
|
from dbt_platform_helper.providers.io import ClickIOProvider
|
|
16
|
-
from dbt_platform_helper.providers.load_balancers import ListenerNotFoundException
|
|
17
17
|
from dbt_platform_helper.providers.load_balancers import ListenerRuleNotFoundException
|
|
18
|
-
from dbt_platform_helper.providers.load_balancers import
|
|
19
|
-
from dbt_platform_helper.providers.load_balancers import (
|
|
20
|
-
get_https_listener_for_application,
|
|
21
|
-
)
|
|
18
|
+
from dbt_platform_helper.providers.load_balancers import LoadBalancerProvider
|
|
22
19
|
from dbt_platform_helper.utils.application import Application
|
|
20
|
+
from dbt_platform_helper.utils.application import (
|
|
21
|
+
ApplicationEnvironmentNotFoundException,
|
|
22
|
+
)
|
|
23
|
+
from dbt_platform_helper.utils.application import ApplicationServiceNotFoundException
|
|
23
24
|
from dbt_platform_helper.utils.application import Environment
|
|
24
25
|
from dbt_platform_helper.utils.application import Service
|
|
25
26
|
|
|
@@ -39,37 +40,17 @@ class FailedToActivateMaintenancePageException(MaintenancePageException):
|
|
|
39
40
|
application_name: str,
|
|
40
41
|
env: str,
|
|
41
42
|
original_exception: Exception,
|
|
42
|
-
rolled_back_rules: dict[str, bool] = {},
|
|
43
43
|
):
|
|
44
44
|
super().__init__(
|
|
45
45
|
f"Maintenance page failed to activate for the {application_name} application in environment {env}."
|
|
46
46
|
)
|
|
47
47
|
self.orginal_exception = original_exception
|
|
48
|
-
self.rolled_back_rules = rolled_back_rules
|
|
49
48
|
|
|
50
49
|
def __str__(self):
|
|
51
|
-
return (
|
|
52
|
-
f"{super().__str__()}\n"
|
|
53
|
-
f"Rolled-back rules: {self.rolled_back_rules }\n"
|
|
54
|
-
f"Original exception: {self.orginal_exception}"
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def get_maintenance_page_type(session: boto3.Session, listener_arn: str) -> Union[str, None]:
|
|
59
|
-
lb_client = session.client("elbv2")
|
|
60
|
-
|
|
61
|
-
rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
|
|
62
|
-
tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
|
|
63
|
-
|
|
64
|
-
maintenance_page_type = None
|
|
65
|
-
for description in tag_descriptions:
|
|
66
|
-
tags = {t["Key"]: t["Value"] for t in description["Tags"]}
|
|
67
|
-
if tags.get("name") == "MaintenancePage":
|
|
68
|
-
maintenance_page_type = tags.get("type")
|
|
69
|
-
|
|
70
|
-
return maintenance_page_type
|
|
50
|
+
return f"{super().__str__()}\n" f"Original exception: {self.orginal_exception}"
|
|
71
51
|
|
|
72
52
|
|
|
53
|
+
# TODO: DBTP-1958: should this be in its own provider, inside the VPC one, what logic is this sepcific too?
|
|
73
54
|
def get_env_ips(vpc: str, application_environment: Environment) -> List[str]:
|
|
74
55
|
account_name = f"{application_environment.session.profile_name}-vpc"
|
|
75
56
|
vpc_name = vpc if vpc else account_name
|
|
@@ -84,164 +65,19 @@ def get_env_ips(vpc: str, application_environment: Environment) -> List[str]:
|
|
|
84
65
|
return [ip.strip() for ip in param_value.split(",")]
|
|
85
66
|
|
|
86
67
|
|
|
87
|
-
def add_maintenance_page(
|
|
88
|
-
session: boto3.Session,
|
|
89
|
-
listener_arn: str,
|
|
90
|
-
app: str,
|
|
91
|
-
env: str,
|
|
92
|
-
services: List[Service],
|
|
93
|
-
allowed_ips: List[str],
|
|
94
|
-
template: str = "default",
|
|
95
|
-
):
|
|
96
|
-
lb_client = session.client("elbv2")
|
|
97
|
-
maintenance_page_content = get_maintenance_page_template(template)
|
|
98
|
-
bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
|
|
99
|
-
|
|
100
|
-
rule_priority = itertools.count(start=1)
|
|
101
|
-
maintenance_page_host_header_conditions = []
|
|
102
|
-
try:
|
|
103
|
-
for svc in services:
|
|
104
|
-
target_group_arn = find_target_group(app, env, svc.name, session)
|
|
105
|
-
|
|
106
|
-
# not all of an application's services are guaranteed to have been deployed to an environment
|
|
107
|
-
if not target_group_arn:
|
|
108
|
-
continue
|
|
109
|
-
|
|
110
|
-
service_conditions = get_host_header_conditions(
|
|
111
|
-
lb_client, listener_arn, target_group_arn
|
|
112
|
-
)
|
|
113
|
-
|
|
114
|
-
for ip in allowed_ips:
|
|
115
|
-
create_header_rule(
|
|
116
|
-
lb_client,
|
|
117
|
-
listener_arn,
|
|
118
|
-
target_group_arn,
|
|
119
|
-
"X-Forwarded-For",
|
|
120
|
-
[ip],
|
|
121
|
-
"AllowedIps",
|
|
122
|
-
next(rule_priority),
|
|
123
|
-
service_conditions,
|
|
124
|
-
)
|
|
125
|
-
create_source_ip_rule(
|
|
126
|
-
lb_client,
|
|
127
|
-
listener_arn,
|
|
128
|
-
target_group_arn,
|
|
129
|
-
[ip],
|
|
130
|
-
"AllowedSourceIps",
|
|
131
|
-
next(rule_priority),
|
|
132
|
-
service_conditions,
|
|
133
|
-
)
|
|
134
|
-
|
|
135
|
-
create_header_rule(
|
|
136
|
-
lb_client,
|
|
137
|
-
listener_arn,
|
|
138
|
-
target_group_arn,
|
|
139
|
-
"Bypass-Key",
|
|
140
|
-
[bypass_value],
|
|
141
|
-
"BypassIpFilter",
|
|
142
|
-
next(rule_priority),
|
|
143
|
-
service_conditions,
|
|
144
|
-
)
|
|
145
|
-
|
|
146
|
-
# add to accumilating list of conditions for maintenace page rule
|
|
147
|
-
maintenance_page_host_header_conditions.extend(service_conditions)
|
|
148
|
-
|
|
149
|
-
click.secho(
|
|
150
|
-
f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
|
|
151
|
-
)
|
|
152
|
-
|
|
153
|
-
lb_client.create_rule(
|
|
154
|
-
ListenerArn=listener_arn,
|
|
155
|
-
Priority=next(rule_priority),
|
|
156
|
-
Conditions=[
|
|
157
|
-
{
|
|
158
|
-
"Field": "path-pattern",
|
|
159
|
-
"PathPatternConfig": {"Values": ["/*"]},
|
|
160
|
-
},
|
|
161
|
-
{
|
|
162
|
-
"Field": "host-header",
|
|
163
|
-
"HostHeaderConfig": {
|
|
164
|
-
"Values": sorted(
|
|
165
|
-
list(
|
|
166
|
-
{
|
|
167
|
-
value
|
|
168
|
-
for condition in maintenance_page_host_header_conditions
|
|
169
|
-
for value in condition["HostHeaderConfig"]["Values"]
|
|
170
|
-
}
|
|
171
|
-
)
|
|
172
|
-
)
|
|
173
|
-
},
|
|
174
|
-
},
|
|
175
|
-
],
|
|
176
|
-
Actions=[
|
|
177
|
-
{
|
|
178
|
-
"Type": "fixed-response",
|
|
179
|
-
"FixedResponseConfig": {
|
|
180
|
-
"StatusCode": "503",
|
|
181
|
-
"ContentType": "text/html",
|
|
182
|
-
"MessageBody": maintenance_page_content,
|
|
183
|
-
},
|
|
184
|
-
}
|
|
185
|
-
],
|
|
186
|
-
Tags=[
|
|
187
|
-
{"Key": "name", "Value": "MaintenancePage"},
|
|
188
|
-
{"Key": "type", "Value": template},
|
|
189
|
-
],
|
|
190
|
-
)
|
|
191
|
-
except Exception as e:
|
|
192
|
-
deleted_rules = clean_up_maintenance_page_rules(session, listener_arn)
|
|
193
|
-
raise FailedToActivateMaintenancePageException(
|
|
194
|
-
app, env, f"{e}:\n {traceback.format_exc()}", deleted_rules
|
|
195
|
-
)
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
def clean_up_maintenance_page_rules(
|
|
199
|
-
session: boto3.Session, listener_arn: str, fail_when_not_deleted: bool = False
|
|
200
|
-
) -> dict[str, bool]:
|
|
201
|
-
lb_client = session.client("elbv2")
|
|
202
|
-
|
|
203
|
-
rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
|
|
204
|
-
tag_descriptions = get_rules_tag_descriptions(rules, lb_client)
|
|
205
|
-
|
|
206
|
-
deletes = {}
|
|
207
|
-
for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
|
|
208
|
-
deleted = delete_listener_rule(tag_descriptions, name, lb_client)
|
|
209
|
-
deletes[name] = bool(deleted)
|
|
210
|
-
if fail_when_not_deleted and name == "MaintenancePage" and not deleted:
|
|
211
|
-
raise ListenerRuleNotFoundException()
|
|
212
|
-
|
|
213
|
-
return deletes
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
def remove_maintenance_page(session: boto3.Session, listener_arn: str):
|
|
217
|
-
clean_up_maintenance_page_rules(session, listener_arn, True)
|
|
218
|
-
|
|
219
|
-
|
|
220
68
|
class MaintenancePage:
|
|
221
69
|
def __init__(
|
|
222
70
|
self,
|
|
223
71
|
application: Application,
|
|
224
72
|
io: ClickIOProvider = ClickIOProvider(),
|
|
225
|
-
|
|
226
|
-
[boto3.Session, str, str], str
|
|
227
|
-
] = get_https_listener_for_application,
|
|
228
|
-
# TODO refactor get_maintenance_page_type, add_maintenance_page, remove_maintenance_page into MaintenancePage class with LoadBalancerProvider as the dependency
|
|
229
|
-
get_maintenance_page_type: Callable[
|
|
230
|
-
[boto3.Session, str], Union[str, None]
|
|
231
|
-
] = get_maintenance_page_type,
|
|
73
|
+
load_balancer_provider: LoadBalancerProvider = LoadBalancerProvider,
|
|
232
74
|
get_env_ips: Callable[[str, Environment], List[str]] = get_env_ips,
|
|
233
|
-
add_maintenance_page: Callable[
|
|
234
|
-
[boto3.Session, str, str, str, List[Service], tuple, str], None
|
|
235
|
-
] = add_maintenance_page,
|
|
236
|
-
remove_maintenance_page: Callable[[boto3.Session, str], None] = remove_maintenance_page,
|
|
237
75
|
):
|
|
238
76
|
self.application = application
|
|
239
|
-
self.get_https_listener_for_application = get_https_listener_for_application
|
|
240
77
|
self.io = io
|
|
241
|
-
self.
|
|
78
|
+
self.load_balancer_provider = load_balancer_provider # TODO: DBTP-1962: requires session from environment in application object which is only known during method execution
|
|
79
|
+
self.load_balancer: LoadBalancerProvider = None
|
|
242
80
|
self.get_env_ips = get_env_ips
|
|
243
|
-
self.add_maintenance_page = add_maintenance_page
|
|
244
|
-
self.remove_maintenance_page = remove_maintenance_page
|
|
245
81
|
|
|
246
82
|
def _get_deployed_load_balanced_web_services(self, app: Application, svc: List[str]):
|
|
247
83
|
if "*" in svc:
|
|
@@ -253,155 +89,288 @@ class MaintenancePage:
|
|
|
253
89
|
raise LoadBalancedWebServiceNotFoundException(app.name)
|
|
254
90
|
return services
|
|
255
91
|
|
|
92
|
+
# TODO: DBTP-1962: inject load balancer provider in activate method to avoid passing load balancer provider in init?
|
|
256
93
|
def activate(self, env: str, services: List[str], template: str, vpc: Union[str, None]):
|
|
257
94
|
|
|
258
95
|
services = self._get_deployed_load_balanced_web_services(self.application, services)
|
|
259
96
|
application_environment = get_app_environment(self.application, env)
|
|
97
|
+
self.load_balancer = self.load_balancer_provider(application_environment.session)
|
|
260
98
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
99
|
+
https_listener = self.load_balancer.get_https_listener_for_application(
|
|
100
|
+
self.application.name, env
|
|
101
|
+
)
|
|
102
|
+
current_maintenance_page = self.__get_maintenance_page_type(https_listener)
|
|
103
|
+
remove_current_maintenance_page = False
|
|
104
|
+
if current_maintenance_page:
|
|
105
|
+
remove_current_maintenance_page = self.io.confirm(
|
|
106
|
+
f"There is currently a '{current_maintenance_page}' maintenance page for the {env} "
|
|
107
|
+
f"environment in {self.application.name}.\nWould you like to replace it with a '{template}' "
|
|
108
|
+
f"maintenance page?"
|
|
267
109
|
)
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
remove_current_maintenance_page = self.io.confirm(
|
|
271
|
-
f"There is currently a '{current_maintenance_page}' maintenance page for the {env} "
|
|
272
|
-
f"environment in {self.application.name}.\nWould you like to replace it with a '{template}' "
|
|
273
|
-
f"maintenance page?"
|
|
274
|
-
)
|
|
275
|
-
if not remove_current_maintenance_page:
|
|
276
|
-
return
|
|
277
|
-
|
|
278
|
-
if remove_current_maintenance_page or self.io.confirm(
|
|
279
|
-
f"You are about to enable the '{template}' maintenance page for the {env} "
|
|
280
|
-
f"environment in {self.application.name}.\nWould you like to continue?"
|
|
281
|
-
):
|
|
282
|
-
if current_maintenance_page and remove_current_maintenance_page:
|
|
283
|
-
self.remove_maintenance_page(application_environment.session, https_listener)
|
|
284
|
-
|
|
285
|
-
allowed_ips = self.get_env_ips(vpc, application_environment)
|
|
286
|
-
|
|
287
|
-
self.add_maintenance_page(
|
|
288
|
-
application_environment.session,
|
|
289
|
-
https_listener,
|
|
290
|
-
self.application.name,
|
|
291
|
-
env,
|
|
292
|
-
services,
|
|
293
|
-
allowed_ips,
|
|
294
|
-
template,
|
|
295
|
-
)
|
|
296
|
-
self.io.info(
|
|
297
|
-
f"Maintenance page '{template}' added for environment {env} in application {self.application.name}",
|
|
298
|
-
)
|
|
110
|
+
if not remove_current_maintenance_page:
|
|
111
|
+
return
|
|
299
112
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
self.
|
|
303
|
-
|
|
113
|
+
if remove_current_maintenance_page or self.io.confirm(
|
|
114
|
+
f"You are about to enable the '{template}' maintenance page for the {env} "
|
|
115
|
+
f"environment in {self.application.name}.\nWould you like to continue?"
|
|
116
|
+
):
|
|
117
|
+
if current_maintenance_page and remove_current_maintenance_page:
|
|
118
|
+
self.__remove_maintenance_page(https_listener)
|
|
119
|
+
|
|
120
|
+
allowed_ips = self.get_env_ips(vpc, application_environment)
|
|
121
|
+
|
|
122
|
+
self.add_maintenance_page(
|
|
123
|
+
https_listener,
|
|
124
|
+
self.application.name,
|
|
125
|
+
env,
|
|
126
|
+
services,
|
|
127
|
+
allowed_ips,
|
|
128
|
+
template,
|
|
304
129
|
)
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
# TODO push exception to command layer
|
|
308
|
-
self.io.abort_with_error(
|
|
309
|
-
f"No HTTPS listener found for environment {env} in the application {self.application.name}.",
|
|
130
|
+
self.io.info(
|
|
131
|
+
f"Maintenance page '{template}' added for environment {env} in application {self.application.name}",
|
|
310
132
|
)
|
|
311
133
|
|
|
312
134
|
def deactivate(self, env: str):
|
|
313
135
|
application_environment = get_app_environment(self.application, env)
|
|
314
136
|
|
|
137
|
+
self.load_balancer = self.load_balancer_provider(application_environment.session)
|
|
138
|
+
|
|
139
|
+
https_listener = self.load_balancer.get_https_listener_for_application(
|
|
140
|
+
self.application.name, env
|
|
141
|
+
)
|
|
142
|
+
current_maintenance_page = self.__get_maintenance_page_type(https_listener)
|
|
143
|
+
|
|
144
|
+
if not current_maintenance_page:
|
|
145
|
+
self.io.warn("There is no current maintenance page to remove")
|
|
146
|
+
return
|
|
147
|
+
|
|
148
|
+
if not self.io.confirm(
|
|
149
|
+
f"There is currently a '{current_maintenance_page}' maintenance page, "
|
|
150
|
+
f"would you like to remove it?"
|
|
151
|
+
):
|
|
152
|
+
return
|
|
153
|
+
|
|
154
|
+
self.__remove_maintenance_page(https_listener)
|
|
155
|
+
self.io.info(
|
|
156
|
+
f"Maintenance page removed from environment {env} in application {self.application.name}",
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
def add_maintenance_page(
|
|
160
|
+
self,
|
|
161
|
+
listener_arn: str,
|
|
162
|
+
app: str,
|
|
163
|
+
env: str,
|
|
164
|
+
services: List[Service],
|
|
165
|
+
allowed_ips: List[str],
|
|
166
|
+
template: str = "default",
|
|
167
|
+
):
|
|
168
|
+
|
|
169
|
+
maintenance_page_content = get_maintenance_page_template(template)
|
|
170
|
+
bypass_value = "".join(random.choices(string.ascii_lowercase + string.digits, k=12))
|
|
171
|
+
|
|
172
|
+
rule_priority = itertools.count(start=1)
|
|
173
|
+
maintenance_page_host_header_conditions = []
|
|
315
174
|
try:
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
)
|
|
319
|
-
current_maintenance_page = self.get_maintenance_page_type(
|
|
320
|
-
application_environment.session, https_listener
|
|
321
|
-
)
|
|
175
|
+
for svc in services:
|
|
176
|
+
target_group_arn = self.load_balancer.find_target_group(app, env, svc.name)
|
|
322
177
|
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
return
|
|
178
|
+
# not all of an application's services are guaranteed to have been deployed to an environment
|
|
179
|
+
if not target_group_arn:
|
|
180
|
+
continue
|
|
327
181
|
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
182
|
+
service_conditions = self.load_balancer.get_host_header_conditions(
|
|
183
|
+
listener_arn, target_group_arn
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
self.io.debug(
|
|
187
|
+
f"""
|
|
188
|
+
#----------------------------------------------------------#
|
|
189
|
+
# Creating listener rules for service {svc.name.ljust(21, " ")}#
|
|
190
|
+
#----------------------------------------------------------#
|
|
191
|
+
|
|
192
|
+
""",
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
for ip in allowed_ips:
|
|
196
|
+
self.load_balancer.create_header_rule(
|
|
197
|
+
listener_arn,
|
|
198
|
+
target_group_arn,
|
|
199
|
+
"X-Forwarded-For",
|
|
200
|
+
[ip],
|
|
201
|
+
"AllowedIps",
|
|
202
|
+
next(rule_priority),
|
|
203
|
+
service_conditions,
|
|
204
|
+
[
|
|
205
|
+
{"Key": "application", "Value": app},
|
|
206
|
+
{"Key": "environment", "Value": env},
|
|
207
|
+
{"Key": "reason", "Value": MAINTENANCE_PAGE_REASON},
|
|
208
|
+
{"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
|
|
209
|
+
{"Key": "service", "Value": svc.name},
|
|
210
|
+
],
|
|
211
|
+
)
|
|
212
|
+
self.load_balancer.create_source_ip_rule(
|
|
213
|
+
listener_arn,
|
|
214
|
+
target_group_arn,
|
|
215
|
+
[ip],
|
|
216
|
+
"AllowedSourceIps",
|
|
217
|
+
next(rule_priority),
|
|
218
|
+
service_conditions,
|
|
219
|
+
[
|
|
220
|
+
{"Key": "application", "Value": app},
|
|
221
|
+
{"Key": "environment", "Value": env},
|
|
222
|
+
{"Key": "reason", "Value": MAINTENANCE_PAGE_REASON},
|
|
223
|
+
{"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
|
|
224
|
+
{"Key": "service", "Value": svc.name},
|
|
225
|
+
],
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
self.load_balancer.create_header_rule(
|
|
229
|
+
listener_arn,
|
|
230
|
+
target_group_arn,
|
|
231
|
+
"Bypass-Key",
|
|
232
|
+
[bypass_value],
|
|
233
|
+
"BypassIpFilter",
|
|
234
|
+
next(rule_priority),
|
|
235
|
+
service_conditions,
|
|
236
|
+
[
|
|
237
|
+
{"Key": "application", "Value": app},
|
|
238
|
+
{"Key": "environment", "Value": env},
|
|
239
|
+
{"Key": "reason", "Value": MAINTENANCE_PAGE_REASON},
|
|
240
|
+
{"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
|
|
241
|
+
{"Key": "service", "Value": svc.name},
|
|
242
|
+
],
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
# add to accumilating list of conditions for maintenace page rule
|
|
246
|
+
maintenance_page_host_header_conditions.extend(service_conditions)
|
|
333
247
|
|
|
334
|
-
self.remove_maintenance_page(application_environment.session, https_listener)
|
|
335
248
|
self.io.info(
|
|
336
|
-
f"
|
|
249
|
+
f"\nUse a browser plugin to add `Bypass-Key` header with value {bypass_value} to your requests. For more detail, visit https://platform.readme.trade.gov.uk/next-steps/put-a-service-under-maintenance/",
|
|
337
250
|
)
|
|
338
251
|
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
252
|
+
unique_sorted_host_headers = sorted(
|
|
253
|
+
list(
|
|
254
|
+
{
|
|
255
|
+
value
|
|
256
|
+
for condition in maintenance_page_host_header_conditions
|
|
257
|
+
for value in condition["HostHeaderConfig"]["Values"]
|
|
258
|
+
}
|
|
259
|
+
)
|
|
343
260
|
)
|
|
344
261
|
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
262
|
+
# Can only set 4 host headers per rule as listener rules have a max conditions of 5
|
|
263
|
+
for i in range(0, len(unique_sorted_host_headers), 4):
|
|
264
|
+
self.load_balancer.create_rule(
|
|
265
|
+
listener_arn=listener_arn,
|
|
266
|
+
priority=next(rule_priority),
|
|
267
|
+
conditions=[
|
|
268
|
+
{
|
|
269
|
+
"Field": "path-pattern",
|
|
270
|
+
"PathPatternConfig": {"Values": ["/*"]},
|
|
271
|
+
},
|
|
272
|
+
{
|
|
273
|
+
"Field": "host-header",
|
|
274
|
+
"HostHeaderConfig": {
|
|
275
|
+
"Values": unique_sorted_host_headers[i : i + 4],
|
|
276
|
+
},
|
|
277
|
+
},
|
|
278
|
+
],
|
|
279
|
+
actions=[
|
|
280
|
+
{
|
|
281
|
+
"Type": "fixed-response",
|
|
282
|
+
"FixedResponseConfig": {
|
|
283
|
+
"StatusCode": "503",
|
|
284
|
+
"ContentType": "text/html",
|
|
285
|
+
"MessageBody": maintenance_page_content,
|
|
286
|
+
},
|
|
287
|
+
}
|
|
288
|
+
],
|
|
289
|
+
tags=[
|
|
290
|
+
{"Key": "application", "Value": app},
|
|
291
|
+
{"Key": "environment", "Value": env},
|
|
292
|
+
{"Key": "reason", "Value": MAINTENANCE_PAGE_REASON},
|
|
293
|
+
{"Key": "name", "Value": "MaintenancePage"},
|
|
294
|
+
{"Key": "type", "Value": template},
|
|
295
|
+
{"Key": "managed-by", "Value": MANAGED_BY_PLATFORM},
|
|
296
|
+
],
|
|
297
|
+
)
|
|
298
|
+
except Exception as e:
|
|
299
|
+
self.__clean_up_maintenance_page_rules(listener_arn)
|
|
300
|
+
raise FailedToActivateMaintenancePageException(
|
|
301
|
+
app, env, f"{e}:\n {traceback.format_exc()}"
|
|
349
302
|
)
|
|
350
303
|
|
|
304
|
+
def __clean_up_maintenance_page_rules(
|
|
305
|
+
self, listener_arn: str, fail_when_not_deleted: bool = False
|
|
306
|
+
) -> None:
|
|
351
307
|
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
if not application_service:
|
|
356
|
-
# TODO raise exception instead of abort
|
|
357
|
-
click.secho(
|
|
358
|
-
f"The service {svc_name} was not found in the application {application.name}. "
|
|
359
|
-
f"It either does not exist, or has not been deployed.",
|
|
360
|
-
fg="red",
|
|
308
|
+
tag_descriptions = self.load_balancer.get_rules_tag_descriptions_by_listener_arn(
|
|
309
|
+
listener_arn
|
|
361
310
|
)
|
|
362
|
-
raise click.Abort
|
|
363
311
|
|
|
364
|
-
|
|
312
|
+
# keep track of rules deleted
|
|
313
|
+
deleted_rules = {"MaintenancePage": 0}
|
|
314
|
+
for name in ["MaintenancePage", "AllowedIps", "BypassIpFilter", "AllowedSourceIps"]:
|
|
315
|
+
deleted_list = self.load_balancer.delete_listener_rule_by_tags(tag_descriptions, name)
|
|
316
|
+
|
|
317
|
+
# track the rules deleted grouped by service
|
|
318
|
+
for deleted_rule in deleted_list:
|
|
319
|
+
tags = {t["Key"]: t["Value"] for t in deleted_rule["Tags"]}
|
|
320
|
+
if "service" in tags:
|
|
321
|
+
if tags["service"] not in deleted_rules:
|
|
322
|
+
deleted_rules[tags["service"]] = {
|
|
323
|
+
"AllowedIps": 0,
|
|
324
|
+
"BypassIpFilter": 0,
|
|
325
|
+
"AllowedSourceIps": 0,
|
|
326
|
+
}
|
|
327
|
+
deleted_rules[tags["service"]][name] += 1
|
|
328
|
+
elif tags.get("name") == "MaintenancePage":
|
|
329
|
+
deleted_rules["MaintenancePage"] += 1
|
|
330
|
+
|
|
331
|
+
if (
|
|
332
|
+
fail_when_not_deleted
|
|
333
|
+
and name == "MaintenancePage"
|
|
334
|
+
and deleted_rules["MaintenancePage"] == 0
|
|
335
|
+
):
|
|
336
|
+
raise ListenerRuleNotFoundException()
|
|
365
337
|
|
|
338
|
+
self.io.warn(
|
|
339
|
+
f"Rules deleted by type and grouped by service: {deleted_rules}",
|
|
340
|
+
)
|
|
366
341
|
|
|
367
|
-
def
|
|
368
|
-
|
|
342
|
+
def __remove_maintenance_page(self, listener_arn: str) -> dict[str, bool]:
|
|
343
|
+
self.__clean_up_maintenance_page_rules(listener_arn, True)
|
|
369
344
|
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
f"It either does not exist, or has not been deployed.",
|
|
374
|
-
fg="red",
|
|
345
|
+
def __get_maintenance_page_type(self, listener_arn: str) -> Union[str, None]:
|
|
346
|
+
tag_descriptions = self.load_balancer.get_rules_tag_descriptions_by_listener_arn(
|
|
347
|
+
listener_arn
|
|
375
348
|
)
|
|
376
|
-
|
|
349
|
+
maintenance_page_type = None
|
|
350
|
+
for description in tag_descriptions:
|
|
351
|
+
tags = {t["Key"]: t["Value"] for t in description["Tags"]}
|
|
352
|
+
if tags.get("name") == "MaintenancePage":
|
|
353
|
+
maintenance_page_type = tags.get("type")
|
|
377
354
|
|
|
378
|
-
|
|
355
|
+
return maintenance_page_type
|
|
379
356
|
|
|
380
357
|
|
|
381
|
-
def
|
|
382
|
-
|
|
383
|
-
chunk_size = 20
|
|
358
|
+
def get_app_service(application: Application, svc_name: str) -> Service:
|
|
359
|
+
application_service = application.services.get(svc_name)
|
|
384
360
|
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
resource_arns = [r["RuleArn"] for r in chunk]
|
|
388
|
-
response = lb_client.describe_tags(ResourceArns=resource_arns)
|
|
389
|
-
tag_descriptions.extend(response["TagDescriptions"])
|
|
361
|
+
if not application_service:
|
|
362
|
+
raise ApplicationServiceNotFoundException(application.name, svc_name)
|
|
390
363
|
|
|
391
|
-
return
|
|
364
|
+
return application_service
|
|
392
365
|
|
|
393
366
|
|
|
394
|
-
def
|
|
395
|
-
|
|
367
|
+
def get_app_environment(application: Application, env_name: str) -> Environment:
|
|
368
|
+
application_environment = application.environments.get(env_name)
|
|
396
369
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
if tags.get("name") == tag_name:
|
|
400
|
-
current_rule_arn = description["ResourceArn"]
|
|
401
|
-
if current_rule_arn:
|
|
402
|
-
lb_client.delete_rule(RuleArn=current_rule_arn)
|
|
370
|
+
if not application_environment:
|
|
371
|
+
raise ApplicationEnvironmentNotFoundException(application.name, env_name)
|
|
403
372
|
|
|
404
|
-
return
|
|
373
|
+
return application_environment
|
|
405
374
|
|
|
406
375
|
|
|
407
376
|
def get_maintenance_page_template(template) -> str:
|
|
@@ -416,148 +385,3 @@ def get_maintenance_page_template(template) -> str:
|
|
|
416
385
|
|
|
417
386
|
# [^\S]\s+ - Remove any space that is not preceded by a non-space character.
|
|
418
387
|
return re.sub(r"[^\S]\s+", "", template_contents)
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
def find_target_group(app: str, env: str, svc: str, session: boto3.Session) -> str:
|
|
422
|
-
rg_tagging_client = session.client("resourcegroupstaggingapi")
|
|
423
|
-
response = rg_tagging_client.get_resources(
|
|
424
|
-
TagFilters=[
|
|
425
|
-
{
|
|
426
|
-
"Key": "copilot-application",
|
|
427
|
-
"Values": [
|
|
428
|
-
app,
|
|
429
|
-
],
|
|
430
|
-
"Key": "copilot-environment",
|
|
431
|
-
"Values": [
|
|
432
|
-
env,
|
|
433
|
-
],
|
|
434
|
-
"Key": "copilot-service",
|
|
435
|
-
"Values": [
|
|
436
|
-
svc,
|
|
437
|
-
],
|
|
438
|
-
},
|
|
439
|
-
],
|
|
440
|
-
ResourceTypeFilters=[
|
|
441
|
-
"elasticloadbalancing:targetgroup",
|
|
442
|
-
],
|
|
443
|
-
)
|
|
444
|
-
for resource in response["ResourceTagMappingList"]:
|
|
445
|
-
tags = {tag["Key"]: tag["Value"] for tag in resource["Tags"]}
|
|
446
|
-
|
|
447
|
-
if (
|
|
448
|
-
"copilot-service" in tags
|
|
449
|
-
and tags["copilot-service"] == svc
|
|
450
|
-
and "copilot-environment" in tags
|
|
451
|
-
and tags["copilot-environment"] == env
|
|
452
|
-
and "copilot-application" in tags
|
|
453
|
-
and tags["copilot-application"] == app
|
|
454
|
-
):
|
|
455
|
-
return resource["ResourceARN"]
|
|
456
|
-
|
|
457
|
-
click.secho(
|
|
458
|
-
f"No target group found for application: {app}, environment: {env}, service: {svc}",
|
|
459
|
-
fg="red",
|
|
460
|
-
)
|
|
461
|
-
|
|
462
|
-
return None
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
def create_header_rule(
|
|
466
|
-
lb_client: boto3.client,
|
|
467
|
-
listener_arn: str,
|
|
468
|
-
target_group_arn: str,
|
|
469
|
-
header_name: str,
|
|
470
|
-
values: list,
|
|
471
|
-
rule_name: str,
|
|
472
|
-
priority: int,
|
|
473
|
-
conditions: list,
|
|
474
|
-
):
|
|
475
|
-
# add new condition to existing conditions
|
|
476
|
-
combined_conditions = [
|
|
477
|
-
{
|
|
478
|
-
"Field": "http-header",
|
|
479
|
-
"HttpHeaderConfig": {"HttpHeaderName": header_name, "Values": values},
|
|
480
|
-
}
|
|
481
|
-
] + conditions
|
|
482
|
-
|
|
483
|
-
lb_client.create_rule(
|
|
484
|
-
ListenerArn=listener_arn,
|
|
485
|
-
Priority=priority,
|
|
486
|
-
Conditions=combined_conditions,
|
|
487
|
-
Actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
|
|
488
|
-
Tags=[
|
|
489
|
-
{"Key": "name", "Value": rule_name},
|
|
490
|
-
],
|
|
491
|
-
)
|
|
492
|
-
|
|
493
|
-
click.secho(
|
|
494
|
-
f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\n\nIf request header {header_name} contains one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.",
|
|
495
|
-
fg="green",
|
|
496
|
-
)
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
def normalise_to_cidr(ip: str):
|
|
500
|
-
if "/" in ip:
|
|
501
|
-
return ip
|
|
502
|
-
SINGLE_IPV4_CIDR_PREFIX_LENGTH = "32"
|
|
503
|
-
return f"{ip}/{SINGLE_IPV4_CIDR_PREFIX_LENGTH}"
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
def create_source_ip_rule(
|
|
507
|
-
lb_client: boto3.client,
|
|
508
|
-
listener_arn: str,
|
|
509
|
-
target_group_arn: str,
|
|
510
|
-
values: list,
|
|
511
|
-
rule_name: str,
|
|
512
|
-
priority: int,
|
|
513
|
-
conditions: list,
|
|
514
|
-
):
|
|
515
|
-
# add new condition to existing conditions
|
|
516
|
-
|
|
517
|
-
combined_conditions = [
|
|
518
|
-
{
|
|
519
|
-
"Field": "source-ip",
|
|
520
|
-
"SourceIpConfig": {"Values": [normalise_to_cidr(value) for value in values]},
|
|
521
|
-
}
|
|
522
|
-
] + conditions
|
|
523
|
-
|
|
524
|
-
lb_client.create_rule(
|
|
525
|
-
ListenerArn=listener_arn,
|
|
526
|
-
Priority=priority,
|
|
527
|
-
Conditions=combined_conditions,
|
|
528
|
-
Actions=[{"Type": "forward", "TargetGroupArn": target_group_arn}],
|
|
529
|
-
Tags=[
|
|
530
|
-
{"Key": "name", "Value": rule_name},
|
|
531
|
-
],
|
|
532
|
-
)
|
|
533
|
-
|
|
534
|
-
click.secho(
|
|
535
|
-
f"Creating listener rule {rule_name} for HTTPS Listener with arn {listener_arn}.\n\nIf request source ip matches one of the values {values}, the request will be forwarded to target group with arn {target_group_arn}.",
|
|
536
|
-
fg="green",
|
|
537
|
-
)
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
def get_host_header_conditions(
|
|
541
|
-
lb_client: boto3.client, listener_arn: str, target_group_arn: str
|
|
542
|
-
) -> list:
|
|
543
|
-
rules = lb_client.describe_rules(ListenerArn=listener_arn)["Rules"]
|
|
544
|
-
|
|
545
|
-
# Get current set of forwarding conditions for the target group
|
|
546
|
-
for rule in rules:
|
|
547
|
-
for action in rule["Actions"]:
|
|
548
|
-
if action["Type"] == "forward" and action["TargetGroupArn"] == target_group_arn:
|
|
549
|
-
conditions = rule["Conditions"]
|
|
550
|
-
|
|
551
|
-
# filter to host-header conditions
|
|
552
|
-
conditions = [
|
|
553
|
-
{i: condition[i] for i in condition if i != "Values"}
|
|
554
|
-
for condition in conditions
|
|
555
|
-
if condition["Field"] == "host-header"
|
|
556
|
-
]
|
|
557
|
-
|
|
558
|
-
# remove internal hosts
|
|
559
|
-
conditions[0]["HostHeaderConfig"]["Values"] = [
|
|
560
|
-
v for v in conditions[0]["HostHeaderConfig"]["Values"]
|
|
561
|
-
]
|
|
562
|
-
|
|
563
|
-
return conditions
|