qontract-reconcile 0.10.2.dev51__py3-none-any.whl → 0.10.2.dev53__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qontract_reconcile-0.10.2.dev51.dist-info → qontract_reconcile-0.10.2.dev53.dist-info}/METADATA +1 -1
- {qontract_reconcile-0.10.2.dev51.dist-info → qontract_reconcile-0.10.2.dev53.dist-info}/RECORD +25 -26
- reconcile/aws_cloudwatch_log_retention/integration.py +10 -17
- reconcile/dashdotdb_dora.py +5 -4
- reconcile/gitlab_housekeeping.py +10 -6
- reconcile/terraform_tgw_attachments.py +5 -5
- reconcile/terraform_vpc_peerings.py +1 -1
- reconcile/utils/aggregated_list.py +30 -20
- reconcile/utils/aws_api.py +599 -171
- reconcile/utils/aws_helper.py +7 -7
- reconcile/utils/binary.py +14 -7
- reconcile/utils/config.py +9 -6
- reconcile/utils/defer.py +4 -2
- reconcile/utils/elasticsearch_exceptions.py +7 -4
- reconcile/utils/environ.py +5 -3
- reconcile/utils/exceptions.py +5 -2
- reconcile/utils/git.py +6 -4
- reconcile/utils/gitlab_api.py +103 -82
- reconcile/utils/mr/base.py +6 -3
- reconcile/utils/mr/update_access_report_base.py +2 -2
- reconcile/utils/vcs.py +5 -3
- reconcile/vpc_peerings_validator.py +21 -15
- tools/qontract_cli.py +26 -17
- reconcile/utils/data_structures.py +0 -13
- {qontract_reconcile-0.10.2.dev51.dist-info → qontract_reconcile-0.10.2.dev53.dist-info}/WHEEL +0 -0
- {qontract_reconcile-0.10.2.dev51.dist-info → qontract_reconcile-0.10.2.dev53.dist-info}/entry_points.txt +0 -0
reconcile/utils/aws_api.py
CHANGED
@@ -16,19 +16,24 @@ from typing import (
|
|
16
16
|
TYPE_CHECKING,
|
17
17
|
Any,
|
18
18
|
Literal,
|
19
|
+
Self,
|
20
|
+
cast,
|
21
|
+
overload,
|
19
22
|
)
|
20
23
|
|
21
24
|
import botocore
|
22
25
|
from boto3 import Session
|
26
|
+
from botocore.client import BaseClient
|
23
27
|
from botocore.config import Config
|
24
28
|
from pydantic import BaseModel
|
25
29
|
from sretoolbox.utils import threaded
|
26
30
|
|
27
31
|
import reconcile.utils.aws_helper as awsh
|
28
32
|
import reconcile.utils.lean_terraform_client as terraform
|
29
|
-
from reconcile.utils.secret_reader import SecretReader
|
33
|
+
from reconcile.utils.secret_reader import SecretReader, SecretReaderBase
|
30
34
|
|
31
35
|
if TYPE_CHECKING:
|
36
|
+
from mypy_boto3_dynamodb import DynamoDBClient, DynamoDBServiceResource
|
32
37
|
from mypy_boto3_ec2 import (
|
33
38
|
EC2Client,
|
34
39
|
EC2ServiceResource,
|
@@ -45,12 +50,17 @@ if TYPE_CHECKING:
|
|
45
50
|
VpcEndpointTypeDef,
|
46
51
|
VpcTypeDef,
|
47
52
|
)
|
48
|
-
from
|
53
|
+
from mypy_boto3_ecr import ECRClient
|
54
|
+
from mypy_boto3_elb import ElasticLoadBalancingClient
|
55
|
+
from mypy_boto3_iam import IAMClient, IAMServiceResource
|
49
56
|
from mypy_boto3_iam.type_defs import AccessKeyMetadataTypeDef
|
57
|
+
from mypy_boto3_logs import CloudWatchLogsClient
|
58
|
+
from mypy_boto3_logs.type_defs import LogGroupTypeDef
|
50
59
|
from mypy_boto3_organizations import OrganizationsClient
|
51
60
|
from mypy_boto3_rds import RDSClient
|
52
61
|
from mypy_boto3_rds.type_defs import (
|
53
62
|
DBInstanceMessageTypeDef,
|
63
|
+
DBRecommendationsMessageTypeDef,
|
54
64
|
UpgradeTargetTypeDef,
|
55
65
|
)
|
56
66
|
from mypy_boto3_route53 import Route53Client
|
@@ -59,19 +69,30 @@ if TYPE_CHECKING:
|
|
59
69
|
ResourceRecordSetTypeDef,
|
60
70
|
ResourceRecordTypeDef,
|
61
71
|
)
|
62
|
-
from mypy_boto3_s3 import S3Client
|
72
|
+
from mypy_boto3_s3 import S3Client, S3ServiceResource
|
73
|
+
from mypy_boto3_sqs import SQSClient, SQSServiceResource
|
74
|
+
from mypy_boto3_sts import STSClient
|
75
|
+
from mypy_boto3_support import SupportClient
|
76
|
+
from mypy_boto3_support.type_defs import CaseDetailsTypeDef
|
77
|
+
|
63
78
|
else:
|
64
|
-
|
65
|
-
|
66
|
-
) =
|
67
|
-
|
68
|
-
) =
|
69
|
-
FilterTypeDef
|
70
|
-
) = Route53Client = ResourceRecordSetTypeDef = ResourceRecordTypeDef = (
|
79
|
+
AccessKeyMetadataTypeDef = CaseDetailsTypeDef = CloudWatchLogsClient = (
|
80
|
+
DBInstanceMessageTypeDef
|
81
|
+
) = DBRecommendationsMessageTypeDef = DynamoDBClient = DynamoDBServiceResource = (
|
82
|
+
EC2Client
|
83
|
+
) = EC2ServiceResource = ECRClient = ElasticLoadBalancingClient = FilterTypeDef = (
|
71
84
|
HostedZoneTypeDef
|
72
|
-
) =
|
73
|
-
|
74
|
-
) =
|
85
|
+
) = IAMClient = IAMServiceResource = ImageTypeDef = (
|
86
|
+
LaunchPermissionModificationsTypeDef
|
87
|
+
) = LogGroupTypeDef = OrganizationsClient = RDSClient = ResourceRecordSetTypeDef = (
|
88
|
+
ResourceRecordTypeDef
|
89
|
+
) = Route53Client = RouteTableTypeDef = S3Client = S3ServiceResource = SQSClient = (
|
90
|
+
SQSServiceResource
|
91
|
+
) = STSClient = SubnetTypeDef = SupportClient = TagTypeDef = (
|
92
|
+
TransitGatewayTypeDef
|
93
|
+
) = TransitGatewayVpcAttachmentTypeDef = UpgradeTargetTypeDef = (
|
94
|
+
VpcEndpointTypeDef
|
95
|
+
) = VpcTypeDef = object
|
75
96
|
|
76
97
|
|
77
98
|
class InvalidResourceTypeError(Exception):
|
@@ -82,7 +103,7 @@ class MissingARNError(Exception):
|
|
82
103
|
pass
|
83
104
|
|
84
105
|
|
85
|
-
KeyStatus = Literal["Active"
|
106
|
+
KeyStatus = Literal["Active", "Inactive"]
|
86
107
|
|
87
108
|
GOVCLOUD_PARTITION = "aws-us-gov"
|
88
109
|
|
@@ -92,19 +113,50 @@ class AmiTag(BaseModel):
|
|
92
113
|
value: str
|
93
114
|
|
94
115
|
|
95
|
-
|
116
|
+
SERVICE_NAME = Literal[
|
117
|
+
"dynamodb",
|
118
|
+
"ec2",
|
119
|
+
"ecr",
|
120
|
+
"elb",
|
121
|
+
"iam",
|
122
|
+
"logs",
|
123
|
+
"organizations",
|
124
|
+
"rds",
|
125
|
+
"route53",
|
126
|
+
"s3",
|
127
|
+
"sqs",
|
128
|
+
"sts",
|
129
|
+
"support",
|
130
|
+
]
|
131
|
+
RESOURCE_NAME = Literal[
|
132
|
+
"dynamodb",
|
133
|
+
"ec2",
|
134
|
+
"iam",
|
135
|
+
"s3",
|
136
|
+
"sqs",
|
137
|
+
]
|
138
|
+
RESOURCE_TYPE = Literal[
|
139
|
+
"dynamodb",
|
140
|
+
"rds",
|
141
|
+
"rds_snapshots",
|
142
|
+
"s3",
|
143
|
+
"sqs",
|
144
|
+
]
|
145
|
+
|
146
|
+
|
147
|
+
class AWSApi:
|
96
148
|
"""Wrapper around AWS SDK"""
|
97
149
|
|
98
150
|
def __init__(
|
99
151
|
self,
|
100
|
-
thread_pool_size,
|
101
|
-
accounts,
|
102
|
-
settings=None,
|
103
|
-
secret_reader=None,
|
104
|
-
init_ecr_auth_tokens=False,
|
105
|
-
init_users=True,
|
106
|
-
):
|
107
|
-
self._session_clients = []
|
152
|
+
thread_pool_size: int,
|
153
|
+
accounts: Iterable[awsh.Account],
|
154
|
+
settings: Mapping | None = None,
|
155
|
+
secret_reader: SecretReaderBase | None = None,
|
156
|
+
init_ecr_auth_tokens: bool = False,
|
157
|
+
init_users: bool = True,
|
158
|
+
) -> None:
|
159
|
+
self._session_clients: list[BaseClient] = []
|
108
160
|
self.thread_pool_size = thread_pool_size
|
109
161
|
if secret_reader:
|
110
162
|
self.secret_reader = secret_reader
|
@@ -113,10 +165,14 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
113
165
|
self.init_sessions_and_resources(accounts)
|
114
166
|
if init_ecr_auth_tokens:
|
115
167
|
self.init_ecr_auth_tokens(accounts)
|
116
|
-
if init_users:
|
117
|
-
self.init_users()
|
118
168
|
self._lock = Lock()
|
119
|
-
self.resource_types = [
|
169
|
+
self.resource_types: list[RESOURCE_TYPE] = [
|
170
|
+
"s3",
|
171
|
+
"sqs",
|
172
|
+
"dynamodb",
|
173
|
+
"rds",
|
174
|
+
"rds_snapshots",
|
175
|
+
]
|
120
176
|
|
121
177
|
# store the app-interface accounts in a dictionary indexed by name
|
122
178
|
self.accounts = {acc["name"]: acc for acc in accounts}
|
@@ -125,21 +181,24 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
125
181
|
# https://stackoverflow.com/questions/33672412/python-functools-lru-cache-with-class-methods-release-object
|
126
182
|
# using @lru_cache decorators on methods would lek AWSApi instances
|
127
183
|
# since the cache keeps a reference to self.
|
128
|
-
self._get_assume_role_session = lru_cache()(self._get_assume_role_session)
|
129
|
-
self._get_session_resource = lru_cache()(self._get_session_resource)
|
130
|
-
self.get_account_amis = lru_cache()(self.get_account_amis)
|
131
|
-
self.get_account_vpcs = lru_cache()(self.get_account_vpcs)
|
132
|
-
self.get_session_client = lru_cache()(self.get_session_client)
|
133
|
-
self.get_transit_gateway_vpc_attachments = lru_cache()(
|
184
|
+
self._get_assume_role_session = lru_cache()(self._get_assume_role_session) # type: ignore[method-assign]
|
185
|
+
self._get_session_resource = lru_cache()(self._get_session_resource) # type: ignore[method-assign, assignment]
|
186
|
+
self.get_account_amis = lru_cache()(self.get_account_amis) # type: ignore[method-assign]
|
187
|
+
self.get_account_vpcs = lru_cache()(self.get_account_vpcs) # type: ignore[method-assign]
|
188
|
+
self.get_session_client = lru_cache()(self.get_session_client) # type: ignore[method-assign, assignment]
|
189
|
+
self.get_transit_gateway_vpc_attachments = lru_cache()( # type: ignore[method-assign]
|
134
190
|
self.get_transit_gateway_vpc_attachments
|
135
191
|
)
|
136
|
-
self.get_transit_gateways = lru_cache()(self.get_transit_gateways)
|
137
|
-
self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id)
|
138
|
-
self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables)
|
139
|
-
self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets)
|
140
|
-
self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints)
|
192
|
+
self.get_transit_gateways = lru_cache()(self.get_transit_gateways) # type: ignore[method-assign]
|
193
|
+
self.get_vpc_default_sg_id = lru_cache()(self.get_vpc_default_sg_id) # type: ignore[method-assign]
|
194
|
+
self.get_vpc_route_tables = lru_cache()(self.get_vpc_route_tables) # type: ignore[method-assign]
|
195
|
+
self.get_vpc_subnets = lru_cache()(self.get_vpc_subnets) # type: ignore[method-assign]
|
196
|
+
self._get_vpc_endpoints = lru_cache()(self._get_vpc_endpoints) # type: ignore[method-assign]
|
141
197
|
|
142
|
-
|
198
|
+
if init_users:
|
199
|
+
self.init_users()
|
200
|
+
|
201
|
+
def init_sessions_and_resources(self, accounts: Iterable[awsh.Account]) -> None:
|
143
202
|
results = threaded.run(
|
144
203
|
awsh.get_tf_secrets,
|
145
204
|
accounts,
|
@@ -168,13 +227,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
168
227
|
self.sessions[account_name] = session
|
169
228
|
self.resources[account_name] = {}
|
170
229
|
|
171
|
-
def __enter__(self):
|
230
|
+
def __enter__(self) -> Self:
|
172
231
|
return self
|
173
232
|
|
174
|
-
def __exit__(self, *exc):
|
233
|
+
def __exit__(self, *exc: Any) -> None:
|
175
234
|
self.cleanup()
|
176
235
|
|
177
|
-
def cleanup(self):
|
236
|
+
def cleanup(self) -> None:
|
178
237
|
"""
|
179
238
|
Close all session clients
|
180
239
|
:return:
|
@@ -185,12 +244,129 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
185
244
|
def get_session(self, account: str) -> Session:
|
186
245
|
return self.sessions[account]
|
187
246
|
|
188
|
-
|
247
|
+
@overload
|
248
|
+
def get_session_client(
|
249
|
+
self,
|
250
|
+
session: Session,
|
251
|
+
service_name: Literal["ec2"],
|
252
|
+
region_name: str | None = None,
|
253
|
+
) -> EC2Client: ...
|
254
|
+
|
255
|
+
@overload
|
256
|
+
def get_session_client(
|
257
|
+
self,
|
258
|
+
session: Session,
|
259
|
+
service_name: Literal["elb"],
|
260
|
+
region_name: str | None = None,
|
261
|
+
) -> ElasticLoadBalancingClient: ...
|
262
|
+
|
263
|
+
@overload
|
189
264
|
def get_session_client(
|
190
265
|
self,
|
191
266
|
session: Session,
|
192
|
-
service_name,
|
267
|
+
service_name: Literal["route53"],
|
193
268
|
region_name: str | None = None,
|
269
|
+
) -> Route53Client: ...
|
270
|
+
|
271
|
+
@overload
|
272
|
+
def get_session_client(
|
273
|
+
self,
|
274
|
+
session: Session,
|
275
|
+
service_name: Literal["rds"],
|
276
|
+
region_name: str | None = None,
|
277
|
+
) -> RDSClient: ...
|
278
|
+
|
279
|
+
@overload
|
280
|
+
def get_session_client(
|
281
|
+
self,
|
282
|
+
session: Session,
|
283
|
+
service_name: Literal["logs"],
|
284
|
+
region_name: str | None = None,
|
285
|
+
) -> CloudWatchLogsClient: ...
|
286
|
+
|
287
|
+
@overload
|
288
|
+
def get_session_client(
|
289
|
+
self,
|
290
|
+
session: Session,
|
291
|
+
service_name: Literal["organizations"],
|
292
|
+
region_name: str | None = None,
|
293
|
+
) -> OrganizationsClient: ...
|
294
|
+
|
295
|
+
@overload
|
296
|
+
def get_session_client(
|
297
|
+
self,
|
298
|
+
session: Session,
|
299
|
+
service_name: Literal["s3"],
|
300
|
+
region_name: str | None = None,
|
301
|
+
) -> S3Client: ...
|
302
|
+
|
303
|
+
@overload
|
304
|
+
def get_session_client(
|
305
|
+
self,
|
306
|
+
session: Session,
|
307
|
+
service_name: Literal["iam"],
|
308
|
+
region_name: str | None = None,
|
309
|
+
) -> IAMClient: ...
|
310
|
+
|
311
|
+
@overload
|
312
|
+
def get_session_client(
|
313
|
+
self,
|
314
|
+
session: Session,
|
315
|
+
service_name: Literal["sqs"],
|
316
|
+
region_name: str | None = None,
|
317
|
+
) -> SQSClient: ...
|
318
|
+
|
319
|
+
@overload
|
320
|
+
def get_session_client(
|
321
|
+
self,
|
322
|
+
session: Session,
|
323
|
+
service_name: Literal["dynamodb"],
|
324
|
+
region_name: str | None = None,
|
325
|
+
) -> DynamoDBClient: ...
|
326
|
+
|
327
|
+
@overload
|
328
|
+
def get_session_client(
|
329
|
+
self,
|
330
|
+
session: Session,
|
331
|
+
service_name: Literal["ecr"],
|
332
|
+
region_name: str | None = None,
|
333
|
+
) -> ECRClient: ...
|
334
|
+
|
335
|
+
@overload
|
336
|
+
def get_session_client(
|
337
|
+
self,
|
338
|
+
session: Session,
|
339
|
+
service_name: Literal["support"],
|
340
|
+
region_name: str | None = None,
|
341
|
+
) -> SupportClient: ...
|
342
|
+
|
343
|
+
@overload
|
344
|
+
def get_session_client(
|
345
|
+
self,
|
346
|
+
session: Session,
|
347
|
+
service_name: Literal["sts"],
|
348
|
+
region_name: str | None = None,
|
349
|
+
) -> STSClient: ...
|
350
|
+
|
351
|
+
def get_session_client(
|
352
|
+
self,
|
353
|
+
session: Session,
|
354
|
+
service_name: SERVICE_NAME,
|
355
|
+
region_name: str | None = None,
|
356
|
+
) -> (
|
357
|
+
CloudWatchLogsClient
|
358
|
+
| DynamoDBClient
|
359
|
+
| EC2Client
|
360
|
+
| ECRClient
|
361
|
+
| ElasticLoadBalancingClient
|
362
|
+
| IAMClient
|
363
|
+
| OrganizationsClient
|
364
|
+
| RDSClient
|
365
|
+
| Route53Client
|
366
|
+
| S3Client
|
367
|
+
| SQSClient
|
368
|
+
| STSClient
|
369
|
+
| SupportClient
|
194
370
|
):
|
195
371
|
region = region_name or session.region_name
|
196
372
|
client = session.client(
|
@@ -201,10 +377,55 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
201
377
|
self._session_clients.append(client)
|
202
378
|
return client
|
203
379
|
|
380
|
+
@overload
|
381
|
+
@staticmethod
|
382
|
+
def _get_session_resource(
|
383
|
+
session: Session,
|
384
|
+
service_name: Literal["dynamodb"],
|
385
|
+
region_name: str | None = None,
|
386
|
+
) -> DynamoDBServiceResource: ...
|
387
|
+
|
388
|
+
@overload
|
389
|
+
@staticmethod
|
390
|
+
def _get_session_resource(
|
391
|
+
session: Session,
|
392
|
+
service_name: Literal["ec2"],
|
393
|
+
region_name: str | None = None,
|
394
|
+
) -> EC2ServiceResource: ...
|
395
|
+
|
396
|
+
@overload
|
397
|
+
@staticmethod
|
398
|
+
def _get_session_resource(
|
399
|
+
session: Session,
|
400
|
+
service_name: Literal["iam"],
|
401
|
+
region_name: str | None = None,
|
402
|
+
) -> IAMServiceResource: ...
|
403
|
+
|
404
|
+
@overload
|
405
|
+
@staticmethod
|
406
|
+
def _get_session_resource(
|
407
|
+
session: Session,
|
408
|
+
service_name: Literal["s3"],
|
409
|
+
region_name: str | None = None,
|
410
|
+
) -> S3ServiceResource: ...
|
411
|
+
|
412
|
+
@overload
|
204
413
|
@staticmethod
|
205
|
-
# pylint: disable=method-hidden
|
206
414
|
def _get_session_resource(
|
207
|
-
session: Session,
|
415
|
+
session: Session,
|
416
|
+
service_name: Literal["sqs"],
|
417
|
+
region_name: str | None = None,
|
418
|
+
) -> SQSServiceResource: ...
|
419
|
+
|
420
|
+
@staticmethod
|
421
|
+
def _get_session_resource(
|
422
|
+
session: Session, service_name: RESOURCE_NAME, region_name: str | None = None
|
423
|
+
) -> (
|
424
|
+
DynamoDBServiceResource
|
425
|
+
| EC2ServiceResource
|
426
|
+
| IAMServiceResource
|
427
|
+
| S3ServiceResource
|
428
|
+
| SQSServiceResource
|
208
429
|
):
|
209
430
|
region = region_name or session.region_name
|
210
431
|
return session.resource(service_name, region_name=region)
|
@@ -235,7 +456,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
235
456
|
|
236
457
|
def _account_cloudwatch_client(
|
237
458
|
self, account_name: str, region_name: str | None = None
|
238
|
-
):
|
459
|
+
) -> CloudWatchLogsClient:
|
239
460
|
session = self.get_session(account_name)
|
240
461
|
return self.get_session_client(session, "logs", region_name)
|
241
462
|
|
@@ -249,9 +470,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
249
470
|
self, account_name: str, region_name: str | None = None
|
250
471
|
) -> S3Client:
|
251
472
|
session = self.get_session(account_name)
|
252
|
-
return self.get_session_client(session, "s3", region_name)
|
473
|
+
return cast(S3Client, self.get_session_client(session, "s3", region_name))
|
253
474
|
|
254
|
-
def init_users(self):
|
475
|
+
def init_users(self) -> None:
|
255
476
|
self.users = {}
|
256
477
|
for account, s in self.sessions.items():
|
257
478
|
iam = self.get_session_client(s, "iam")
|
@@ -259,26 +480,29 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
259
480
|
users = [u["UserName"] for u in users]
|
260
481
|
self.users[account] = users
|
261
482
|
|
262
|
-
def map_resources(self):
|
483
|
+
def map_resources(self) -> None:
|
263
484
|
threaded.run(self.map_resource, self.resource_types, self.thread_pool_size)
|
264
485
|
|
265
|
-
def map_resource(self, resource_type):
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
486
|
+
def map_resource(self, resource_type: str) -> None:
|
487
|
+
match resource_type:
|
488
|
+
case "s3":
|
489
|
+
self.map_s3_resources()
|
490
|
+
case "sqs":
|
491
|
+
self.map_sqs_resources()
|
492
|
+
case "dynamodb":
|
493
|
+
self.map_dynamodb_resources()
|
494
|
+
case "rds":
|
495
|
+
self.map_rds_resources()
|
496
|
+
case "rds_snapshots":
|
497
|
+
self.map_rds_snapshots()
|
498
|
+
case "route53":
|
499
|
+
self.map_route53_resources()
|
500
|
+
case "ecr":
|
501
|
+
self.map_ecr_resources()
|
502
|
+
case _:
|
503
|
+
raise InvalidResourceTypeError(resource_type)
|
504
|
+
|
505
|
+
def map_s3_resources(self) -> None:
|
282
506
|
for account, s in self.sessions.items():
|
283
507
|
s3 = self.get_session_client(s, "s3")
|
284
508
|
buckets_list = s3.list_buckets()
|
@@ -292,7 +516,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
292
516
|
)
|
293
517
|
self.set_resouces(account, "s3_no_owner", unfiltered_buckets)
|
294
518
|
|
295
|
-
def map_sqs_resources(self):
|
519
|
+
def map_sqs_resources(self) -> None:
|
296
520
|
for account, s in self.sessions.items():
|
297
521
|
sqs = self.get_session_client(s, "sqs")
|
298
522
|
queues_list = sqs.list_queues()
|
@@ -306,7 +530,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
306
530
|
)
|
307
531
|
self.set_resouces(account, "sqs_no_owner", unfiltered_queues)
|
308
532
|
|
309
|
-
def map_dynamodb_resources(self):
|
533
|
+
def map_dynamodb_resources(self) -> None:
|
310
534
|
for account, s in self.sessions.items():
|
311
535
|
dynamodb = self.get_session_client(s, "dynamodb")
|
312
536
|
tables = self.paginate(dynamodb, "list_tables", "TableNames")
|
@@ -317,7 +541,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
317
541
|
)
|
318
542
|
self.set_resouces(account, "dynamodb_no_owner", unfiltered_tables)
|
319
543
|
|
320
|
-
def map_rds_resources(self):
|
544
|
+
def map_rds_resources(self) -> None:
|
321
545
|
for account, s in self.sessions.items():
|
322
546
|
rds = self.get_session_client(s, "rds")
|
323
547
|
results = self.paginate(rds, "describe_db_instances", "DBInstances")
|
@@ -331,7 +555,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
331
555
|
)
|
332
556
|
self.set_resouces(account, "rds_no_owner", unfiltered_instances)
|
333
557
|
|
334
|
-
def map_rds_snapshots(self):
|
558
|
+
def map_rds_snapshots(self) -> None:
|
335
559
|
self.wait_for_resource("rds")
|
336
560
|
for account, s in self.sessions.items():
|
337
561
|
rds = self.get_session_client(s, "rds")
|
@@ -348,7 +572,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
348
572
|
)
|
349
573
|
self.set_resouces(account, "rds_snapshots_no_owner", unfiltered_snapshots)
|
350
574
|
|
351
|
-
def map_route53_resources(self):
|
575
|
+
def map_route53_resources(self) -> None:
|
352
576
|
for account, s in self.sessions.items():
|
353
577
|
client = self.get_session_client(s, "route53")
|
354
578
|
results = self.paginate(client, "list_hosted_zones", "HostedZones")
|
@@ -363,7 +587,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
363
587
|
zone["records"] = results
|
364
588
|
self.set_resouces(account, "route53", zones)
|
365
589
|
|
366
|
-
def map_ecr_resources(self):
|
590
|
+
def map_ecr_resources(self) -> None:
|
367
591
|
for account, s in self.sessions.items():
|
368
592
|
client = self.get_session_client(s, "ecr")
|
369
593
|
repositories = self.paginate(
|
@@ -372,7 +596,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
372
596
|
self.set_resouces(account, "ecr", repositories)
|
373
597
|
|
374
598
|
@staticmethod
|
375
|
-
def paginate(
|
599
|
+
def paginate(
|
600
|
+
client: BaseClient, method: str, key: str, params: Mapping | None = None
|
601
|
+
) -> Iterable:
|
376
602
|
"""paginate returns an aggregated list of the specified key
|
377
603
|
from all pages returned by executing the client's specified method."""
|
378
604
|
if params is None:
|
@@ -384,7 +610,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
384
610
|
for values in page.get(key, [])
|
385
611
|
]
|
386
612
|
|
387
|
-
def wait_for_resource(self, resource):
|
613
|
+
def wait_for_resource(self, resource: str) -> None:
|
388
614
|
"""wait_for_resource waits until the specified resource type
|
389
615
|
is ready for all accounts.
|
390
616
|
When we have more resource types then threads,
|
@@ -398,14 +624,16 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
398
624
|
if wait:
|
399
625
|
time.sleep(2)
|
400
626
|
|
401
|
-
def set_resouces(self, account, key, value):
|
627
|
+
def set_resouces(self, account: str, key: str, value: Any) -> None:
|
402
628
|
with self._lock:
|
403
629
|
self.resources[account][key] = value
|
404
630
|
|
405
|
-
def get_resources_without_owner(
|
631
|
+
def get_resources_without_owner(
|
632
|
+
self, account: str, resources: Iterable[str]
|
633
|
+
) -> list[str]:
|
406
634
|
return [r for r in resources if not self.has_owner(account, r)]
|
407
635
|
|
408
|
-
def has_owner(self, account, resource):
|
636
|
+
def has_owner(self, account: str, resource: str) -> bool:
|
409
637
|
has_owner = False
|
410
638
|
for u in self.users[account]:
|
411
639
|
if resource.lower().startswith(u.lower()):
|
@@ -417,20 +645,24 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
417
645
|
break
|
418
646
|
return has_owner
|
419
647
|
|
420
|
-
def custom_s3_filter(
|
648
|
+
def custom_s3_filter(
|
649
|
+
self, account: str, s3: S3Client, buckets: Iterable[str]
|
650
|
+
) -> list[str]:
|
421
651
|
type = "s3 bucket"
|
422
652
|
unfiltered_buckets = []
|
423
653
|
for b in buckets:
|
424
654
|
try:
|
425
655
|
tags = s3.get_bucket_tagging(Bucket=b)
|
426
656
|
except botocore.exceptions.ClientError:
|
427
|
-
tags = {}
|
657
|
+
tags = {} # type: ignore
|
428
658
|
if not self.should_filter(account, type, b, tags, "TagSet"):
|
429
659
|
unfiltered_buckets.append(b)
|
430
660
|
|
431
661
|
return unfiltered_buckets
|
432
662
|
|
433
|
-
def custom_sqs_filter(
|
663
|
+
def custom_sqs_filter(
|
664
|
+
self, account: str, sqs: SQSClient, queues: Iterable[str]
|
665
|
+
) -> list[str]:
|
434
666
|
type = "sqs queue"
|
435
667
|
unfiltered_queues = []
|
436
668
|
for q in queues:
|
@@ -440,7 +672,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
440
672
|
|
441
673
|
return unfiltered_queues
|
442
674
|
|
443
|
-
def custom_dynamodb_filter(
|
675
|
+
def custom_dynamodb_filter(
|
676
|
+
self,
|
677
|
+
account: str,
|
678
|
+
session: Session,
|
679
|
+
dynamodb: DynamoDBClient,
|
680
|
+
tables: Iterable[str],
|
681
|
+
) -> list[str]:
|
444
682
|
type = "dynamodb table"
|
445
683
|
dynamodb_resource = self._get_session_resource(session, "dynamodb")
|
446
684
|
unfiltered_tables = []
|
@@ -452,7 +690,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
452
690
|
|
453
691
|
return unfiltered_tables
|
454
692
|
|
455
|
-
def custom_rds_filter(
|
693
|
+
def custom_rds_filter(
|
694
|
+
self, account: str, rds: RDSClient, instances: Iterable[str]
|
695
|
+
) -> list[str]:
|
456
696
|
type = "rds instance"
|
457
697
|
unfiltered_instances = []
|
458
698
|
for i in instances:
|
@@ -464,7 +704,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
464
704
|
|
465
705
|
return unfiltered_instances
|
466
706
|
|
467
|
-
def custom_rds_snapshot_filter(
|
707
|
+
def custom_rds_snapshot_filter(
|
708
|
+
self, account: str, rds: RDSClient, snapshots: Iterable[str]
|
709
|
+
) -> list[str]:
|
468
710
|
type = "rds snapshots"
|
469
711
|
unfiltered_snapshots = []
|
470
712
|
for s in snapshots:
|
@@ -477,8 +719,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
477
719
|
return unfiltered_snapshots
|
478
720
|
|
479
721
|
def should_filter(
|
480
|
-
self,
|
481
|
-
|
722
|
+
self,
|
723
|
+
account: str,
|
724
|
+
resource_type: str,
|
725
|
+
resource_name: str,
|
726
|
+
resource_tags: Mapping,
|
727
|
+
tags_key: str,
|
728
|
+
) -> bool:
|
482
729
|
if self.resource_has_special_name(account, resource_type, resource_name):
|
483
730
|
return True
|
484
731
|
if tags_key in resource_tags:
|
@@ -491,7 +738,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
491
738
|
return False
|
492
739
|
|
493
740
|
@staticmethod
|
494
|
-
def resource_has_special_name(account, type, resource):
|
741
|
+
def resource_has_special_name(account: str, type: str, resource: str) -> bool:
|
495
742
|
skip_msg = f"[{account}] skipping {type} " + "({} related) {}"
|
496
743
|
|
497
744
|
ignore_names = {
|
@@ -508,7 +755,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
508
755
|
|
509
756
|
return False
|
510
757
|
|
511
|
-
def resource_has_special_tags(
|
758
|
+
def resource_has_special_tags(
|
759
|
+
self, account: str, type: str, resource: str, tags: Mapping | list[Mapping]
|
760
|
+
) -> bool:
|
512
761
|
skip_msg = f"[{account}] skipping {type} " + "({}={}) {}"
|
513
762
|
|
514
763
|
ignore_tags = {
|
@@ -529,7 +778,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
529
778
|
return False
|
530
779
|
|
531
780
|
@staticmethod
|
532
|
-
def get_tag_value(tags, tag):
|
781
|
+
def get_tag_value(tags: Mapping | list[Mapping], tag: str) -> str:
|
533
782
|
if isinstance(tags, dict):
|
534
783
|
return tags.get(tag, "")
|
535
784
|
if isinstance(tags, list):
|
@@ -539,7 +788,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
539
788
|
|
540
789
|
return ""
|
541
790
|
|
542
|
-
def delete_resources_without_owner(self, dry_run):
|
791
|
+
def delete_resources_without_owner(self, dry_run: bool) -> None:
|
543
792
|
for account, s in self.sessions.items():
|
544
793
|
for rt in self.resource_types:
|
545
794
|
for r in self.resources[account].get(rt + "_no_owner", []):
|
@@ -547,42 +796,50 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
547
796
|
if not dry_run:
|
548
797
|
self.delete_resource(s, rt, r)
|
549
798
|
|
550
|
-
def delete_resource(
|
551
|
-
|
552
|
-
|
553
|
-
|
554
|
-
|
555
|
-
|
556
|
-
|
557
|
-
|
558
|
-
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
566
|
-
|
567
|
-
|
799
|
+
def delete_resource(
|
800
|
+
self, session: Session, resource_type: RESOURCE_TYPE, resource_name: str
|
801
|
+
) -> None:
|
802
|
+
match resource_type:
|
803
|
+
case "s3":
|
804
|
+
self.delete_bucket(
|
805
|
+
self._get_session_resource(session, resource_type), resource_name
|
806
|
+
)
|
807
|
+
case "sqs":
|
808
|
+
self.delete_queue(
|
809
|
+
self.get_session_client(session, resource_type), resource_name
|
810
|
+
)
|
811
|
+
case "dynamodb":
|
812
|
+
self.delete_table(
|
813
|
+
self._get_session_resource(session, resource_type), resource_name
|
814
|
+
)
|
815
|
+
case "rds":
|
816
|
+
self.delete_instance(
|
817
|
+
self.get_session_client(session, resource_type), resource_name
|
818
|
+
)
|
819
|
+
case "rds_snapshots":
|
820
|
+
self.delete_snapshot(
|
821
|
+
self.get_session_client(session, "rds"), resource_name
|
822
|
+
)
|
823
|
+
case _:
|
824
|
+
raise InvalidResourceTypeError(resource_type)
|
568
825
|
|
569
826
|
@staticmethod
|
570
|
-
def delete_bucket(s3, bucket_name):
|
827
|
+
def delete_bucket(s3: S3ServiceResource, bucket_name: str) -> None:
|
571
828
|
bucket = s3.Bucket(bucket_name)
|
572
829
|
bucket.object_versions.delete()
|
573
830
|
bucket.delete()
|
574
831
|
|
575
832
|
@staticmethod
|
576
|
-
def delete_queue(sqs, queue_url):
|
833
|
+
def delete_queue(sqs: SQSClient, queue_url: str) -> None:
|
577
834
|
sqs.delete_queue(QueueUrl=queue_url)
|
578
835
|
|
579
836
|
@staticmethod
|
580
|
-
def delete_table(dynamodb, table_name):
|
837
|
+
def delete_table(dynamodb: DynamoDBServiceResource, table_name: str) -> None:
|
581
838
|
table = dynamodb.Table(table_name)
|
582
839
|
table.delete()
|
583
840
|
|
584
841
|
@staticmethod
|
585
|
-
def delete_instance(rds, instance_name):
|
842
|
+
def delete_instance(rds: RDSClient, instance_name: str) -> None:
|
586
843
|
rds.delete_db_instance(
|
587
844
|
DBInstanceIdentifier=instance_name,
|
588
845
|
SkipFinalSnapshot=True,
|
@@ -590,11 +847,11 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
590
847
|
)
|
591
848
|
|
592
849
|
@staticmethod
|
593
|
-
def delete_snapshot(rds, snapshot_identifier):
|
850
|
+
def delete_snapshot(rds: RDSClient, snapshot_identifier: str) -> None:
|
594
851
|
rds.delete_db_snapshot(DBSnapshotIdentifier=snapshot_identifier)
|
595
852
|
|
596
853
|
@staticmethod
|
597
|
-
def determine_key_type(iam, user):
|
854
|
+
def determine_key_type(iam: IAMClient, user: str) -> str:
|
598
855
|
tags = iam.list_user_tags(UserName=user)["Tags"]
|
599
856
|
managed_by_integration_tag = [
|
600
857
|
t["Value"] for t in tags if t["Key"] == "managed_by_integration"
|
@@ -621,8 +878,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
621
878
|
raise InvalidResourceTypeError(huh)
|
622
879
|
|
623
880
|
def delete_keys(
|
624
|
-
self,
|
625
|
-
|
881
|
+
self,
|
882
|
+
dry_run: bool,
|
883
|
+
keys_to_delete: Mapping,
|
884
|
+
working_dirs: Mapping[str, str],
|
885
|
+
disable_service_account_keys: bool,
|
886
|
+
) -> tuple[bool, bool]:
|
626
887
|
error = False
|
627
888
|
service_account_recycle_complete = True
|
628
889
|
users_keys = self.get_users_keys()
|
@@ -637,11 +898,13 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
637
898
|
]
|
638
899
|
if not user_and_user_keys:
|
639
900
|
continue
|
640
|
-
|
641
|
-
|
642
|
-
|
643
|
-
|
644
|
-
|
901
|
+
|
902
|
+
if len(user_and_user_keys) > 1:
|
903
|
+
raise RuntimeError(
|
904
|
+
f"key {key} returned multiple users: {user_and_user_keys}"
|
905
|
+
)
|
906
|
+
user = user_and_user_keys[0][0]
|
907
|
+
user_keys = user_and_user_keys[0][1]
|
645
908
|
key_type = self.determine_key_type(iam, user)
|
646
909
|
key_status = self.get_user_key_status(iam, user, key)
|
647
910
|
if key_type == "unmanaged" and key_status == "Active":
|
@@ -702,7 +965,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
702
965
|
|
703
966
|
return error, service_account_recycle_complete
|
704
967
|
|
705
|
-
def get_users_keys(self):
|
968
|
+
def get_users_keys(self) -> dict:
|
706
969
|
users_keys = {}
|
707
970
|
for account, s in self.sessions.items():
|
708
971
|
iam = self.get_session_client(s, "iam")
|
@@ -712,12 +975,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
712
975
|
|
713
976
|
return users_keys
|
714
977
|
|
715
|
-
def reset_password(self, account, user_name):
|
978
|
+
def reset_password(self, account: str, user_name: str) -> None:
|
716
979
|
s = self.sessions[account]
|
717
980
|
iam = self.get_session_client(s, "iam")
|
718
981
|
iam.delete_login_profile(UserName=user_name)
|
719
982
|
|
720
|
-
def reset_mfa(self, account, user_name):
|
983
|
+
def reset_mfa(self, account: str, user_name: str) -> None:
|
721
984
|
s = self.sessions[account]
|
722
985
|
iam = self.get_session_client(s, "iam")
|
723
986
|
mfa_devices = iam.list_mfa_devices(UserName=user_name)["MFADevices"]
|
@@ -741,7 +1004,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
741
1004
|
key_list = self._get_user_key_list(iam, user)
|
742
1005
|
return next(k["Status"] for k in key_list if k["AccessKeyId"] == key)
|
743
1006
|
|
744
|
-
def get_support_cases(self):
|
1007
|
+
def get_support_cases(self) -> dict[str, list[CaseDetailsTypeDef]]:
|
745
1008
|
all_support_cases = {}
|
746
1009
|
for account, s in self.sessions.items():
|
747
1010
|
if not self.accounts[account].get("premiumSupport"):
|
@@ -826,9 +1089,8 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
826
1089
|
return (account["name"], account.get("assume_role"), account["assume_region"])
|
827
1090
|
|
828
1091
|
@staticmethod
|
829
|
-
# pylint: disable=method-hidden
|
830
1092
|
def _get_assume_role_session(
|
831
|
-
sts, account_name: str, assume_role: str, assume_region: str
|
1093
|
+
sts: STSClient, account_name: str, assume_role: str, assume_region: str
|
832
1094
|
) -> Session:
|
833
1095
|
"""
|
834
1096
|
Returns a session for a supplied role to assume:
|
@@ -859,13 +1121,144 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
859
1121
|
|
860
1122
|
return assumed_session
|
861
1123
|
|
1124
|
+
@overload
|
862
1125
|
def _get_assumed_role_client(
|
863
1126
|
self,
|
864
1127
|
account_name: str,
|
865
1128
|
assume_role: str | None,
|
866
1129
|
assume_region: str,
|
867
|
-
client_type
|
868
|
-
) ->
|
1130
|
+
client_type: Literal["logs"],
|
1131
|
+
) -> CloudWatchLogsClient: ...
|
1132
|
+
|
1133
|
+
@overload
|
1134
|
+
def _get_assumed_role_client(
|
1135
|
+
self,
|
1136
|
+
account_name: str,
|
1137
|
+
assume_role: str | None,
|
1138
|
+
assume_region: str,
|
1139
|
+
client_type: Literal["dynamodb"],
|
1140
|
+
) -> DynamoDBClient: ...
|
1141
|
+
|
1142
|
+
@overload
|
1143
|
+
def _get_assumed_role_client(
|
1144
|
+
self,
|
1145
|
+
account_name: str,
|
1146
|
+
assume_role: str | None,
|
1147
|
+
assume_region: str,
|
1148
|
+
client_type: Literal["ec2"],
|
1149
|
+
) -> EC2Client: ...
|
1150
|
+
|
1151
|
+
@overload
|
1152
|
+
def _get_assumed_role_client(
|
1153
|
+
self,
|
1154
|
+
account_name: str,
|
1155
|
+
assume_role: str | None,
|
1156
|
+
assume_region: str,
|
1157
|
+
client_type: Literal["ecr"],
|
1158
|
+
) -> ECRClient: ...
|
1159
|
+
|
1160
|
+
@overload
|
1161
|
+
def _get_assumed_role_client(
|
1162
|
+
self,
|
1163
|
+
account_name: str,
|
1164
|
+
assume_role: str | None,
|
1165
|
+
assume_region: str,
|
1166
|
+
client_type: Literal["elb"],
|
1167
|
+
) -> ElasticLoadBalancingClient: ...
|
1168
|
+
|
1169
|
+
@overload
|
1170
|
+
def _get_assumed_role_client(
|
1171
|
+
self,
|
1172
|
+
account_name: str,
|
1173
|
+
assume_role: str | None,
|
1174
|
+
assume_region: str,
|
1175
|
+
client_type: Literal["iam"],
|
1176
|
+
) -> IAMClient: ...
|
1177
|
+
|
1178
|
+
@overload
|
1179
|
+
def _get_assumed_role_client(
|
1180
|
+
self,
|
1181
|
+
account_name: str,
|
1182
|
+
assume_role: str | None,
|
1183
|
+
assume_region: str,
|
1184
|
+
client_type: Literal["organizations"],
|
1185
|
+
) -> OrganizationsClient: ...
|
1186
|
+
|
1187
|
+
@overload
|
1188
|
+
def _get_assumed_role_client(
|
1189
|
+
self,
|
1190
|
+
account_name: str,
|
1191
|
+
assume_role: str | None,
|
1192
|
+
assume_region: str,
|
1193
|
+
client_type: Literal["rds"],
|
1194
|
+
) -> RDSClient: ...
|
1195
|
+
|
1196
|
+
@overload
|
1197
|
+
def _get_assumed_role_client(
|
1198
|
+
self,
|
1199
|
+
account_name: str,
|
1200
|
+
assume_role: str | None,
|
1201
|
+
assume_region: str,
|
1202
|
+
client_type: Literal["route53"],
|
1203
|
+
) -> Route53Client: ...
|
1204
|
+
|
1205
|
+
@overload
|
1206
|
+
def _get_assumed_role_client(
|
1207
|
+
self,
|
1208
|
+
account_name: str,
|
1209
|
+
assume_role: str | None,
|
1210
|
+
assume_region: str,
|
1211
|
+
client_type: Literal["s3"],
|
1212
|
+
) -> S3Client: ...
|
1213
|
+
|
1214
|
+
@overload
|
1215
|
+
def _get_assumed_role_client(
|
1216
|
+
self,
|
1217
|
+
account_name: str,
|
1218
|
+
assume_role: str | None,
|
1219
|
+
assume_region: str,
|
1220
|
+
client_type: Literal["sqs"],
|
1221
|
+
) -> SQSClient: ...
|
1222
|
+
|
1223
|
+
@overload
|
1224
|
+
def _get_assumed_role_client(
|
1225
|
+
self,
|
1226
|
+
account_name: str,
|
1227
|
+
assume_role: str | None,
|
1228
|
+
assume_region: str,
|
1229
|
+
client_type: Literal["sts"],
|
1230
|
+
) -> STSClient: ...
|
1231
|
+
|
1232
|
+
@overload
|
1233
|
+
def _get_assumed_role_client(
|
1234
|
+
self,
|
1235
|
+
account_name: str,
|
1236
|
+
assume_role: str | None,
|
1237
|
+
assume_region: str,
|
1238
|
+
client_type: Literal["support"],
|
1239
|
+
) -> SupportClient: ...
|
1240
|
+
|
1241
|
+
def _get_assumed_role_client(
|
1242
|
+
self,
|
1243
|
+
account_name: str,
|
1244
|
+
assume_role: str | None,
|
1245
|
+
assume_region: str,
|
1246
|
+
client_type: SERVICE_NAME = "ec2",
|
1247
|
+
) -> (
|
1248
|
+
CloudWatchLogsClient
|
1249
|
+
| DynamoDBClient
|
1250
|
+
| EC2Client
|
1251
|
+
| ECRClient
|
1252
|
+
| ElasticLoadBalancingClient
|
1253
|
+
| IAMClient
|
1254
|
+
| OrganizationsClient
|
1255
|
+
| RDSClient
|
1256
|
+
| Route53Client
|
1257
|
+
| S3Client
|
1258
|
+
| SQSClient
|
1259
|
+
| STSClient
|
1260
|
+
| SupportClient
|
1261
|
+
):
|
869
1262
|
session = self.get_session(account_name)
|
870
1263
|
if not assume_role:
|
871
1264
|
return self.get_session_client(
|
@@ -878,13 +1271,11 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
878
1271
|
return self.get_session_client(assumed_session, client_type)
|
879
1272
|
|
880
1273
|
@staticmethod
|
881
|
-
# pylint: disable=method-hidden
|
882
1274
|
def get_account_vpcs(ec2: EC2Client) -> list[VpcTypeDef]:
|
883
1275
|
vpcs = ec2.describe_vpcs()
|
884
1276
|
return vpcs.get("Vpcs", [])
|
885
1277
|
|
886
1278
|
@staticmethod
|
887
|
-
# pylint: disable=method-hidden
|
888
1279
|
def get_account_amis(ec2: EC2Client, owner: str) -> list[ImageTypeDef]:
|
889
1280
|
amis = ec2.describe_images(Owners=[owner])
|
890
1281
|
return amis.get("Images", [])
|
@@ -904,7 +1295,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
904
1295
|
return res
|
905
1296
|
|
906
1297
|
@staticmethod
|
907
|
-
# pylint: disable=method-hidden
|
908
1298
|
def get_vpc_route_tables(vpc_id: str, ec2: EC2Client) -> list[RouteTableTypeDef]:
|
909
1299
|
rts = ec2.describe_route_tables(
|
910
1300
|
Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
|
@@ -912,14 +1302,17 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
912
1302
|
return rts.get("RouteTables", [])
|
913
1303
|
|
914
1304
|
@staticmethod
|
915
|
-
# pylint: disable=method-hidden
|
916
1305
|
def get_vpc_subnets(vpc_id: str, ec2: EC2Client) -> list[SubnetTypeDef]:
|
917
1306
|
subnets = ec2.describe_subnets(Filters=[{"Name": "vpc-id", "Values": [vpc_id]}])
|
918
1307
|
return subnets.get("Subnets", [])
|
919
1308
|
|
920
1309
|
def get_cluster_vpc_details(
|
921
|
-
self,
|
922
|
-
|
1310
|
+
self,
|
1311
|
+
account: awsh.Account,
|
1312
|
+
route_tables: bool = False,
|
1313
|
+
subnets: bool = False,
|
1314
|
+
hcp_vpc_endpoint_sg: bool = False,
|
1315
|
+
) -> tuple[str | None, list[str] | None, list[dict[str, str]] | None, str | None]:
|
923
1316
|
"""
|
924
1317
|
Returns a cluster VPC details:
|
925
1318
|
- VPC ID
|
@@ -935,7 +1328,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
935
1328
|
use to find the matching VPC
|
936
1329
|
"""
|
937
1330
|
assume_role_data = self._get_account_assume_data(account)
|
938
|
-
assumed_ec2 = self._get_assumed_role_client(
|
1331
|
+
assumed_ec2 = self._get_assumed_role_client(
|
1332
|
+
account_name=assume_role_data[0],
|
1333
|
+
assume_role=assume_role_data[1],
|
1334
|
+
assume_region=assume_role_data[2],
|
1335
|
+
client_type="ec2",
|
1336
|
+
)
|
939
1337
|
vpcs = self.get_account_vpcs(assumed_ec2)
|
940
1338
|
vpc_id = None
|
941
1339
|
for vpc in vpcs:
|
@@ -963,7 +1361,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
963
1361
|
|
964
1362
|
return vpc_id, route_table_ids, subnets_id_az, api_security_group_id
|
965
1363
|
|
966
|
-
def _get_api_security_group_id(
|
1364
|
+
def _get_api_security_group_id(
|
1365
|
+
self, assumed_ec2: EC2Client, vpc_id: str
|
1366
|
+
) -> str | None:
|
967
1367
|
endpoints = AWSApi._get_vpc_endpoints(
|
968
1368
|
[
|
969
1369
|
{"Name": "vpc-id", "Values": [vpc_id]},
|
@@ -997,9 +1397,16 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
997
1397
|
)
|
998
1398
|
return security_groups[0]["GroupId"]
|
999
1399
|
|
1000
|
-
def get_cluster_nat_gateways_egress_ips(
|
1001
|
-
|
1002
|
-
|
1400
|
+
def get_cluster_nat_gateways_egress_ips(
|
1401
|
+
self, account: dict[str, Any], vpc_id: str
|
1402
|
+
) -> set[str]:
|
1403
|
+
assume_role_data = self._get_account_assume_data(account)
|
1404
|
+
assumed_ec2 = self._get_assumed_role_client(
|
1405
|
+
account_name=assume_role_data[0],
|
1406
|
+
assume_role=assume_role_data[1],
|
1407
|
+
assume_region=assume_role_data[2],
|
1408
|
+
client_type="ec2",
|
1409
|
+
)
|
1003
1410
|
nat_gateways = assumed_ec2.describe_nat_gateways()
|
1004
1411
|
egress_ips: set[str] = set()
|
1005
1412
|
for nat in nat_gateways.get("NatGateways") or []:
|
@@ -1011,7 +1418,12 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1011
1418
|
|
1012
1419
|
return egress_ips
|
1013
1420
|
|
1014
|
-
def get_vpcs_details(
|
1421
|
+
def get_vpcs_details(
|
1422
|
+
self,
|
1423
|
+
account: awsh.Account,
|
1424
|
+
tags: Mapping[str, str] | None = None,
|
1425
|
+
route_tables: bool = False,
|
1426
|
+
) -> list[dict[str, Any]]:
|
1015
1427
|
results = []
|
1016
1428
|
ec2 = self._account_ec2_client(account["name"])
|
1017
1429
|
regions = [r["RegionName"] for r in ec2.describe_regions()["Regions"]]
|
@@ -1076,7 +1488,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1076
1488
|
share_account_uid: str,
|
1077
1489
|
image_id: str,
|
1078
1490
|
region: str | None = None,
|
1079
|
-
):
|
1491
|
+
) -> None:
|
1080
1492
|
ec2 = self._account_ec2_resource(account["name"], region)
|
1081
1493
|
image = ec2.Image(image_id)
|
1082
1494
|
launch_permission: LaunchPermissionModificationsTypeDef = {
|
@@ -1106,7 +1518,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1106
1518
|
self,
|
1107
1519
|
account_name: str,
|
1108
1520
|
region_name: str | None = None,
|
1109
|
-
) -> Iterator[
|
1521
|
+
) -> Iterator[LogGroupTypeDef]:
|
1110
1522
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1111
1523
|
paginator = client.get_paginator("describe_log_groups")
|
1112
1524
|
for page in paginator.paginate():
|
@@ -1130,7 +1542,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1130
1542
|
group_name: str,
|
1131
1543
|
retention_days: int,
|
1132
1544
|
region_name: str | None = None,
|
1133
|
-
):
|
1545
|
+
) -> None:
|
1134
1546
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1135
1547
|
client.put_retention_policy(
|
1136
1548
|
logGroupName=group_name, retentionInDays=retention_days
|
@@ -1141,21 +1553,33 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1141
1553
|
account_name: str,
|
1142
1554
|
group_name: str,
|
1143
1555
|
region_name: str | None = None,
|
1144
|
-
):
|
1556
|
+
) -> None:
|
1145
1557
|
client = self._account_cloudwatch_client(account_name, region_name=region_name)
|
1146
1558
|
client.delete_log_group(logGroupName=group_name)
|
1147
1559
|
|
1148
1560
|
def create_tag(
|
1149
1561
|
self, account: Mapping[str, Any], resource_id: str, tag: Mapping[str, str]
|
1150
|
-
):
|
1562
|
+
) -> None:
|
1151
1563
|
ec2 = self._account_ec2_client(account["name"])
|
1152
1564
|
tag_type_def: TagTypeDef = {"Key": tag["Key"], "Value": tag["Value"]}
|
1153
1565
|
ec2.create_tags(Resources=[resource_id], Tags=[tag_type_def])
|
1154
1566
|
|
1155
|
-
def get_alb_network_interface_ips(
|
1567
|
+
def get_alb_network_interface_ips(
|
1568
|
+
self, account: awsh.Account, service_name: str
|
1569
|
+
) -> set[str]:
|
1156
1570
|
assumed_role_data = self._get_account_assume_data(account)
|
1157
|
-
ec2_client = self._get_assumed_role_client(
|
1158
|
-
|
1571
|
+
ec2_client = self._get_assumed_role_client(
|
1572
|
+
account_name=assumed_role_data[0],
|
1573
|
+
assume_role=assumed_role_data[1],
|
1574
|
+
assume_region=assumed_role_data[2],
|
1575
|
+
client_type="ec2",
|
1576
|
+
)
|
1577
|
+
elb_client = self._get_assumed_role_client(
|
1578
|
+
account_name=assumed_role_data[0],
|
1579
|
+
assume_role=assumed_role_data[1],
|
1580
|
+
assume_region=assumed_role_data[2],
|
1581
|
+
client_type="elb",
|
1582
|
+
)
|
1159
1583
|
service_tag = {"Key": "kubernetes.io/service-name", "Value": service_name}
|
1160
1584
|
nis = ec2_client.describe_network_interfaces()["NetworkInterfaces"]
|
1161
1585
|
lbs = elb_client.describe_load_balancers()["LoadBalancerDescriptions"]
|
@@ -1183,7 +1607,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1183
1607
|
return result_ips
|
1184
1608
|
|
1185
1609
|
@staticmethod
|
1186
|
-
# pylint: disable=method-hidden
|
1187
1610
|
def get_vpc_default_sg_id(vpc_id: str, ec2: EC2Client) -> str | None:
|
1188
1611
|
vpc_security_groups = ec2.describe_security_groups(
|
1189
1612
|
Filters=[
|
@@ -1198,7 +1621,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1198
1621
|
return None
|
1199
1622
|
|
1200
1623
|
@staticmethod
|
1201
|
-
# pylint: disable=method-hidden
|
1202
1624
|
def get_transit_gateways(ec2: EC2Client) -> list[TransitGatewayTypeDef]:
|
1203
1625
|
tgws = ec2.describe_transit_gateways()
|
1204
1626
|
return tgws.get("TransitGateways", [])
|
@@ -1221,7 +1643,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1221
1643
|
return None
|
1222
1644
|
|
1223
1645
|
@staticmethod
|
1224
|
-
# pylint: disable=method-hidden
|
1225
1646
|
def get_transit_gateway_vpc_attachments(
|
1226
1647
|
tgw_id: str, ec2: EC2Client
|
1227
1648
|
) -> list[TransitGatewayVpcAttachmentTypeDef]:
|
@@ -1232,23 +1653,23 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1232
1653
|
|
1233
1654
|
def get_tgws_details(
|
1234
1655
|
self,
|
1235
|
-
account,
|
1236
|
-
region_name,
|
1237
|
-
routes_cidr_block,
|
1238
|
-
tags
|
1239
|
-
route_tables=False,
|
1240
|
-
security_groups=False,
|
1241
|
-
route53_associations=False,
|
1242
|
-
):
|
1656
|
+
account: awsh.Account,
|
1657
|
+
region_name: str,
|
1658
|
+
routes_cidr_block: str,
|
1659
|
+
tags: Mapping,
|
1660
|
+
route_tables: bool = False,
|
1661
|
+
security_groups: bool = False,
|
1662
|
+
route53_associations: bool = False,
|
1663
|
+
) -> list[dict[str, Any]]:
|
1243
1664
|
results = []
|
1244
1665
|
ec2 = self._account_ec2_client(account["name"], region_name)
|
1245
1666
|
tgws = ec2.describe_transit_gateways(
|
1246
1667
|
Filters=[{"Name": f"tag:{k}", "Values": [v]} for k, v in tags.items()]
|
1247
1668
|
)
|
1248
|
-
for tgw in tgws.get("TransitGateways"):
|
1669
|
+
for tgw in tgws.get("TransitGateways") or []:
|
1249
1670
|
tgw_id = tgw["TransitGatewayId"]
|
1250
1671
|
tgw_arn = tgw["TransitGatewayArn"]
|
1251
|
-
item = {
|
1672
|
+
item: dict[str, str | list[str] | list[dict]] = {
|
1252
1673
|
"tgw_id": tgw_id,
|
1253
1674
|
"tgw_arn": tgw_arn,
|
1254
1675
|
"region": region_name,
|
@@ -1280,7 +1701,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1280
1701
|
attachments = ec2.describe_transit_gateway_peering_attachments(
|
1281
1702
|
Filters=[{"Name": "transit-gateway-id", "Values": [tgw_id]}]
|
1282
1703
|
)
|
1283
|
-
for a in attachments.get("TransitGatewayPeeringAttachments"):
|
1704
|
+
for a in attachments.get("TransitGatewayPeeringAttachments") or []:
|
1284
1705
|
tgw_attachment_id = a["TransitGatewayAttachmentId"]
|
1285
1706
|
tgw_attachment_state = a["State"]
|
1286
1707
|
if tgw_attachment_state != "available":
|
@@ -1377,7 +1798,6 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1377
1798
|
return results
|
1378
1799
|
|
1379
1800
|
@staticmethod
|
1380
|
-
# pylint: disable=method-hidden
|
1381
1801
|
def _get_vpc_endpoints(
|
1382
1802
|
filters: Sequence[FilterTypeDef], ec2: EC2Client
|
1383
1803
|
) -> list["VpcEndpointTypeDef"]:
|
@@ -1411,11 +1831,15 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1411
1831
|
]
|
1412
1832
|
|
1413
1833
|
@staticmethod
|
1414
|
-
def _extract_records(
|
1834
|
+
def _extract_records(
|
1835
|
+
resource_records: Iterable[ResourceRecordTypeDef],
|
1836
|
+
) -> list[str]:
|
1415
1837
|
# [{'Value': 'ns.example.com.'}, ...]
|
1416
1838
|
return [r["Value"].rstrip(".") for r in resource_records]
|
1417
1839
|
|
1418
|
-
def get_route53_zone_ns_records(
|
1840
|
+
def get_route53_zone_ns_records(
|
1841
|
+
self, account_name: str, zone_name: str, region: str
|
1842
|
+
) -> list[str]:
|
1419
1843
|
route53 = self._account_route53_client(account_name, region)
|
1420
1844
|
record_sets = self._get_hosted_zone_record_sets(route53, zone_name)
|
1421
1845
|
filtered_record_sets = self._filter_record_sets(record_sets, zone_name, "NS")
|
@@ -1425,7 +1849,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1425
1849
|
ns_records = self._extract_records(resource_records)
|
1426
1850
|
return ns_records
|
1427
1851
|
|
1428
|
-
def get_route53_zones(self):
|
1852
|
+
def get_route53_zones(self) -> dict[str, list[dict[str, str]]]:
|
1429
1853
|
"""
|
1430
1854
|
Return a list of (str, dict) representing Route53 DNS zones per account
|
1431
1855
|
|
@@ -1437,7 +1861,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1437
1861
|
for account, _ in self.sessions.items()
|
1438
1862
|
}
|
1439
1863
|
|
1440
|
-
def create_route53_zone(self, account_name, zone_name):
|
1864
|
+
def create_route53_zone(self, account_name: str, zone_name: str) -> None:
|
1441
1865
|
"""
|
1442
1866
|
Create a Route53 DNS zone
|
1443
1867
|
|
@@ -1467,7 +1891,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1467
1891
|
except Exception as e:
|
1468
1892
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1469
1893
|
|
1470
|
-
def delete_route53_zone(self, account_name, zone_id):
|
1894
|
+
def delete_route53_zone(self, account_name: str, zone_id: str) -> None:
|
1471
1895
|
"""
|
1472
1896
|
Delete a Route53 DNS zone
|
1473
1897
|
|
@@ -1492,7 +1916,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1492
1916
|
except Exception as e:
|
1493
1917
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1494
1918
|
|
1495
|
-
def delete_route53_record(
|
1919
|
+
def delete_route53_record(
|
1920
|
+
self, account_name: str, zone_id: str, awsdata: ResourceRecordSetTypeDef
|
1921
|
+
) -> None:
|
1496
1922
|
"""
|
1497
1923
|
Delete a Route53 DNS zone record
|
1498
1924
|
|
@@ -1526,7 +1952,9 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1526
1952
|
except Exception as e:
|
1527
1953
|
logging.error(f"[{account_name}] unhandled exception: {e}")
|
1528
1954
|
|
1529
|
-
def upsert_route53_record(
|
1955
|
+
def upsert_route53_record(
|
1956
|
+
self, account_name: str, zone_id: str, recordset: ResourceRecordSetTypeDef
|
1957
|
+
) -> None:
|
1530
1958
|
"""
|
1531
1959
|
Upsert a Route53 DNS zone record
|
1532
1960
|
|
@@ -1612,7 +2040,7 @@ class AWSApi: # pylint: disable=too-many-public-methods
|
|
1612
2040
|
self,
|
1613
2041
|
account_name: str,
|
1614
2042
|
region_name: str | None = None,
|
1615
|
-
):
|
2043
|
+
) -> DBRecommendationsMessageTypeDef:
|
1616
2044
|
rds = self._account_rds_client(account_name, region_name)
|
1617
2045
|
return rds.describe_db_recommendations()
|
1618
2046
|
|