labagent 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- labagent/__init__.py +3 -0
- labagent/audit.py +233 -0
- labagent/cleanup.py +67 -0
- labagent/cli.py +63 -0
- labagent/config.py +82 -0
- labagent/context.py +51 -0
- labagent/deleters.py +507 -0
- labagent/discovery.py +162 -0
- labagent/envelope.py +48 -0
- labagent/iam_ops.py +133 -0
- labagent/plan.py +40 -0
- labagent/rules.py +153 -0
- labagent/run.py +58 -0
- labagent/secrets_box.py +36 -0
- labagent/sweep.py +107 -0
- labagent/terraform/cohort_iam/.terraform.lock.hcl +29 -0
- labagent/terraform/cohort_iam/main.tf +101 -0
- labagent/terraform/cohort_iam/outputs.tf +21 -0
- labagent/terraform/cohort_iam/sandbox-policy.json.tftpl +33 -0
- labagent/terraform/cohort_iam/tests/cohort.tftest.hcl +63 -0
- labagent/terraform/cohort_iam/variables.tf +55 -0
- labagent/terraform/cohort_iam/versions.tf +15 -0
- labagent/terraform/state_backend/.terraform.lock.hcl +26 -0
- labagent/terraform/state_backend/main.tf +193 -0
- labagent/terraform/state_backend/tests/bootstrap.tftest.hcl +37 -0
- labagent/tf.py +118 -0
- labagent-0.1.0.dist-info/METADATA +73 -0
- labagent-0.1.0.dist-info/RECORD +32 -0
- labagent-0.1.0.dist-info/WHEEL +5 -0
- labagent-0.1.0.dist-info/entry_points.txt +3 -0
- labagent-0.1.0.dist-info/licenses/LICENSE +202 -0
- labagent-0.1.0.dist-info/top_level.txt +1 -0
labagent/deleters.py
ADDED
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
"""One deleter per resource type. Each returns a Result and is safe to re-run (idempotent)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Callable, Dict, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
from botocore.exceptions import ClientError
|
|
9
|
+
|
|
10
|
+
from .context import Ctx
|
|
11
|
+
from .discovery import Resource
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Result:
|
|
16
|
+
state: str # deleted | pending | throttled | failed | gone
|
|
17
|
+
reason: Optional[str] = None
|
|
18
|
+
retry_after_s: Optional[int] = None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _code(e: ClientError) -> str:
|
|
22
|
+
return e.response.get("Error", {}).get("Code", "")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
NOT_FOUND = {"EntityNotFoundException", "DashboardNotFoundError", "ResourceNotFound", "InvalidVpcID.NotFound", "NotFound", "AWS.SimpleQueueService.NonExistentQueue", "QueueDoesNotExist", "RepositoryNotFoundException",
|
|
26
|
+
"InvalidKeyPair.NotFound", "ResourceNotFoundException", "NotFoundException", "NoSuchBucket", "NoSuchDistribution",
|
|
27
|
+
"NoSuchEntity", "InvalidInstanceID.NotFound", "InvalidAllocationID.NotFound",
|
|
28
|
+
"InvalidGroup.NotFound", "InvalidVolume.NotFound"}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def guarded(fn: Callable[[Ctx, Resource], Result]) -> Callable[[Ctx, Resource], Result]:
|
|
32
|
+
def wrapper(ctx: Ctx, r: Resource) -> Result:
|
|
33
|
+
try:
|
|
34
|
+
return fn(ctx, r)
|
|
35
|
+
except ClientError as e:
|
|
36
|
+
c = _code(e)
|
|
37
|
+
if c in NOT_FOUND:
|
|
38
|
+
return Result("gone")
|
|
39
|
+
if c in ("TooManyRequestsException", "Throttling", "ThrottlingException"):
|
|
40
|
+
return Result("throttled", c, 31)
|
|
41
|
+
if c in ("DependencyViolation", "VolumeInUse", "ResourceInUseException", "PreconditionFailed",
|
|
42
|
+
"DistributionNotDisabled", "DeleteConflict"):
|
|
43
|
+
return Result("pending", c, 300)
|
|
44
|
+
return Result("failed", f"{c}: {e.response['Error'].get('Message', '')[:120]}")
|
|
45
|
+
return wrapper
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@guarded
|
|
49
|
+
def del_lambda(ctx, r):
|
|
50
|
+
ctx.client("lambda", r.region).delete_function(FunctionName=r.rid)
|
|
51
|
+
return Result("deleted")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@guarded
|
|
55
|
+
def del_bucket(ctx, r):
|
|
56
|
+
s3 = ctx.session.resource("s3")
|
|
57
|
+
b = s3.Bucket(r.rid)
|
|
58
|
+
b.object_versions.delete()
|
|
59
|
+
b.objects.all().delete()
|
|
60
|
+
b.delete()
|
|
61
|
+
return Result("deleted")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@guarded
|
|
65
|
+
def del_table(ctx, r):
|
|
66
|
+
ctx.client("dynamodb", r.region).delete_table(TableName=r.rid)
|
|
67
|
+
return Result("deleted")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@guarded
|
|
71
|
+
def del_loggroup(ctx, r):
|
|
72
|
+
ctx.client("logs", r.region).delete_log_group(logGroupName=r.rid)
|
|
73
|
+
return Result("deleted")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@guarded
|
|
77
|
+
def del_instance(ctx, r):
|
|
78
|
+
ctx.client("ec2", r.region).terminate_instances(InstanceIds=[r.rid])
|
|
79
|
+
return Result("deleted")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@guarded
|
|
83
|
+
def del_eip(ctx, r):
|
|
84
|
+
ec2 = ctx.client("ec2", r.region)
|
|
85
|
+
addr = ec2.describe_addresses(AllocationIds=[r.rid])["Addresses"][0]
|
|
86
|
+
if "AssociationId" in addr:
|
|
87
|
+
ec2.disassociate_address(AssociationId=addr["AssociationId"])
|
|
88
|
+
ec2.release_address(AllocationId=r.rid)
|
|
89
|
+
return Result("deleted")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@guarded
|
|
93
|
+
def del_sg(ctx, r):
|
|
94
|
+
"""Groups slated for deletion often reference each other (A allows B, B allows A), which blocks both. Remove this
|
|
95
|
+
group's own rules that point at other groups first (it is being deleted anyway), then delete. A group that is
|
|
96
|
+
still referenced by a not-yet-deleted peer returns pending and is finished by the next run."""
|
|
97
|
+
ec2 = ctx.client("ec2", r.region)
|
|
98
|
+
try:
|
|
99
|
+
g = ec2.describe_security_groups(GroupIds=[r.rid])["SecurityGroups"][0]
|
|
100
|
+
except ClientError as e:
|
|
101
|
+
if _code(e) == "InvalidGroup.NotFound":
|
|
102
|
+
return Result("gone")
|
|
103
|
+
raise
|
|
104
|
+
for key, revoke in (("IpPermissions", ec2.revoke_security_group_ingress), ("IpPermissionsEgress", ec2.revoke_security_group_egress)):
|
|
105
|
+
refs = [{**{k: v for k, v in p.items() if k in ("IpProtocol", "FromPort", "ToPort")},
|
|
106
|
+
"UserIdGroupPairs": [{"GroupId": x["GroupId"]} for x in p["UserIdGroupPairs"]]}
|
|
107
|
+
for p in g.get(key, []) if p.get("UserIdGroupPairs")]
|
|
108
|
+
if refs:
|
|
109
|
+
revoke(GroupId=r.rid, IpPermissions=refs)
|
|
110
|
+
ec2.delete_security_group(GroupId=r.rid)
|
|
111
|
+
return Result("deleted")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@guarded
|
|
115
|
+
def del_volume(ctx, r):
|
|
116
|
+
ctx.client("ec2", r.region).delete_volume(VolumeId=r.rid)
|
|
117
|
+
return Result("deleted")
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@guarded
|
|
121
|
+
def del_restapi(ctx, r):
|
|
122
|
+
ctx.client("apigateway", r.region).delete_rest_api(restApiId=r.rid)
|
|
123
|
+
return Result("deleted")
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@guarded
|
|
127
|
+
def del_httpapi(ctx, r):
|
|
128
|
+
ctx.client("apigatewayv2", r.region).delete_api(ApiId=r.rid)
|
|
129
|
+
return Result("deleted")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _s3_bucket_of(domain: str):
|
|
133
|
+
"""Bucket name for a bucket-style origin domain, else None (<bucket>.s3.<region>.amazonaws.com, .s3.amazonaws.com)."""
|
|
134
|
+
m = re.match(r"^(.+?)\.s3[.-]", domain)
|
|
135
|
+
return m.group(1) if m and domain.endswith(".amazonaws.com") else None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _neutralize_missing_s3_origins(ctx, cfg) -> int:
|
|
139
|
+
"""CloudFront refuses to update a distribution whose S3 origin bucket was deleted ("The parameter S3 bucket has
|
|
140
|
+
been deleted"), so it cannot even be disabled. Repoint such origins at a placeholder custom origin; the
|
|
141
|
+
distribution is about to be deleted anyway. Only origins whose bucket is confirmed missing are touched."""
|
|
142
|
+
s3, fixed = ctx.client("s3"), 0
|
|
143
|
+
for o in cfg.get("Origins", {}).get("Items", []):
|
|
144
|
+
bucket = _s3_bucket_of(o["DomainName"])
|
|
145
|
+
if not bucket:
|
|
146
|
+
continue
|
|
147
|
+
try:
|
|
148
|
+
s3.head_bucket(Bucket=bucket)
|
|
149
|
+
continue
|
|
150
|
+
except ClientError as e:
|
|
151
|
+
if _code(e) not in ("404", "NoSuchBucket", "NotFound"):
|
|
152
|
+
continue # exists but not ours / no permission: leave alone
|
|
153
|
+
o["DomainName"] = "example.com"
|
|
154
|
+
o.pop("S3OriginConfig", None)
|
|
155
|
+
o["OriginAccessControlId"] = ""
|
|
156
|
+
o["CustomOriginConfig"] = {"HTTPPort": 80, "HTTPSPort": 443, "OriginProtocolPolicy": "https-only",
|
|
157
|
+
"OriginSslProtocols": {"Quantity": 1, "Items": ["TLSv1.2"]},
|
|
158
|
+
"OriginReadTimeout": 30, "OriginKeepaliveTimeout": 5}
|
|
159
|
+
fixed += 1
|
|
160
|
+
return fixed
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
@guarded
|
|
164
|
+
def del_distribution(ctx, r):
|
|
165
|
+
"""Two phase: disable now, delete on a later run once Deployed (15 to 30 minutes)."""
|
|
166
|
+
cf = ctx.client("cloudfront", "us-east-1")
|
|
167
|
+
cur = cf.get_distribution_config(Id=r.rid)
|
|
168
|
+
cfg, etag = cur["DistributionConfig"], cur["ETag"]
|
|
169
|
+
if cfg.get("Enabled"):
|
|
170
|
+
cfg["Enabled"] = False
|
|
171
|
+
try:
|
|
172
|
+
cf.update_distribution(Id=r.rid, IfMatch=etag, DistributionConfig=cfg)
|
|
173
|
+
except ClientError as e:
|
|
174
|
+
msg = e.response["Error"].get("Message", "")
|
|
175
|
+
if _code(e) == "InvalidArgument" and "S3 bucket" in msg and "deleted" in msg \
|
|
176
|
+
and _neutralize_missing_s3_origins(ctx, cfg):
|
|
177
|
+
cf.update_distribution(Id=r.rid, IfMatch=etag, DistributionConfig=cfg)
|
|
178
|
+
else:
|
|
179
|
+
raise
|
|
180
|
+
return Result("pending", "disabling", 900)
|
|
181
|
+
status = cf.get_distribution(Id=r.rid)["Distribution"]["Status"]
|
|
182
|
+
if status != "Deployed":
|
|
183
|
+
return Result("pending", f"status={status}", 600)
|
|
184
|
+
cf.delete_distribution(Id=r.rid, IfMatch=etag)
|
|
185
|
+
return Result("deleted")
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _vpc_blockers(ec2, elbv2, vpc_id: str) -> str:
|
|
189
|
+
"""Anything that makes deleting a VPC unsafe or impossible. Empty string = clear to delete."""
|
|
190
|
+
f = [{"Name": "vpc-id", "Values": [vpc_id]}]
|
|
191
|
+
insts = [i for r in ec2.describe_instances(Filters=f)["Reservations"] for i in r["Instances"]
|
|
192
|
+
if i["State"]["Name"] != "terminated"]
|
|
193
|
+
if insts:
|
|
194
|
+
return f"{len(insts)} EC2 instance(s)"
|
|
195
|
+
nats = [n for n in ec2.describe_nat_gateways(Filter=f)["NatGateways"] if n["State"] != "deleted"]
|
|
196
|
+
if nats:
|
|
197
|
+
return f"{len(nats)} NAT gateway(s)"
|
|
198
|
+
lbs = [lb for lb in elbv2.describe_load_balancers()["LoadBalancers"] if lb.get("VpcId") == vpc_id]
|
|
199
|
+
if lbs:
|
|
200
|
+
return f"{len(lbs)} load balancer(s)"
|
|
201
|
+
peer = ec2.describe_vpc_peering_connections(Filters=[{"Name": "requester-vpc-info.vpc-id", "Values": [vpc_id]}])["VpcPeeringConnections"]
|
|
202
|
+
if [p for p in peer if p["Status"]["Code"] not in ("deleted", "rejected", "failed", "expired")]:
|
|
203
|
+
return "VPC peering connection(s)"
|
|
204
|
+
# interface endpoints own ENIs that disappear once the endpoint goes, so only count other ENIs
|
|
205
|
+
enis = [n for n in ec2.describe_network_interfaces(Filters=f)["NetworkInterfaces"]
|
|
206
|
+
if n.get("InterfaceType") != "vpc_endpoint"]
|
|
207
|
+
if enis:
|
|
208
|
+
kinds = sorted({(n.get("Description") or n.get("InterfaceType") or "eni")[:40] for n in enis})
|
|
209
|
+
return f"{len(enis)} network interface(s) in use, e.g. {kinds[0]}"
|
|
210
|
+
return ""
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
@guarded
|
|
214
|
+
def del_vpc(ctx, r):
|
|
215
|
+
"""Delete a VPC and its dependents in order. Refuses (failed, with the reason) while anything else still uses it."""
|
|
216
|
+
ec2 = ctx.client("ec2", r.region)
|
|
217
|
+
vpc = r.rid
|
|
218
|
+
why = _vpc_blockers(ec2, ctx.client("elbv2", r.region), vpc)
|
|
219
|
+
if why:
|
|
220
|
+
return Result("failed", f"in use: {why}")
|
|
221
|
+
f = [{"Name": "vpc-id", "Values": [vpc]}]
|
|
222
|
+
for ep in ec2.describe_vpc_endpoints(Filters=f)["VpcEndpoints"]:
|
|
223
|
+
ec2.delete_vpc_endpoints(VpcEndpointIds=[ep["VpcEndpointId"]])
|
|
224
|
+
for ig in ec2.describe_internet_gateways(Filters=[{"Name": "attachment.vpc-id", "Values": [vpc]}])["InternetGateways"]:
|
|
225
|
+
ec2.detach_internet_gateway(InternetGatewayId=ig["InternetGatewayId"], VpcId=vpc)
|
|
226
|
+
ec2.delete_internet_gateway(InternetGatewayId=ig["InternetGatewayId"])
|
|
227
|
+
for sn in ec2.describe_subnets(Filters=f)["Subnets"]:
|
|
228
|
+
ec2.delete_subnet(SubnetId=sn["SubnetId"])
|
|
229
|
+
for rt in ec2.describe_route_tables(Filters=f)["RouteTables"]:
|
|
230
|
+
if any(a.get("Main") for a in rt["Associations"]):
|
|
231
|
+
continue
|
|
232
|
+
for a in rt["Associations"]:
|
|
233
|
+
ec2.disassociate_route_table(AssociationId=a["RouteTableAssociationId"])
|
|
234
|
+
ec2.delete_route_table(RouteTableId=rt["RouteTableId"])
|
|
235
|
+
for acl in ec2.describe_network_acls(Filters=f)["NetworkAcls"]:
|
|
236
|
+
if not acl["IsDefault"]:
|
|
237
|
+
ec2.delete_network_acl(NetworkAclId=acl["NetworkAclId"])
|
|
238
|
+
sgs = [g for g in ec2.describe_security_groups(Filters=f)["SecurityGroups"] if g["GroupName"] != "default"]
|
|
239
|
+
for g in sgs: # break cross references between groups first
|
|
240
|
+
if g["IpPermissions"]:
|
|
241
|
+
ec2.revoke_security_group_ingress(GroupId=g["GroupId"], IpPermissions=g["IpPermissions"])
|
|
242
|
+
for g in sgs:
|
|
243
|
+
ec2.delete_security_group(GroupId=g["GroupId"])
|
|
244
|
+
ec2.delete_vpc(VpcId=vpc)
|
|
245
|
+
return Result("deleted")
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
# (service, rtype) -> (deleter, order). Lower order runs first.
|
|
249
|
+
REGISTRY: Dict[Tuple[str, str], Tuple[Callable, int]] = {
|
|
250
|
+
("cloudfront", "distribution"): (del_distribution, 10),
|
|
251
|
+
("apigateway", "restapi"): (del_restapi, 20),
|
|
252
|
+
("apigateway", "httpapi"): (del_httpapi, 20),
|
|
253
|
+
("lambda", "function"): (del_lambda, 30),
|
|
254
|
+
("ec2", "instance"): (del_instance, 40),
|
|
255
|
+
("ec2", "elastic-ip"): (del_eip, 50),
|
|
256
|
+
("ec2", "volume"): (del_volume, 55),
|
|
257
|
+
("ec2", "security-group"): (del_sg, 60),
|
|
258
|
+
("s3", "bucket"): (del_bucket, 70),
|
|
259
|
+
("dynamodb", "table"): (del_table, 80),
|
|
260
|
+
("logs", "log-group"): (del_loggroup, 90),
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
# The tag-driven cohort sweep only ever touches these. IAM is owned by Terraform; the rest are
|
|
265
|
+
# only deleted through an explicit cleanup ruleset (see rules.py).
|
|
266
|
+
TAG_SWEEP_TYPES = list(REGISTRY.keys())
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def supported_types():
|
|
270
|
+
return list(TAG_SWEEP_TYPES)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def cleanup_types():
|
|
274
|
+
return list(REGISTRY.keys())
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def order_of(r: Resource) -> int:
|
|
278
|
+
return REGISTRY.get(r.key, (None, 999))[1]
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
# ---- additional types found in real accounts ---------------------------------------------------
|
|
282
|
+
|
|
283
|
+
@guarded
|
|
284
|
+
def del_keypair(ctx, r):
|
|
285
|
+
ctx.client("ec2", r.region).delete_key_pair(KeyName=r.rid)
|
|
286
|
+
return Result("deleted")
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@guarded
|
|
290
|
+
def del_topic(ctx, r):
|
|
291
|
+
sns = ctx.client("sns", r.region)
|
|
292
|
+
arn = next((t["TopicArn"] for p in sns.get_paginator("list_topics").paginate() for t in p["Topics"]
|
|
293
|
+
if t["TopicArn"].split(":")[-1] == r.rid), None)
|
|
294
|
+
if arn is None:
|
|
295
|
+
return Result("gone")
|
|
296
|
+
sns.delete_topic(TopicArn=arn)
|
|
297
|
+
return Result("deleted")
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
@guarded
|
|
301
|
+
def del_queue(ctx, r):
|
|
302
|
+
sqs = ctx.client("sqs", r.region)
|
|
303
|
+
sqs.delete_queue(QueueUrl=sqs.get_queue_url(QueueName=r.rid)["QueueUrl"])
|
|
304
|
+
return Result("deleted")
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
@guarded
|
|
308
|
+
def del_secret(ctx, r):
|
|
309
|
+
# 7 day recovery window, not an immediate hard delete
|
|
310
|
+
ctx.client("secretsmanager", r.region).delete_secret(SecretId=r.rid, RecoveryWindowInDays=7)
|
|
311
|
+
return Result("deleted")
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@guarded
|
|
315
|
+
def del_repository(ctx, r):
|
|
316
|
+
ctx.client("ecr", r.region).delete_repository(repositoryName=r.rid, force=True)
|
|
317
|
+
return Result("deleted")
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
@guarded
|
|
321
|
+
def del_stack(ctx, r):
|
|
322
|
+
cfn = ctx.client("cloudformation", r.region)
|
|
323
|
+
try:
|
|
324
|
+
st = cfn.describe_stacks(StackName=r.rid)["Stacks"][0]["StackStatus"]
|
|
325
|
+
except ClientError as e:
|
|
326
|
+
if "does not exist" in e.response["Error"].get("Message", ""):
|
|
327
|
+
return Result("gone")
|
|
328
|
+
raise
|
|
329
|
+
if st == "DELETE_COMPLETE":
|
|
330
|
+
return Result("gone")
|
|
331
|
+
if st == "DELETE_FAILED":
|
|
332
|
+
return Result("failed", "DELETE_FAILED: inspect the stack events")
|
|
333
|
+
if st != "DELETE_IN_PROGRESS":
|
|
334
|
+
cfn.delete_stack(StackName=r.rid)
|
|
335
|
+
return Result("pending", "deleting", 120)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _tolerant(fn, **kw):
|
|
339
|
+
try:
|
|
340
|
+
fn(**kw)
|
|
341
|
+
except ClientError as e:
|
|
342
|
+
if _code(e) != "NoSuchEntity":
|
|
343
|
+
raise
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
@guarded
|
|
347
|
+
def del_iam_user(ctx, r):
|
|
348
|
+
iam, name = ctx.client("iam"), r.rid
|
|
349
|
+
# 1. Access keys first: deactivate (cuts off use immediately), then delete.
|
|
350
|
+
for k in iam.list_access_keys(UserName=name)["AccessKeyMetadata"]:
|
|
351
|
+
if k["Status"] != "Inactive":
|
|
352
|
+
iam.update_access_key(UserName=name, AccessKeyId=k["AccessKeyId"], Status="Inactive")
|
|
353
|
+
iam.delete_access_key(UserName=name, AccessKeyId=k["AccessKeyId"])
|
|
354
|
+
# 2. Same for signing certificates, and MFA devices are deactivated before they are deleted.
|
|
355
|
+
for c in iam.list_signing_certificates(UserName=name)["Certificates"]:
|
|
356
|
+
if c["Status"] != "Inactive":
|
|
357
|
+
iam.update_signing_certificate(UserName=name, CertificateId=c["CertificateId"], Status="Inactive")
|
|
358
|
+
iam.delete_signing_certificate(UserName=name, CertificateId=c["CertificateId"])
|
|
359
|
+
for m in iam.list_mfa_devices(UserName=name)["MFADevices"]:
|
|
360
|
+
iam.deactivate_mfa_device(UserName=name, SerialNumber=m["SerialNumber"])
|
|
361
|
+
_tolerant(iam.delete_virtual_mfa_device, SerialNumber=m["SerialNumber"])
|
|
362
|
+
# 3. Then console access, then everything else that blocks DeleteUser, then the user.
|
|
363
|
+
_tolerant(iam.delete_login_profile, UserName=name)
|
|
364
|
+
for k in iam.list_ssh_public_keys(UserName=name)["SSHPublicKeys"]:
|
|
365
|
+
iam.delete_ssh_public_key(UserName=name, SSHPublicKeyId=k["SSHPublicKeyId"])
|
|
366
|
+
for sc in iam.list_service_specific_credentials(UserName=name)["ServiceSpecificCredentials"]:
|
|
367
|
+
iam.delete_service_specific_credential(UserName=name, ServiceSpecificCredentialId=sc["ServiceSpecificCredentialId"])
|
|
368
|
+
for p in iam.list_user_policies(UserName=name)["PolicyNames"]:
|
|
369
|
+
iam.delete_user_policy(UserName=name, PolicyName=p)
|
|
370
|
+
for p in iam.list_attached_user_policies(UserName=name)["AttachedPolicies"]:
|
|
371
|
+
iam.detach_user_policy(UserName=name, PolicyArn=p["PolicyArn"])
|
|
372
|
+
for g in iam.list_groups_for_user(UserName=name)["Groups"]:
|
|
373
|
+
iam.remove_user_from_group(UserName=name, GroupName=g["GroupName"])
|
|
374
|
+
iam.delete_user(UserName=name)
|
|
375
|
+
return Result("deleted")
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
@guarded
|
|
379
|
+
def del_iam_group(ctx, r):
|
|
380
|
+
iam, name = ctx.client("iam"), r.rid
|
|
381
|
+
for p in iam.list_attached_group_policies(GroupName=name)["AttachedPolicies"]:
|
|
382
|
+
iam.detach_group_policy(GroupName=name, PolicyArn=p["PolicyArn"])
|
|
383
|
+
for p in iam.list_group_policies(GroupName=name)["PolicyNames"]:
|
|
384
|
+
iam.delete_group_policy(GroupName=name, PolicyName=p)
|
|
385
|
+
for u in iam.get_group(GroupName=name)["Users"]:
|
|
386
|
+
iam.remove_user_from_group(GroupName=name, UserName=u["UserName"])
|
|
387
|
+
iam.delete_group(GroupName=name)
|
|
388
|
+
return Result("deleted")
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
@guarded
|
|
392
|
+
def del_iam_role(ctx, r):
|
|
393
|
+
iam, name = ctx.client("iam"), r.rid
|
|
394
|
+
for p in iam.list_attached_role_policies(RoleName=name)["AttachedPolicies"]:
|
|
395
|
+
iam.detach_role_policy(RoleName=name, PolicyArn=p["PolicyArn"])
|
|
396
|
+
for p in iam.list_role_policies(RoleName=name)["PolicyNames"]:
|
|
397
|
+
iam.delete_role_policy(RoleName=name, PolicyName=p)
|
|
398
|
+
for ip in iam.list_instance_profiles_for_role(RoleName=name)["InstanceProfiles"]:
|
|
399
|
+
iam.remove_role_from_instance_profile(InstanceProfileName=ip["InstanceProfileName"], RoleName=name)
|
|
400
|
+
iam.delete_role(RoleName=name)
|
|
401
|
+
return Result("deleted")
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
@guarded
|
|
405
|
+
def del_iam_policy(ctx, r):
|
|
406
|
+
iam = ctx.client("iam")
|
|
407
|
+
arn = r.attrs.get("arn_override") or r.arn
|
|
408
|
+
if not arn.startswith("arn:"):
|
|
409
|
+
arn = next(p["Arn"] for pg in iam.get_paginator("list_policies").paginate(Scope="Local")
|
|
410
|
+
for p in pg["Policies"] if p["PolicyName"] == r.rid)
|
|
411
|
+
ents = iam.list_entities_for_policy(PolicyArn=arn)
|
|
412
|
+
for u in ents["PolicyUsers"]:
|
|
413
|
+
iam.detach_user_policy(UserName=u["UserName"], PolicyArn=arn)
|
|
414
|
+
for g in ents["PolicyGroups"]:
|
|
415
|
+
iam.detach_group_policy(GroupName=g["GroupName"], PolicyArn=arn)
|
|
416
|
+
for ro in ents["PolicyRoles"]:
|
|
417
|
+
iam.detach_role_policy(RoleName=ro["RoleName"], PolicyArn=arn)
|
|
418
|
+
for v in iam.list_policy_versions(PolicyArn=arn)["Versions"]:
|
|
419
|
+
if not v["IsDefaultVersion"]:
|
|
420
|
+
iam.delete_policy_version(PolicyArn=arn, VersionId=v["VersionId"])
|
|
421
|
+
iam.delete_policy(PolicyArn=arn)
|
|
422
|
+
return Result("deleted")
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
REGISTRY.update({
|
|
426
|
+
("ec2", "vpc"): (del_vpc, 66),
|
|
427
|
+
("cloudformation", "stack"): (del_stack, 5),
|
|
428
|
+
("ec2", "key-pair"): (del_keypair, 58),
|
|
429
|
+
("sns", "topic"): (del_topic, 92),
|
|
430
|
+
("sqs", "queue"): (del_queue, 93),
|
|
431
|
+
("secretsmanager", "secret"): (del_secret, 94),
|
|
432
|
+
("ecr", "repository"): (del_repository, 95),
|
|
433
|
+
("iam", "user"): (del_iam_user, 100),
|
|
434
|
+
("iam", "group"): (del_iam_group, 105),
|
|
435
|
+
("iam", "role"): (del_iam_role, 110),
|
|
436
|
+
("iam", "policy"): (del_iam_policy, 120),
|
|
437
|
+
})
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
# ---- analytics and observability leftovers ----------------------------------------------------
|
|
441
|
+
|
|
442
|
+
@guarded
|
|
443
|
+
def del_glue_database(ctx, r):
|
|
444
|
+
ctx.client("glue", r.region).delete_database(Name=r.rid) # catalog metadata only, no S3 data
|
|
445
|
+
return Result("deleted")
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
@guarded
|
|
449
|
+
def del_glue_crawler(ctx, r):
|
|
450
|
+
g = ctx.client("glue", r.region)
|
|
451
|
+
if g.get_crawler(Name=r.rid)["Crawler"]["State"] != "READY":
|
|
452
|
+
return Result("pending", "crawler is running or stopping", 120)
|
|
453
|
+
g.delete_crawler(Name=r.rid)
|
|
454
|
+
return Result("deleted")
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
@guarded
|
|
458
|
+
def del_glue_job(ctx, r):
|
|
459
|
+
ctx.client("glue", r.region).delete_job(JobName=r.rid)
|
|
460
|
+
return Result("deleted")
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
@guarded
|
|
464
|
+
def del_athena_workgroup(ctx, r):
|
|
465
|
+
a = ctx.client("athena", r.region)
|
|
466
|
+
try:
|
|
467
|
+
a.delete_work_group(WorkGroup=r.rid, RecursiveDeleteOption=True)
|
|
468
|
+
except ClientError as e:
|
|
469
|
+
if _code(e) == "InvalidRequestException" and "not found" in e.response["Error"].get("Message", "").lower():
|
|
470
|
+
return Result("gone")
|
|
471
|
+
raise
|
|
472
|
+
return Result("deleted")
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
@guarded
|
|
476
|
+
def del_alarm(ctx, r):
|
|
477
|
+
ctx.client("cloudwatch", r.region).delete_alarms(AlarmNames=[r.rid])
|
|
478
|
+
return Result("deleted")
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
@guarded
|
|
482
|
+
def del_dashboard(ctx, r):
|
|
483
|
+
ctx.client("cloudwatch", r.region).delete_dashboards(DashboardNames=[r.rid])
|
|
484
|
+
return Result("deleted")
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
@guarded
|
|
488
|
+
def del_layer(ctx, r):
|
|
489
|
+
lam = ctx.client("lambda", r.region)
|
|
490
|
+
versions = [v["Version"] for p in lam.get_paginator("list_layer_versions").paginate(LayerName=r.rid)
|
|
491
|
+
for v in p["LayerVersions"]]
|
|
492
|
+
if not versions:
|
|
493
|
+
return Result("gone")
|
|
494
|
+
for v in versions:
|
|
495
|
+
lam.delete_layer_version(LayerName=r.rid, VersionNumber=v)
|
|
496
|
+
return Result("deleted")
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
REGISTRY.update({
|
|
500
|
+
("lambda", "layer"): (del_layer, 31),
|
|
501
|
+
("glue", "crawler"): (del_glue_crawler, 84),
|
|
502
|
+
("glue", "job"): (del_glue_job, 85),
|
|
503
|
+
("glue", "database"): (del_glue_database, 86),
|
|
504
|
+
("athena", "workgroup"): (del_athena_workgroup, 87),
|
|
505
|
+
("cloudwatch", "alarm"): (del_alarm, 91),
|
|
506
|
+
("cloudwatch", "dashboard"): (del_dashboard, 91),
|
|
507
|
+
})
|
labagent/discovery.py
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Discovery via the Resource Groups Tagging API, plus attribution and classification."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import re
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import Dict, Iterable, List, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
from .config import Config
|
|
9
|
+
from .context import Ctx
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class Resource:
|
|
14
|
+
arn: str
|
|
15
|
+
service: str
|
|
16
|
+
rtype: str
|
|
17
|
+
rid: str
|
|
18
|
+
region: str
|
|
19
|
+
name: str
|
|
20
|
+
tags: Dict[str, str] = field(default_factory=dict)
|
|
21
|
+
attrs: Dict[str, object] = field(default_factory=dict) # created, last_used, enabled, attachments, ...
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def key(self) -> Tuple[str, str]:
|
|
25
|
+
return (self.service, self.rtype)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def parse_arn(arn: str, tags: Optional[Dict[str, str]] = None) -> Resource:
|
|
29
|
+
"""arn:partition:service:region:account:resource. Resource part formats vary by service."""
|
|
30
|
+
parts = arn.split(":", 5)
|
|
31
|
+
service, region, res = parts[2], parts[3], parts[5]
|
|
32
|
+
rtype, rid = service, res
|
|
33
|
+
if service == "s3":
|
|
34
|
+
rtype, rid = "bucket", res.split("/")[0]
|
|
35
|
+
elif service == "lambda" and res.startswith("function:"):
|
|
36
|
+
rtype, rid = "function", res.split(":")[1]
|
|
37
|
+
elif service == "dynamodb" and res.startswith("table/"):
|
|
38
|
+
rtype, rid = "table", res.split("/")[1]
|
|
39
|
+
elif service == "logs" and res.startswith("log-group:"):
|
|
40
|
+
rtype, rid = "log-group", res[len("log-group:"):].rstrip(":*")
|
|
41
|
+
elif service == "cloudfront" and res.startswith("distribution/"):
|
|
42
|
+
rtype, rid = "distribution", res.split("/")[1]
|
|
43
|
+
elif service == "apigateway":
|
|
44
|
+
# arn:aws:apigateway:region::/restapis/<id> or /apis/<id>
|
|
45
|
+
segs = [s for s in res.split("/") if s]
|
|
46
|
+
if segs and segs[0] == "restapis" and len(segs) == 2:
|
|
47
|
+
rtype, rid = "restapi", segs[1]
|
|
48
|
+
elif segs and segs[0] == "apis" and len(segs) == 2:
|
|
49
|
+
rtype, rid = "httpapi", segs[1]
|
|
50
|
+
else:
|
|
51
|
+
rtype, rid = "other", res
|
|
52
|
+
elif "/" in res:
|
|
53
|
+
rtype, rid = res.split("/", 1)
|
|
54
|
+
elif ":" in res:
|
|
55
|
+
rtype, rid = res.split(":", 1)
|
|
56
|
+
t = tags or {}
|
|
57
|
+
name = t.get("Name") or rid
|
|
58
|
+
return Resource(arn=arn, service=service, rtype=rtype, rid=rid, region=region or "global", name=name, tags=t)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def discover(ctx: Ctx) -> List[Resource]:
|
|
62
|
+
cfg = ctx.cfg
|
|
63
|
+
regions = list(dict.fromkeys(list(cfg.regions) + ["us-east-1"])) # us-east-1 hosts global services
|
|
64
|
+
seen: Dict[str, Resource] = {}
|
|
65
|
+
for region in regions:
|
|
66
|
+
c = ctx.client("resourcegroupstaggingapi", region)
|
|
67
|
+
pager = c.get_paginator("get_resources")
|
|
68
|
+
for page in pager.paginate(TagFilters=[{"Key": cfg.workshop_tag, "Values": [cfg.workshop]}],
|
|
69
|
+
ResourcesPerPage=100):
|
|
70
|
+
for m in page["ResourceTagMappingList"]:
|
|
71
|
+
tags = {t["Key"]: t["Value"] for t in m.get("Tags", [])}
|
|
72
|
+
r = parse_arn(m["ResourceARN"], tags)
|
|
73
|
+
if r.region == "global":
|
|
74
|
+
r.region = "us-east-1"
|
|
75
|
+
seen[r.arn] = r
|
|
76
|
+
for r in _ec2_extras(ctx, regions):
|
|
77
|
+
seen.setdefault(r.arn, r)
|
|
78
|
+
return sorted(seen.values(), key=lambda r: (r.service, r.rtype, r.arn))
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _ec2_extras(ctx: Ctx, regions: List[str]) -> List[Resource]:
|
|
82
|
+
"""EC2 types read directly by tag filter: the Tagging API is eventually consistent and does not
|
|
83
|
+
reliably return every EC2 type (Elastic IPs, volumes, security groups), and leaked EIPs cost money."""
|
|
84
|
+
cfg, out = ctx.cfg, []
|
|
85
|
+
flt = [{"Name": f"tag:{cfg.workshop_tag}", "Values": [cfg.workshop]}]
|
|
86
|
+
for region in regions:
|
|
87
|
+
ec2 = ctx.client("ec2", region)
|
|
88
|
+
acct = ctx.account_id
|
|
89
|
+
|
|
90
|
+
def add(rtype, rid, tags):
|
|
91
|
+
t = {x["Key"]: x["Value"] for x in tags or []}
|
|
92
|
+
out.append(parse_arn(f"arn:aws:ec2:{region}:{acct}:{rtype}/{rid}", t))
|
|
93
|
+
for a in ec2.describe_addresses(Filters=flt)["Addresses"]:
|
|
94
|
+
if "AllocationId" in a:
|
|
95
|
+
add("elastic-ip", a["AllocationId"], a.get("Tags"))
|
|
96
|
+
for page in ec2.get_paginator("describe_volumes").paginate(Filters=flt):
|
|
97
|
+
for v in page["Volumes"]:
|
|
98
|
+
add("volume", v["VolumeId"], v.get("Tags"))
|
|
99
|
+
for page in ec2.get_paginator("describe_security_groups").paginate(Filters=flt):
|
|
100
|
+
for g in page["SecurityGroups"]:
|
|
101
|
+
if g["GroupName"] != "default":
|
|
102
|
+
add("security-group", g["GroupId"], g.get("Tags"))
|
|
103
|
+
return out
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass
|
|
107
|
+
class Scope:
|
|
108
|
+
cohort_id: Optional[str] = None
|
|
109
|
+
cohort_date: Optional[str] = None
|
|
110
|
+
all_cohorts: bool = False
|
|
111
|
+
|
|
112
|
+
def validate(self) -> None:
|
|
113
|
+
if not (self.cohort_id or self.cohort_date or self.all_cohorts):
|
|
114
|
+
raise ValueError("scope requires a cohort id/date or explicit all_cohorts")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def tag_get(tags: Dict[str, str], key: str) -> Optional[str]:
|
|
118
|
+
lk = key.lower()
|
|
119
|
+
for k, v in tags.items():
|
|
120
|
+
if k.lower() == lk:
|
|
121
|
+
return v
|
|
122
|
+
return None
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def classify(res: Resource, cfg: Config, scope: Scope, supported: Iterable[Tuple[str, str]]) -> Tuple[str, Optional[str]]:
|
|
126
|
+
"""Return ('delete', None) or ('skip', reason). Skips are always reported, never silent."""
|
|
127
|
+
if (res.service, res.rtype) not in set(supported):
|
|
128
|
+
return "skip", f"unsupported:{res.service}:{res.rtype}"
|
|
129
|
+
if any(res.name.startswith(p) or res.rid.startswith(p) for p in cfg.protected_name_prefixes):
|
|
130
|
+
return "skip", "protected-name"
|
|
131
|
+
if (tag_get(res.tags, cfg.autodelete_tag) or "").strip().lower() == "false":
|
|
132
|
+
return "skip", "autodelete=false"
|
|
133
|
+
if not scope.all_cohorts:
|
|
134
|
+
cohort = tag_get(res.tags, cfg.cohort_tag)
|
|
135
|
+
date = tag_get(res.tags, cfg.date_tag)
|
|
136
|
+
match = (scope.cohort_id and cohort == scope.cohort_id) or (scope.cohort_date and date == scope.cohort_date)
|
|
137
|
+
if not match:
|
|
138
|
+
return "skip", "other-cohort"
|
|
139
|
+
return "delete", None
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def attribute(res: Resource, cfg: Config, slugs: Dict[str, str]) -> Optional[str]:
|
|
143
|
+
"""Best-effort owner (learner_id). Owner tags win, then a slug match on the resource name at token boundaries."""
|
|
144
|
+
for k in cfg.owner_tag_keys:
|
|
145
|
+
v = tag_get(res.tags, k)
|
|
146
|
+
if v:
|
|
147
|
+
for lid, slug in slugs.items():
|
|
148
|
+
if v == lid or v == slug:
|
|
149
|
+
return lid
|
|
150
|
+
name = res.name.lower()
|
|
151
|
+
for lid, slug in slugs.items():
|
|
152
|
+
if re.search(rf"(^|[^a-z0-9]){re.escape(slug.lower())}($|[^a-z0-9])", name):
|
|
153
|
+
return lid
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def slugs_from_roster(learners: List[Dict[str, str]]) -> Dict[str, str]:
|
|
158
|
+
out = {}
|
|
159
|
+
for l in learners:
|
|
160
|
+
u = l.get("username", "")
|
|
161
|
+
out[str(l["learner_id"])] = u.split("@")[0]
|
|
162
|
+
return out
|