labagent 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- labagent/__init__.py +3 -0
- labagent/audit.py +233 -0
- labagent/cleanup.py +67 -0
- labagent/cli.py +63 -0
- labagent/config.py +82 -0
- labagent/context.py +51 -0
- labagent/deleters.py +507 -0
- labagent/discovery.py +162 -0
- labagent/envelope.py +48 -0
- labagent/iam_ops.py +133 -0
- labagent/plan.py +40 -0
- labagent/rules.py +153 -0
- labagent/run.py +58 -0
- labagent/secrets_box.py +36 -0
- labagent/sweep.py +107 -0
- labagent/terraform/cohort_iam/.terraform.lock.hcl +29 -0
- labagent/terraform/cohort_iam/main.tf +101 -0
- labagent/terraform/cohort_iam/outputs.tf +21 -0
- labagent/terraform/cohort_iam/sandbox-policy.json.tftpl +33 -0
- labagent/terraform/cohort_iam/tests/cohort.tftest.hcl +63 -0
- labagent/terraform/cohort_iam/variables.tf +55 -0
- labagent/terraform/cohort_iam/versions.tf +15 -0
- labagent/terraform/state_backend/.terraform.lock.hcl +26 -0
- labagent/terraform/state_backend/main.tf +193 -0
- labagent/terraform/state_backend/tests/bootstrap.tftest.hcl +37 -0
- labagent/tf.py +118 -0
- labagent-0.1.0.dist-info/METADATA +73 -0
- labagent-0.1.0.dist-info/RECORD +32 -0
- labagent-0.1.0.dist-info/WHEEL +5 -0
- labagent-0.1.0.dist-info/entry_points.txt +3 -0
- labagent-0.1.0.dist-info/licenses/LICENSE +202 -0
- labagent-0.1.0.dist-info/top_level.txt +1 -0
labagent/__init__.py
ADDED
labagent/audit.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
"""Full-account discovery by listing each service (not by tag), for cleaning up accounts that were never tagged.
|
|
2
|
+
|
|
3
|
+
Read-only. Complements discovery.py, which finds tagged lab resources through the Tagging API.
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import concurrent.futures as cf
|
|
8
|
+
from typing import Callable, Dict, List, Optional
|
|
9
|
+
|
|
10
|
+
from botocore.exceptions import ClientError, EndpointConnectionError
|
|
11
|
+
|
|
12
|
+
from .context import Ctx
|
|
13
|
+
from .discovery import Resource
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def synthetic_arn(service: str, rtype: str, region: str, rid: str) -> str:
|
|
17
|
+
return f"labagent:{service}/{rtype}/{region}/{rid}"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _tags(raw) -> Dict[str, str]:
|
|
21
|
+
if isinstance(raw, dict):
|
|
22
|
+
return {str(k): str(v) for k, v in raw.items()}
|
|
23
|
+
return {str(x.get("Key", x.get("key"))): str(x.get("Value", x.get("value"))) for x in (raw or [])}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _mk(service, rtype, region, rid, name=None, tags=None, **attrs) -> Resource:
|
|
27
|
+
return Resource(arn=synthetic_arn(service, rtype, region, rid), service=service, rtype=rtype, rid=rid,
|
|
28
|
+
region=region, name=name or rid, tags=_tags(tags), attrs={k: v for k, v in attrs.items() if v is not None})
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _regional(ctx: Ctx, region: str, out: List[Resource], errors: List[str]) -> None:
|
|
32
|
+
c = lambda s: ctx.client(s, region)
|
|
33
|
+
|
|
34
|
+
def lam():
|
|
35
|
+
for p in c("lambda").get_paginator("list_functions").paginate():
|
|
36
|
+
for f in p["Functions"]:
|
|
37
|
+
tg = c("lambda").list_tags(Resource=f["FunctionArn"]).get("Tags", {})
|
|
38
|
+
out.append(_mk("lambda", "function", region, f["FunctionName"], tags=tg, created=f["LastModified"]))
|
|
39
|
+
|
|
40
|
+
def apigw():
|
|
41
|
+
for p in c("apigateway").get_paginator("get_rest_apis").paginate():
|
|
42
|
+
for a in p["items"]:
|
|
43
|
+
out.append(_mk("apigateway", "restapi", region, a["id"], a.get("name"), a.get("tags"),
|
|
44
|
+
created=str(a.get("createdDate"))))
|
|
45
|
+
for a in c("apigatewayv2").get_apis().get("Items", []):
|
|
46
|
+
out.append(_mk("apigateway", "httpapi", region, a["ApiId"], a.get("Name"), a.get("Tags"),
|
|
47
|
+
created=str(a.get("CreatedDate"))))
|
|
48
|
+
|
|
49
|
+
def ddb():
|
|
50
|
+
for p in c("dynamodb").get_paginator("list_tables").paginate():
|
|
51
|
+
for t in p["TableNames"]:
|
|
52
|
+
arn = c("dynamodb").describe_table(TableName=t)["Table"]["TableArn"]
|
|
53
|
+
out.append(_mk("dynamodb", "table", region, t, tags=c("dynamodb").list_tags_of_resource(ResourceArn=arn).get("Tags")))
|
|
54
|
+
|
|
55
|
+
def ec2():
|
|
56
|
+
e = c("ec2")
|
|
57
|
+
for p in e.get_paginator("describe_instances").paginate():
|
|
58
|
+
for r in p["Reservations"]:
|
|
59
|
+
for i in r["Instances"]:
|
|
60
|
+
if i["State"]["Name"] != "terminated":
|
|
61
|
+
nm = next((x["Value"] for x in i.get("Tags", []) if x["Key"] == "Name"), None)
|
|
62
|
+
out.append(_mk("ec2", "instance", region, i["InstanceId"], nm, i.get("Tags"),
|
|
63
|
+
state=i["State"]["Name"], created=str(i["LaunchTime"])))
|
|
64
|
+
for v in e.describe_volumes()["Volumes"]:
|
|
65
|
+
out.append(_mk("ec2", "volume", region, v["VolumeId"], tags=v.get("Tags"), attached=len(v["Attachments"]) > 0,
|
|
66
|
+
created=str(v.get("CreateTime"))))
|
|
67
|
+
for a in e.describe_addresses()["Addresses"]:
|
|
68
|
+
out.append(_mk("ec2", "elastic-ip", region, a.get("AllocationId", a.get("PublicIp")), tags=a.get("Tags"),
|
|
69
|
+
associated="AssociationId" in a))
|
|
70
|
+
for g in e.describe_security_groups()["SecurityGroups"]:
|
|
71
|
+
if g["GroupName"] != "default":
|
|
72
|
+
out.append(_mk("ec2", "security-group", region, g["GroupId"], g["GroupName"], g.get("Tags")))
|
|
73
|
+
for k in e.describe_key_pairs()["KeyPairs"]:
|
|
74
|
+
out.append(_mk("ec2", "key-pair", region, k["KeyName"], tags=k.get("Tags"), created=str(k.get("CreateTime"))))
|
|
75
|
+
for n in e.describe_nat_gateways()["NatGateways"]:
|
|
76
|
+
if n["State"] != "deleted":
|
|
77
|
+
out.append(_mk("ec2", "nat-gateway", region, n["NatGatewayId"], tags=n.get("Tags"), state=n["State"]))
|
|
78
|
+
for v in e.describe_vpcs()["Vpcs"]:
|
|
79
|
+
nm = next((x["Value"] for x in v.get("Tags", []) if x["Key"] == "Name"), None)
|
|
80
|
+
# default VPCs are listed too (so they are visible) but only an explicit rule can remove them
|
|
81
|
+
out.append(_mk("ec2", "vpc", region, v["VpcId"], nm, v.get("Tags"), is_default=bool(v.get("IsDefault")),
|
|
82
|
+
cidr=v.get("CidrBlock")))
|
|
83
|
+
|
|
84
|
+
def elb():
|
|
85
|
+
for lb in c("elbv2").describe_load_balancers()["LoadBalancers"]:
|
|
86
|
+
out.append(_mk("elbv2", "load-balancer", region, lb["LoadBalancerName"], created=str(lb["CreatedTime"])))
|
|
87
|
+
|
|
88
|
+
def logs():
|
|
89
|
+
for p in c("logs").get_paginator("describe_log_groups").paginate():
|
|
90
|
+
for g in p["logGroups"]:
|
|
91
|
+
out.append(_mk("logs", "log-group", region, g["logGroupName"], created=str(g.get("creationTime")),
|
|
92
|
+
bytes=g.get("storedBytes", 0)))
|
|
93
|
+
|
|
94
|
+
def ecr():
|
|
95
|
+
for p in c("ecr").get_paginator("describe_repositories").paginate():
|
|
96
|
+
for r in p["repositories"]:
|
|
97
|
+
out.append(_mk("ecr", "repository", region, r["repositoryName"]))
|
|
98
|
+
|
|
99
|
+
def cfn():
|
|
100
|
+
for p in c("cloudformation").get_paginator("describe_stacks").paginate():
|
|
101
|
+
for s in p["Stacks"]:
|
|
102
|
+
out.append(_mk("cloudformation", "stack", region, s["StackName"], tags=s.get("Tags"),
|
|
103
|
+
status=s["StackStatus"], created=str(s.get("CreationTime"))))
|
|
104
|
+
|
|
105
|
+
def rds():
|
|
106
|
+
for d in c("rds").describe_db_instances()["DBInstances"]:
|
|
107
|
+
out.append(_mk("rds", "db-instance", region, d["DBInstanceIdentifier"], status=d["DBInstanceStatus"]))
|
|
108
|
+
|
|
109
|
+
def eks():
|
|
110
|
+
for a in c("ecs").list_clusters()["clusterArns"]:
|
|
111
|
+
out.append(_mk("ecs", "cluster", region, a.split("/")[-1]))
|
|
112
|
+
for a in c("eks").list_clusters()["clusters"]:
|
|
113
|
+
out.append(_mk("eks", "cluster", region, a))
|
|
114
|
+
|
|
115
|
+
def sec():
|
|
116
|
+
for p in c("secretsmanager").get_paginator("list_secrets").paginate():
|
|
117
|
+
for s in p["SecretList"]:
|
|
118
|
+
out.append(_mk("secretsmanager", "secret", region, s["Name"], tags=s.get("Tags")))
|
|
119
|
+
|
|
120
|
+
def sns():
|
|
121
|
+
for p in c("sns").get_paginator("list_topics").paginate():
|
|
122
|
+
for t in p["Topics"]:
|
|
123
|
+
out.append(_mk("sns", "topic", region, t["TopicArn"].split(":")[-1]))
|
|
124
|
+
for q in c("sqs").list_queues().get("QueueUrls", []):
|
|
125
|
+
out.append(_mk("sqs", "queue", region, q.split("/")[-1]))
|
|
126
|
+
|
|
127
|
+
def layers():
|
|
128
|
+
for p in c("lambda").get_paginator("list_layers").paginate():
|
|
129
|
+
for l in p["Layers"]:
|
|
130
|
+
out.append(_mk("lambda", "layer", region, l["LayerName"], versions=l["LatestMatchingVersion"]["Version"],
|
|
131
|
+
created=l["LatestMatchingVersion"].get("CreatedDate")))
|
|
132
|
+
|
|
133
|
+
def glue():
|
|
134
|
+
g = c("glue")
|
|
135
|
+
for p in g.get_paginator("get_databases").paginate():
|
|
136
|
+
for d in p["DatabaseList"]:
|
|
137
|
+
out.append(_mk("glue", "database", region, d["Name"], created=str(d.get("CreateTime"))))
|
|
138
|
+
for p in g.get_paginator("get_crawlers").paginate():
|
|
139
|
+
for d in p["Crawlers"]:
|
|
140
|
+
out.append(_mk("glue", "crawler", region, d["Name"], state=d.get("State")))
|
|
141
|
+
for p in g.get_paginator("get_jobs").paginate():
|
|
142
|
+
for d in p["Jobs"]:
|
|
143
|
+
out.append(_mk("glue", "job", region, d["Name"], created=str(d.get("CreatedOn"))))
|
|
144
|
+
|
|
145
|
+
def athena():
|
|
146
|
+
for w in c("athena").list_work_groups()["WorkGroups"]:
|
|
147
|
+
if w["Name"] != "primary":
|
|
148
|
+
out.append(_mk("athena", "workgroup", region, w["Name"], created=str(w.get("CreationTime"))))
|
|
149
|
+
|
|
150
|
+
def alarms():
|
|
151
|
+
for p in c("cloudwatch").get_paginator("describe_alarms").paginate():
|
|
152
|
+
for a in p["MetricAlarms"]:
|
|
153
|
+
out.append(_mk("cloudwatch", "alarm", region, a["AlarmName"], created=str(a.get("AlarmConfigurationUpdatedTimestamp"))))
|
|
154
|
+
|
|
155
|
+
for fn in (lam, apigw, ddb, ec2, elb, logs, ecr, cfn, rds, eks, sec, sns, layers, glue, athena, alarms):
|
|
156
|
+
try:
|
|
157
|
+
fn()
|
|
158
|
+
except (ClientError, EndpointConnectionError) as e:
|
|
159
|
+
code = getattr(e, "response", {}).get("Error", {}).get("Code", type(e).__name__)
|
|
160
|
+
errors.append(f"{region}:{fn.__name__}:{code}")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _global(ctx: Ctx, out: List[Resource], errors: List[str]) -> None:
|
|
164
|
+
def s3():
|
|
165
|
+
c = ctx.client("s3")
|
|
166
|
+
for b in c.list_buckets()["Buckets"]:
|
|
167
|
+
try:
|
|
168
|
+
tg = c.get_bucket_tagging(Bucket=b["Name"])["TagSet"]
|
|
169
|
+
except ClientError:
|
|
170
|
+
tg = []
|
|
171
|
+
try:
|
|
172
|
+
loc = c.get_bucket_location(Bucket=b["Name"])["LocationConstraint"] or "us-east-1"
|
|
173
|
+
except ClientError:
|
|
174
|
+
loc = ctx.cfg.region
|
|
175
|
+
out.append(_mk("s3", "bucket", loc, b["Name"], tags=tg, created=str(b["CreationDate"])))
|
|
176
|
+
|
|
177
|
+
def cloudfront():
|
|
178
|
+
c = ctx.client("cloudfront", "us-east-1")
|
|
179
|
+
for p in c.get_paginator("list_distributions").paginate():
|
|
180
|
+
for d in (p.get("DistributionList", {}).get("Items") or []):
|
|
181
|
+
tg = c.list_tags_for_resource(Resource=d["ARN"])["Tags"]["Items"]
|
|
182
|
+
out.append(_mk("cloudfront", "distribution", "us-east-1", d["Id"], d["DomainName"], tg,
|
|
183
|
+
enabled=d["Enabled"], status=d["Status"], created=str(d.get("LastModifiedTime")), aliases=len(d.get("Aliases", {}).get("Items", []) or []),
|
|
184
|
+
comment=d.get("Comment", "")))
|
|
185
|
+
|
|
186
|
+
def iam():
|
|
187
|
+
c = ctx.client("iam")
|
|
188
|
+
for p in c.get_paginator("list_users").paginate():
|
|
189
|
+
for u in p["Users"]:
|
|
190
|
+
out.append(_mk("iam", "user", "global", u["UserName"], tags=c.list_user_tags(UserName=u["UserName"]).get("Tags"),
|
|
191
|
+
created=str(u["CreateDate"]), last_used=str(u["PasswordLastUsed"]) if u.get("PasswordLastUsed") else None))
|
|
192
|
+
for p in c.get_paginator("list_roles").paginate():
|
|
193
|
+
for r in p["Roles"]:
|
|
194
|
+
if not r["Path"].startswith("/aws-service-role/"):
|
|
195
|
+
out.append(_mk("iam", "role", "global", r["RoleName"], tags=r.get("Tags"), created=str(r["CreateDate"]),
|
|
196
|
+
last_used=str(r.get("RoleLastUsed", {}).get("LastUsedDate")) if r.get("RoleLastUsed", {}).get("LastUsedDate") else None))
|
|
197
|
+
for p in c.get_paginator("list_groups").paginate():
|
|
198
|
+
for g in p["Groups"]:
|
|
199
|
+
out.append(_mk("iam", "group", "global", g["GroupName"], created=str(g["CreateDate"])))
|
|
200
|
+
for p in c.get_paginator("list_policies").paginate(Scope="Local"):
|
|
201
|
+
for g in p["Policies"]:
|
|
202
|
+
r = _mk("iam", "policy", "global", g["PolicyName"], created=str(g["CreateDate"]), attachments=g["AttachmentCount"])
|
|
203
|
+
r.arn = g["Arn"]
|
|
204
|
+
out.append(r)
|
|
205
|
+
|
|
206
|
+
def dashboards():
|
|
207
|
+
# CloudWatch dashboards are account-wide: list once, always through us-east-1
|
|
208
|
+
cw = ctx.client("cloudwatch", "us-east-1")
|
|
209
|
+
for p in cw.get_paginator("list_dashboards").paginate():
|
|
210
|
+
for d in p["DashboardEntries"]:
|
|
211
|
+
out.append(_mk("cloudwatch", "dashboard", "us-east-1", d["DashboardName"], created=str(d.get("LastModified"))))
|
|
212
|
+
|
|
213
|
+
for fn in (s3, cloudfront, iam, dashboards):
|
|
214
|
+
try:
|
|
215
|
+
fn()
|
|
216
|
+
except (ClientError, EndpointConnectionError) as e:
|
|
217
|
+
code = getattr(e, "response", {}).get("Error", {}).get("Code", type(e).__name__)
|
|
218
|
+
errors.append(f"global:{fn.__name__}:{code}")
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def discover_all(ctx: Ctx, regions: Optional[List[str]] = None, workers: int = 12) -> "tuple[List[Resource], List[str]]":
|
|
222
|
+
"""Returns (resources, errors). `regions` defaults to config.regions; pass ['*'] for every enabled region."""
|
|
223
|
+
regs = regions or ctx.cfg.regions
|
|
224
|
+
if regs == ["*"]:
|
|
225
|
+
regs = [r["RegionName"] for r in ctx.client("ec2").describe_regions()["Regions"]]
|
|
226
|
+
out: List[Resource] = []
|
|
227
|
+
errors: List[str] = []
|
|
228
|
+
with cf.ThreadPoolExecutor(workers) as ex:
|
|
229
|
+
futs = [ex.submit(_regional, ctx, r, out, errors) for r in regs]
|
|
230
|
+
futs.append(ex.submit(_global, ctx, out, errors))
|
|
231
|
+
for f in futs:
|
|
232
|
+
f.result()
|
|
233
|
+
return sorted(out, key=lambda r: (r.service, r.rtype, r.region, r.rid)), sorted(errors)
|
labagent/cleanup.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Ruleset-driven audit and cleanup of an existing account. Plan, then apply the plan with sweep_apply."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
from collections import Counter
|
|
5
|
+
from typing import Any, Dict, Optional
|
|
6
|
+
|
|
7
|
+
from .audit import discover_all
|
|
8
|
+
from .context import Ctx, log
|
|
9
|
+
from .deleters import REGISTRY, cleanup_types
|
|
10
|
+
from .envelope import Envelope
|
|
11
|
+
from .plan import build_plan
|
|
12
|
+
from .rules import RulesetError, decide, validate
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _decide_all(ctx: Ctx, ruleset: Dict[str, Any], regions=None):
|
|
16
|
+
validate(ruleset)
|
|
17
|
+
resources, errors = discover_all(ctx, regions)
|
|
18
|
+
supported = set(cleanup_types())
|
|
19
|
+
rows = []
|
|
20
|
+
for r in resources:
|
|
21
|
+
d = decide(r, ruleset, ctx.cfg)
|
|
22
|
+
action, reason = d.action, d.reason
|
|
23
|
+
if action == "delete" and (r.service, r.rtype) not in supported:
|
|
24
|
+
action, reason = "review", f"matched rule {d.rule} but labagent has no deleter for {r.service}/{r.rtype}"
|
|
25
|
+
rows.append({"arn": r.arn, "service": r.service, "type": r.rtype, "id": r.rid, "region": r.region,
|
|
26
|
+
"name": r.name, "user": None, "action": action, "rule": d.rule, "reason": reason,
|
|
27
|
+
"state": "planned"})
|
|
28
|
+
return rows, errors
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _summary(rows) -> Dict[str, Any]:
|
|
32
|
+
by_action = Counter(r["action"] for r in rows)
|
|
33
|
+
by_type: Dict[str, Dict[str, int]] = {}
|
|
34
|
+
for r in rows:
|
|
35
|
+
by_type.setdefault(f"{r['service']}/{r['type']}", Counter())[r["action"]] += 1
|
|
36
|
+
return {"total": len(rows), "by_action": dict(by_action), "by_type": {k: dict(v) for k, v in sorted(by_type.items())}}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def audit(ctx: Ctx, ruleset: Optional[Dict[str, Any]], regions=None, run_id: Optional[str] = None) -> Envelope:
|
|
40
|
+
"""Read-only. Lists everything and what the ruleset would do with it. No plan is created."""
|
|
41
|
+
env = Envelope(command="audit", mode="read", run_id=run_id, cohort=(ruleset or {}).get("name"))
|
|
42
|
+
try:
|
|
43
|
+
rows, errors = _decide_all(ctx, ruleset or {"version": 1, "rules": []}, regions)
|
|
44
|
+
except RulesetError as e:
|
|
45
|
+
return env.fail(f"ruleset invalid: {e}")
|
|
46
|
+
env.items, env.warnings = rows, [f"discovery error: {e}" for e in errors]
|
|
47
|
+
for i in env.items:
|
|
48
|
+
i["state"] = "listed"
|
|
49
|
+
env.summary = _summary(rows)
|
|
50
|
+
log(f"audit: {len(rows)} resources, {env.summary['by_action']}")
|
|
51
|
+
return env.finalize()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def cleanup_plan(ctx: Ctx, ruleset: Dict[str, Any], regions=None, run_id: Optional[str] = None) -> Envelope:
|
|
55
|
+
ctx.guard_account()
|
|
56
|
+
name = ruleset.get("name", "cleanup")
|
|
57
|
+
env = Envelope(command="cleanup", mode="plan", run_id=run_id, cohort=f"cleanup:{name}")
|
|
58
|
+
try:
|
|
59
|
+
rows, errors = _decide_all(ctx, ruleset, regions)
|
|
60
|
+
except RulesetError as e:
|
|
61
|
+
return env.fail(f"ruleset invalid: {e}")
|
|
62
|
+
env.items = rows
|
|
63
|
+
env.warnings = [f"discovery error: {e}" for e in errors]
|
|
64
|
+
env.plan = build_plan(env.cohort, ctx.account_id, rows, ctx.clock())
|
|
65
|
+
env.summary = _summary(rows)
|
|
66
|
+
log(f"cleanup plan {env.plan['id']}: {env.summary['by_action']}")
|
|
67
|
+
return env
|
labagent/cli.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""labagent command line. stdout is silent unless --output - is given; logs go to stderr (ids and counts only)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import boto3
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .config import STATE_BACKEND_DIR, load_config
|
|
13
|
+
from .context import Ctx, GuardError
|
|
14
|
+
from .envelope import Envelope
|
|
15
|
+
from .run import COMMANDS, run_job
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _emit(env: Envelope, output: str) -> None:
|
|
19
|
+
text = json.dumps(env.to_dict(), indent=2)
|
|
20
|
+
if output == "-":
|
|
21
|
+
print(text)
|
|
22
|
+
else:
|
|
23
|
+
Path(output).write_text(text)
|
|
24
|
+
Path(output).chmod(0o600)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def main(argv=None) -> int:
|
|
28
|
+
p = argparse.ArgumentParser(prog="labagent", description=__doc__)
|
|
29
|
+
p.add_argument("--version", action="version", version=__version__)
|
|
30
|
+
p.add_argument("--config", help="JSON config file (or LABAGENT_CONFIG)")
|
|
31
|
+
p.add_argument("--profile", help="AWS profile for local use. CI uses OIDC environment credentials.")
|
|
32
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
33
|
+
r = sub.add_parser("run", help="Execute a job document: the CI entry point")
|
|
34
|
+
r.add_argument("--job", required=True, help="Path to job JSON (contains roster: keep it out of git and logs)")
|
|
35
|
+
r.add_argument("--output", default="result.json", help="Result envelope path, or - for stdout")
|
|
36
|
+
r.add_argument("--ruleset", help="Path to a ruleset JSON for audit/cleanup (merged into the job)")
|
|
37
|
+
sub.add_parser("bootstrap-path", help="Print the path of the packaged Terraform state-backend module")
|
|
38
|
+
sub.add_parser("commands", help="List supported job commands")
|
|
39
|
+
args = p.parse_args(argv)
|
|
40
|
+
|
|
41
|
+
if args.cmd == "bootstrap-path":
|
|
42
|
+
print(STATE_BACKEND_DIR)
|
|
43
|
+
return 0
|
|
44
|
+
if args.cmd == "commands":
|
|
45
|
+
print("\n".join(sorted(COMMANDS)))
|
|
46
|
+
return 0
|
|
47
|
+
try:
|
|
48
|
+
cfg = load_config(args.config)
|
|
49
|
+
job = json.loads(Path(args.job).read_text())
|
|
50
|
+
session = boto3.Session(profile_name=args.profile, region_name=cfg.region)
|
|
51
|
+
if getattr(args, "ruleset", None) and "ruleset" not in job:
|
|
52
|
+
job["ruleset"] = json.loads(Path(args.ruleset).read_text())
|
|
53
|
+
env = run_job(Ctx(cfg, session), job)
|
|
54
|
+
except GuardError as e:
|
|
55
|
+
env = Envelope(command="guard").fail(str(e))
|
|
56
|
+
except Exception as e:
|
|
57
|
+
env = Envelope(command="startup").fail(f"{type(e).__name__}: {str(e)[:200]}")
|
|
58
|
+
_emit(env, args.output)
|
|
59
|
+
return env.exit_code
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
if __name__ == "__main__":
|
|
63
|
+
sys.exit(main())
|
labagent/config.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Configuration. No organization-specific value is hard-coded in the engine."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from dataclasses import dataclass, field, fields
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
9
|
+
|
|
10
|
+
MODULE_DIR = Path(__file__).parent / "terraform" / "cohort_iam"
|
|
11
|
+
STATE_BACKEND_DIR = Path(__file__).parent / "terraform" / "state_backend"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def getenv(name: str) -> Optional[str]:
|
|
15
|
+
"""LABAGENT_<name>, falling back to the legacy LABCTL_<name> used before the rename."""
|
|
16
|
+
return os.environ.get(f"LABAGENT_{name}") or os.environ.get(f"LABCTL_{name}") or None
|
|
17
|
+
# Keys from the earlier boto3-only design: accepted and ignored so old config files keep loading.
|
|
18
|
+
LEGACY_KEYS = {"group_name", "policy_name", "lambda_role_name", "policy_file"}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class Config:
|
|
23
|
+
region: str = "us-east-1"
|
|
24
|
+
# Regions scanned for regional resources. CloudFront (global) is always read via us-east-1.
|
|
25
|
+
regions: List[str] = field(default_factory=lambda: ["us-east-1"])
|
|
26
|
+
# Tag keys
|
|
27
|
+
workshop_tag: str = "workshop"
|
|
28
|
+
autodelete_tag: str = "autodelete"
|
|
29
|
+
cohort_tag: str = "cohort"
|
|
30
|
+
date_tag: str = "date"
|
|
31
|
+
owner_tag_keys: Tuple[str, ...] = ("owner", "student", "learner_id")
|
|
32
|
+
# The workshop tag VALUE that marks lab resources, e.g. "full-stack"
|
|
33
|
+
workshop: str = "full-stack"
|
|
34
|
+
# IAM naming: each cohort gets <name_prefix>-<cohort>-students / -sandbox / -lambda-exec
|
|
35
|
+
name_prefix: str = "labs"
|
|
36
|
+
student_domain: str = "example.com"
|
|
37
|
+
# Initial console passwords: force a change at first sign-in? Set false when the LMS is the
|
|
38
|
+
# place learners read their password, so the stored value stays valid.
|
|
39
|
+
password_reset_required: bool = True
|
|
40
|
+
# Terraform: binary (terraform or tofu) and the S3 backend. Values come from config/env, never from the repo.
|
|
41
|
+
terraform_bin: Optional[str] = None
|
|
42
|
+
backend: Dict[str, Any] = field(default_factory=dict) # bucket, region, kms_key_id, use_lockfile, key_template
|
|
43
|
+
# Safety
|
|
44
|
+
allowed_account_ids: List[str] = field(default_factory=list)
|
|
45
|
+
protected_name_prefixes: Tuple[str, ...] = (
|
|
46
|
+
"aws-cloudtrail-logs-", "aws-config-", "cf-templates-", "elasticbeanstalk-",
|
|
47
|
+
"amplify-", "cdk-", "codepipeline-", "do-not-delete-",
|
|
48
|
+
)
|
|
49
|
+
plan_max_age_s: int = 6 * 3600
|
|
50
|
+
apply_time_budget_s: int = 1500
|
|
51
|
+
password_length: int = 20
|
|
52
|
+
|
|
53
|
+
def backend_settings(self) -> Dict[str, Any]:
|
|
54
|
+
"""Backend config with env overrides (so a private runner can set values as CI variables)."""
|
|
55
|
+
b = {"key_template": "cohorts/{cohort}/terraform.tfstate", "use_lockfile": True, **self.backend}
|
|
56
|
+
for env, key in (("TF_BACKEND_BUCKET", "bucket"), ("TF_BACKEND_REGION", "region"),
|
|
57
|
+
("TF_BACKEND_KMS_KEY_ID", "kms_key_id")):
|
|
58
|
+
if getenv(env):
|
|
59
|
+
b[key] = getenv(env)
|
|
60
|
+
return b
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def load_config(path: Optional[str] = None) -> Config:
|
|
64
|
+
"""Load JSON config. Path from argument or LABAGENT_CONFIG; env LABAGENT_ALLOWED_ACCOUNTS overrides the allow-list."""
|
|
65
|
+
path = path or getenv("CONFIG")
|
|
66
|
+
raw = {}
|
|
67
|
+
if path:
|
|
68
|
+
raw = json.loads(Path(path).read_text())
|
|
69
|
+
known = {f.name for f in fields(Config)}
|
|
70
|
+
for k in LEGACY_KEYS & set(raw):
|
|
71
|
+
raw.pop(k)
|
|
72
|
+
unknown = set(raw) - known
|
|
73
|
+
if unknown:
|
|
74
|
+
raise ValueError(f"unknown config keys: {sorted(unknown)}")
|
|
75
|
+
for k in ("protected_name_prefixes", "owner_tag_keys"):
|
|
76
|
+
if k in raw:
|
|
77
|
+
raw[k] = tuple(raw[k])
|
|
78
|
+
cfg = Config(**raw)
|
|
79
|
+
env_accounts = getenv("ALLOWED_ACCOUNTS")
|
|
80
|
+
if env_accounts:
|
|
81
|
+
cfg.allowed_account_ids = [a.strip() for a in env_accounts.split(",") if a.strip()]
|
|
82
|
+
return cfg
|
labagent/context.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Execution context: AWS session, config, account guard, injectable sleep."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import sys
|
|
5
|
+
import time
|
|
6
|
+
from typing import Any, Callable, Dict, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
import boto3
|
|
9
|
+
|
|
10
|
+
from .config import Config
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class GuardError(Exception):
|
|
14
|
+
pass
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def log(msg: str) -> None:
|
|
18
|
+
"""Logs go to stderr and must never contain names or emails: ids and counts only."""
|
|
19
|
+
print(msg, file=sys.stderr, flush=True)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Ctx:
|
|
23
|
+
def __init__(self, cfg: Config, session: Optional[boto3.Session] = None,
|
|
24
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
25
|
+
clock: Callable[[], float] = time.time):
|
|
26
|
+
self.cfg = cfg
|
|
27
|
+
self.session = session or boto3.Session(region_name=cfg.region)
|
|
28
|
+
self.sleep = sleep
|
|
29
|
+
self.clock = clock
|
|
30
|
+
self._clients: Dict[Tuple[str, str], Any] = {}
|
|
31
|
+
self._account: Optional[str] = None
|
|
32
|
+
|
|
33
|
+
def client(self, service: str, region: Optional[str] = None):
|
|
34
|
+
key = (service, region or self.cfg.region)
|
|
35
|
+
if key not in self._clients:
|
|
36
|
+
self._clients[key] = self.session.client(service, region_name=key[1])
|
|
37
|
+
return self._clients[key]
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def account_id(self) -> str:
|
|
41
|
+
if self._account is None:
|
|
42
|
+
self._account = self.client("sts").get_caller_identity()["Account"]
|
|
43
|
+
return self._account
|
|
44
|
+
|
|
45
|
+
def guard_account(self) -> None:
|
|
46
|
+
"""Refuse to run against any account that is not explicitly allowed."""
|
|
47
|
+
allowed = self.cfg.allowed_account_ids
|
|
48
|
+
if not allowed:
|
|
49
|
+
raise GuardError("allowed_account_ids is empty: refusing to run. Set it in config or LABAGENT_ALLOWED_ACCOUNTS.")
|
|
50
|
+
if self.account_id not in allowed:
|
|
51
|
+
raise GuardError("current AWS account is not in allowed_account_ids: refusing to run.")
|