de-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- de_agent/__init__.py +1 -0
- de_agent/applyplan.py +270 -0
- de_agent/approval.py +110 -0
- de_agent/cli.py +650 -0
- de_agent/cliclass.py +1651 -0
- de_agent/clients.py +74 -0
- de_agent/collectors/__init__.py +31 -0
- de_agent/collectors/_airflow_core.py +343 -0
- de_agent/collectors/_pack_http_core.py +219 -0
- de_agent/collectors/airflow.py +125 -0
- de_agent/collectors/aws/__init__.py +172 -0
- de_agent/collectors/aws/_aws_core.py +823 -0
- de_agent/collectors/base.py +66 -0
- de_agent/collectors/dbt.py +198 -0
- de_agent/collectors/pack_base.py +225 -0
- de_agent/collectors/pack_cli.py +93 -0
- de_agent/collectors/pack_http.py +137 -0
- de_agent/collectors/pack_sql.py +217 -0
- de_agent/collectors/postgres.py +285 -0
- de_agent/collectors/redshift.py +815 -0
- de_agent/config.py +450 -0
- de_agent/configdoc.py +115 -0
- de_agent/data/sql_cases.yaml +1735 -0
- de_agent/diff.py +280 -0
- de_agent/extract.py +662 -0
- de_agent/find.py +218 -0
- de_agent/hubpath.py +32 -0
- de_agent/initcmd.py +49 -0
- de_agent/lint.py +297 -0
- de_agent/packs/__init__.py +5 -0
- de_agent/packs/argv.py +269 -0
- de_agent/packs/bigquery.yaml +67 -0
- de_agent/packs/check.py +228 -0
- de_agent/packs/cli.py +147 -0
- de_agent/packs/dagster.yaml +59 -0
- de_agent/packs/databricks-system.yaml +44 -0
- de_agent/packs/databricks.yaml +56 -0
- de_agent/packs/errors.py +6 -0
- de_agent/packs/expr.py +314 -0
- de_agent/packs/fivetran.yaml +30 -0
- de_agent/packs/gcp.yaml +53 -0
- de_agent/packs/kubernetes.yaml +64 -0
- de_agent/packs/loader.py +85 -0
- de_agent/packs/model.py +154 -0
- de_agent/packs/mssql.yaml +61 -0
- de_agent/packs/mysql.yaml +53 -0
- de_agent/packs/postgres-lite.yaml +65 -0
- de_agent/packs/scaffold.py +75 -0
- de_agent/packs/snowflake.yaml +91 -0
- de_agent/packs/template.py +111 -0
- de_agent/packs/view.py +64 -0
- de_agent/plans.py +621 -0
- de_agent/pyscan.py +214 -0
- de_agent/render.py +854 -0
- de_agent/runner.py +66 -0
- de_agent/scanners/__init__.py +1 -0
- de_agent/scanners/airflow.py +691 -0
- de_agent/scanners/catalog.py +205 -0
- de_agent/scanners/common.py +146 -0
- de_agent/scanners/dbt.py +99 -0
- de_agent/scanners/edges.py +358 -0
- de_agent/scanners/headers.py +117 -0
- de_agent/scanners/pycode.py +719 -0
- de_agent/scanners/pyexpr.py +308 -0
- de_agent/scanners/python.py +240 -0
- de_agent/scanners/run.py +288 -0
- de_agent/scanners/sql.py +131 -0
- de_agent/scanners/terraform.py +769 -0
- de_agent/secretscan.py +28 -0
- de_agent/snapshot.py +159 -0
- de_agent/sqlclass.py +333 -0
- de_agent/sqlexec.py +626 -0
- de_agent/stepexec.py +257 -0
- de_agent/templates/example/integrations/orders_etl--warehouse.md +10 -0
- de_agent/templates/example/map_row.txt +1 -0
- de_agent/templates/example/projects/orders_etl.md +21 -0
- de_agent/templates/hub/README.md +16 -0
- de_agent/templates/hub/conventions.md +15 -0
- de_agent/templates/hub/de-agent.yaml +20 -0
- de_agent/templates/hub/domains/_template.md +10 -0
- de_agent/templates/hub/gitignore.tmpl +1 -0
- de_agent/templates/hub/integrations/_template.md +12 -0
- de_agent/templates/hub/map.md +17 -0
- de_agent/templates/hub/projects/_template.md +24 -0
- de_agent/templates/hub/status.md +23 -0
- de_agent-0.2.0.dist-info/METADATA +94 -0
- de_agent-0.2.0.dist-info/RECORD +91 -0
- de_agent-0.2.0.dist-info/WHEEL +4 -0
- de_agent-0.2.0.dist-info/entry_points.txt +2 -0
- de_agent-0.2.0.dist-info/licenses/LICENSE +202 -0
- de_agent-0.2.0.dist-info/licenses/NOTICE +6 -0
de_agent/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.2.0"
|
de_agent/applyplan.py
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""`dea apply`: run an approved plan (M2 section 1).
|
|
2
|
+
|
|
3
|
+
Order of operations: verify the HMAC, re-hash every statement and refuse anything not in the approved set, THEN run.
|
|
4
|
+
One transaction per step (per source) unless a block says `transaction: false`; verify blocks run after their step's
|
|
5
|
+
statements; the first failed statement or verify stops the run. Progress is written to <id>.applied.json after each step.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
import time
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from datetime import datetime, timezone
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Callable
|
|
16
|
+
|
|
17
|
+
from de_agent import approval, stepexec
|
|
18
|
+
from de_agent.collectors.base import CollectorError
|
|
19
|
+
from de_agent.config import Config
|
|
20
|
+
from de_agent.plans import (
|
|
21
|
+
Block,
|
|
22
|
+
Plan,
|
|
23
|
+
blocks_covered,
|
|
24
|
+
check_plan,
|
|
25
|
+
eval_expect,
|
|
26
|
+
plans_dir,
|
|
27
|
+
repo_path,
|
|
28
|
+
statements,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ApplyRefused(Exception):
|
|
33
|
+
"""Nothing was run: missing/bad approval, unapproved statement, plan errors, or already applied."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class ApplyResult:
|
|
38
|
+
status: str # ok | failed
|
|
39
|
+
mode: str
|
|
40
|
+
steps: list[dict[str, Any]] = field(default_factory=list)
|
|
41
|
+
path: Path | None = None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class Unit:
|
|
46
|
+
kind: str # apply | verify
|
|
47
|
+
source: str
|
|
48
|
+
transaction: bool
|
|
49
|
+
texts: list[str]
|
|
50
|
+
block: Block
|
|
51
|
+
reverse: bool = False # diff units: reverse-apply (rollback)
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def bk(self) -> str:
|
|
55
|
+
return self.block.kind
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _now() -> str:
|
|
59
|
+
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _atomic_json(path: Path, data: dict[str, Any]) -> None:
|
|
63
|
+
tmp = path.with_name(path.name + ".tmp")
|
|
64
|
+
tmp.write_text(json.dumps(data, indent=2, default=str) + "\n", encoding="utf-8")
|
|
65
|
+
tmp.replace(path)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _units(blocks: list[Block], reverse: bool = False) -> list[Unit]:
|
|
69
|
+
"""Execution units: the step's statements in plan order (consecutive transactional SQL blocks on one source share a
|
|
70
|
+
transaction; a diff or shell block ends the run), then its verify blocks."""
|
|
71
|
+
units: list[Unit] = []
|
|
72
|
+
for b in (x for x in blocks if not x.verify):
|
|
73
|
+
if b.kind != "sql":
|
|
74
|
+
units.append(Unit("apply", b.source or "", True, [b.sql], b, reverse and b.kind == "diff"))
|
|
75
|
+
continue
|
|
76
|
+
texts = [t for t, _ in statements(b.sql)]
|
|
77
|
+
last = units[-1] if units else None
|
|
78
|
+
if last and last.bk == "sql" and last.source == b.source and last.transaction and b.transaction:
|
|
79
|
+
last.texts.extend(texts)
|
|
80
|
+
else:
|
|
81
|
+
units.append(Unit("apply", b.source or "", b.transaction, texts, b))
|
|
82
|
+
units += [Unit("verify", b.source or "", True, [b.sql], b) for b in blocks if b.verify]
|
|
83
|
+
return units
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _groups(plan: Plan, rollback: bool, applied: set[tuple[int, int]] | None = None
|
|
87
|
+
) -> list[tuple[int, str, list[Block], bool]]:
|
|
88
|
+
"""(step number, title, blocks, reverse). Rollback = the diffs that were applied, reversed newest first, then the
|
|
89
|
+
Rollback section in plan order."""
|
|
90
|
+
if not rollback:
|
|
91
|
+
return [(s.n, s.title, s.blocks, False) for s in plan.steps]
|
|
92
|
+
out: list[tuple[int, str, list[Block], bool]] = []
|
|
93
|
+
for s in reversed(plan.steps):
|
|
94
|
+
diffs = [b for b in reversed(s.blocks) if b.kind == "diff" and (applied is None or (s.n, b.line) in applied)]
|
|
95
|
+
if diffs:
|
|
96
|
+
out.append((s.n, f"revert step {s.n}: {s.title}", diffs, True))
|
|
97
|
+
if plan.rollback:
|
|
98
|
+
out.append((0, "rollback", plan.rollback, False))
|
|
99
|
+
return out
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _applied_diffs(path: Path) -> set[tuple[int, int]]:
|
|
103
|
+
"""(step, line) of every diff the earlier apply actually applied (a failed run may have applied only some)."""
|
|
104
|
+
prev = _load(path) or {}
|
|
105
|
+
return {(st["step"], a["line"]) for st in prev.get("steps", []) for a in st.get("actions", [])
|
|
106
|
+
if a.get("kind") == "diff" and a.get("mode") == "applied"}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def authorize(hub: Path, cfg: Config, plan: Plan, rollback: bool, key: bytes | None) -> dict[str, Any]:
|
|
110
|
+
"""Raise ApplyRefused unless the plan is checked clean, signed, and every statement to run is approved."""
|
|
111
|
+
rep = check_plan(plan, cfg, filename_id=plan.path.stem if plan.path else None, hub=hub, tree=not rollback)
|
|
112
|
+
if not rep.ok:
|
|
113
|
+
raise ApplyRefused("plan has check errors (run `dea plan check`): " + "; ".join(rep.errors[:3]))
|
|
114
|
+
path = plans_dir(hub) / f"{plan.id}.approved"
|
|
115
|
+
if not path.is_file():
|
|
116
|
+
raise ApplyRefused(
|
|
117
|
+
f"plan {plan.id} is not approved: the human must run `dea approve {plan.id}` in their own terminal")
|
|
118
|
+
data, why = approval.read_approval(str(path), key, plan.id)
|
|
119
|
+
if data is None:
|
|
120
|
+
raise ApplyRefused(f"approval rejected: {why}")
|
|
121
|
+
ok = set(data["hashes"])
|
|
122
|
+
blocks = ([b for s in plan.steps for b in s.blocks if b.kind == "diff"] + plan.rollback) if rollback else \
|
|
123
|
+
[b for s in plan.steps for b in s.blocks]
|
|
124
|
+
if rollback and not blocks:
|
|
125
|
+
raise ApplyRefused("plan has no Rollback section")
|
|
126
|
+
bad = blocks_covered(blocks, ok)
|
|
127
|
+
if bad:
|
|
128
|
+
raise ApplyRefused(f"{len(bad)} statement(s) not in the approved set (plan edited after approval?): "
|
|
129
|
+
f"{bad[0][0].kind} {bad[0][1][:12]}")
|
|
130
|
+
return data
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def dry_run_lines(plan: Plan, rollback: bool) -> list[str]:
|
|
134
|
+
out: list[str] = []
|
|
135
|
+
for n, title, blocks, rev in _groups(plan, rollback):
|
|
136
|
+
out.append(f"== {title if rollback else f'step {n}: {title}'}")
|
|
137
|
+
for u in _units(blocks, rev):
|
|
138
|
+
if u.bk == "diff":
|
|
139
|
+
out.append(f" [{'reverse ' if u.reverse else ''}git apply {u.block.repo}/{u.block.file}]")
|
|
140
|
+
elif u.bk == "shell":
|
|
141
|
+
out.append(f" [{u.kind} shell on {u.block.exec}]")
|
|
142
|
+
out.append(f" {u.texts[0][:100]}")
|
|
143
|
+
else:
|
|
144
|
+
out.append(f" [{u.kind} on {u.source}{'' if u.transaction else ', no transaction'}]")
|
|
145
|
+
out += [f" {t[:100]}" for t in u.texts]
|
|
146
|
+
return out
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _load(p: Path) -> dict[str, Any] | None:
|
|
150
|
+
try:
|
|
151
|
+
d = json.loads(p.read_text(encoding="utf-8"))
|
|
152
|
+
return d if isinstance(d, dict) else None
|
|
153
|
+
except (OSError, ValueError):
|
|
154
|
+
return None
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def apply_plan(
|
|
158
|
+
hub: Path, cfg: Config, plan: Plan, rollback: bool = False, key: bytes | None = None,
|
|
159
|
+
write_fn: Callable[..., list[int | None]] | None = None,
|
|
160
|
+
read_fn: Callable[..., list[dict[str, Any]]] | None = None,
|
|
161
|
+
) -> ApplyResult:
|
|
162
|
+
from de_agent import sqlexec
|
|
163
|
+
|
|
164
|
+
write_fn = write_fn or sqlexec.run_write
|
|
165
|
+
read_fn = read_fn or sqlexec.run_query
|
|
166
|
+
approval_data = authorize(hub, cfg, plan, rollback, key)
|
|
167
|
+
out_path = plans_dir(hub) / (f"{plan.id}.rolledback.json" if rollback else f"{plan.id}.applied.json")
|
|
168
|
+
if not rollback:
|
|
169
|
+
prev = _load(out_path)
|
|
170
|
+
if prev and prev.get("status") == "ok":
|
|
171
|
+
raise ApplyRefused(f"plan {plan.id} was already applied; write a new plan instead of re-running it")
|
|
172
|
+
|
|
173
|
+
groups = _groups(plan, rollback, _applied_diffs(plans_dir(hub) / f"{plan.id}.applied.json") if rollback else None)
|
|
174
|
+
record: dict[str, Any] = {
|
|
175
|
+
"plan_id": plan.id, "mode": "rollback" if rollback else "apply", "status": "running",
|
|
176
|
+
"approved_by": approval_data.get("approver"), "approved_at": approval_data.get("approved_at"),
|
|
177
|
+
"started_at": _now(), "finished_at": None,
|
|
178
|
+
"steps": [{"step": n, "title": t, "status": "pending", "rowcount": None, "duration_s": None,
|
|
179
|
+
"verify": [], "actions": [], "error": None} for n, t, _, _ in groups],
|
|
180
|
+
}
|
|
181
|
+
_atomic_json(out_path, record)
|
|
182
|
+
failed = False
|
|
183
|
+
for idx, (_n, _title, blocks, reverse) in enumerate(groups):
|
|
184
|
+
rec = record["steps"][idx]
|
|
185
|
+
if failed:
|
|
186
|
+
rec["status"] = "skipped"
|
|
187
|
+
continue
|
|
188
|
+
t0 = time.monotonic()
|
|
189
|
+
total, known = 0, False
|
|
190
|
+
try:
|
|
191
|
+
for u in _units(blocks, reverse):
|
|
192
|
+
if u.bk == "diff":
|
|
193
|
+
_run_diff(hub, cfg, u, rec, out_path, record)
|
|
194
|
+
elif u.bk == "shell":
|
|
195
|
+
_run_shell(hub, plan, u, rec)
|
|
196
|
+
else:
|
|
197
|
+
src = cfg.source(u.source)
|
|
198
|
+
if src.conn is None:
|
|
199
|
+
raise CollectorError(f"source {u.source} has no conn")
|
|
200
|
+
timeout = float(plan.statement_timeout_s)
|
|
201
|
+
if u.kind == "apply":
|
|
202
|
+
for c in write_fn(src.conn, u.texts, src.exec, timeout, u.transaction):
|
|
203
|
+
if c is not None:
|
|
204
|
+
total, known = total + c, True
|
|
205
|
+
else:
|
|
206
|
+
rows = read_fn(src.conn, u.texts[0], src.exec, timeout)
|
|
207
|
+
value = next(iter(rows[0].values()), None) if rows else None
|
|
208
|
+
ok = True if u.block.expect is None else eval_expect(u.block.expect, value)
|
|
209
|
+
rec["verify"].append({"source": u.source, "sql": u.texts[0][:200], "expect": u.block.expect,
|
|
210
|
+
"value": value, "ok": ok})
|
|
211
|
+
if not ok:
|
|
212
|
+
raise CollectorError(f"verify failed: got {value!r}, expected {u.block.expect}")
|
|
213
|
+
rec["status"] = "ok"
|
|
214
|
+
except Exception as ex:
|
|
215
|
+
rec["status"] = "failed"
|
|
216
|
+
rec["error"] = stepexec.redact_output(str(ex))[:500]
|
|
217
|
+
failed = True
|
|
218
|
+
rec["rowcount"] = total if known else None
|
|
219
|
+
rec["duration_s"] = round(time.monotonic() - t0, 3)
|
|
220
|
+
_atomic_json(out_path, record)
|
|
221
|
+
record["status"] = "failed" if failed else "ok"
|
|
222
|
+
record["finished_at"] = _now()
|
|
223
|
+
_atomic_json(out_path, record)
|
|
224
|
+
return ApplyResult(record["status"], record["mode"], record["steps"], out_path)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _run_diff(hub: Path, cfg: Config, u: Unit, rec: dict[str, Any], out_path: Path, record: dict[str, Any]) -> None:
|
|
228
|
+
b = u.block
|
|
229
|
+
repo = repo_path(hub, cfg, b.repo)
|
|
230
|
+
if repo is None:
|
|
231
|
+
raise CollectorError(f"unknown repo {b.repo!r}")
|
|
232
|
+
act = {"kind": "diff", "mode": "pending", "repo": b.repo, "file": b.file, "line": b.line, "diff_stat": None}
|
|
233
|
+
rec["actions"].append(act)
|
|
234
|
+
try:
|
|
235
|
+
act["diff_stat"] = stepexec.git_apply(repo, b.sql, b.file or "", reverse=u.reverse)
|
|
236
|
+
except RuntimeError as ex:
|
|
237
|
+
act["mode"] = "failed"
|
|
238
|
+
raise CollectorError(str(ex)) from None
|
|
239
|
+
act["mode"] = "reverted" if u.reverse else "applied"
|
|
240
|
+
_atomic_json(out_path, record)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _run_shell(hub: Path, plan: Plan, u: Unit, rec: dict[str, Any]) -> None:
|
|
244
|
+
b = u.block
|
|
245
|
+
cwd = None
|
|
246
|
+
if b.exec == "local":
|
|
247
|
+
cwd = (hub / b.cwd).resolve() if b.cwd else hub
|
|
248
|
+
if not cwd.is_dir():
|
|
249
|
+
raise CollectorError(f"cwd {b.cwd!r} does not exist")
|
|
250
|
+
timeout = float(b.timeout or plan.statement_timeout_s)
|
|
251
|
+
try:
|
|
252
|
+
res = stepexec.run_shell(b.sql, b.exec, cwd, timeout, remote_cwd=b.cwd)
|
|
253
|
+
except RuntimeError as ex:
|
|
254
|
+
raise CollectorError(str(ex)) from None
|
|
255
|
+
if u.kind == "apply":
|
|
256
|
+
rec["actions"].append({"kind": "shell", "command": b.sql, "exec": b.exec, "exit_code": res.exit_code,
|
|
257
|
+
"stdout_tail": res.stdout_tail, "stderr_tail": res.stderr_tail, "line": b.line})
|
|
258
|
+
if res.exit_code != 0:
|
|
259
|
+
raise CollectorError(f"command exited {res.exit_code}: {(res.stderr_tail or res.stdout_tail)[-300:]}")
|
|
260
|
+
return
|
|
261
|
+
want_exit = b.expect_exit if b.expect_exit is not None or b.expect_stdout is not None else 0
|
|
262
|
+
ok = (want_exit is None or res.exit_code == want_exit) and \
|
|
263
|
+
(b.expect_stdout is None or re.search(b.expect_stdout, res.stdout_full) is not None)
|
|
264
|
+
exp = " and ".join(x for x in (None if want_exit is None else f"exit={want_exit}",
|
|
265
|
+
None if b.expect_stdout is None else f"stdout~={b.expect_stdout}") if x)
|
|
266
|
+
rec["verify"].append({"source": f"shell:{b.exec}", "sql": b.sql[:200], "expect": exp,
|
|
267
|
+
"value": res.exit_code if b.expect_stdout is None else res.stdout_tail[-200:],
|
|
268
|
+
"ok": ok, "exit_code": res.exit_code, "stdout_tail": res.stdout_tail})
|
|
269
|
+
if not ok:
|
|
270
|
+
raise CollectorError(f"verify failed: exit {res.exit_code}, expected {exp}")
|
de_agent/approval.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""HMAC-signed plan approvals (M2 section 1). Stdlib only and Python 3.9 compatible: the guard hooks import this.
|
|
2
|
+
|
|
3
|
+
Threat model (docs/SAFETY.md): the key file stops accidents and casual bypass by a model that cannot read it
|
|
4
|
+
(secret_guard) or run `dea approve` (approval_guard). It is not a sandbox against a hostile local process that runs as you.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import hmac
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import tempfile
|
|
13
|
+
|
|
14
|
+
KEY_NAME = "approve.key"
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def key_path():
|
|
18
|
+
base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config")
|
|
19
|
+
return os.path.join(base, "de-agent", KEY_NAME)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def load_key(create=False):
|
|
23
|
+
"""Return the key bytes, or None when it does not exist (and create is False)."""
|
|
24
|
+
path = key_path()
|
|
25
|
+
try:
|
|
26
|
+
with open(path, "rb") as f:
|
|
27
|
+
data = f.read().strip()
|
|
28
|
+
return data or None
|
|
29
|
+
except OSError:
|
|
30
|
+
pass
|
|
31
|
+
if not create:
|
|
32
|
+
return None
|
|
33
|
+
os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
|
|
34
|
+
key = os.urandom(32).hex().encode()
|
|
35
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
|
36
|
+
with os.fdopen(fd, "wb") as f:
|
|
37
|
+
f.write(key + b"\n")
|
|
38
|
+
return key
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _canonical(payload):
|
|
42
|
+
body = {
|
|
43
|
+
"plan_id": payload.get("plan_id"),
|
|
44
|
+
"hashes": payload.get("hashes"),
|
|
45
|
+
"approver": payload.get("approver"),
|
|
46
|
+
"approved_at": payload.get("approved_at"),
|
|
47
|
+
}
|
|
48
|
+
return json.dumps(body, sort_keys=True, separators=(",", ":")).encode()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def sign(payload, key):
|
|
52
|
+
return hmac.new(key, _canonical(payload), hashlib.sha256).hexdigest()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def make_approval(plan_id, hashes, approver, approved_at, key):
|
|
56
|
+
data = {"plan_id": plan_id, "hashes": list(hashes), "approver": approver, "approved_at": approved_at}
|
|
57
|
+
data["mac"] = sign(data, key)
|
|
58
|
+
return data
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def write_approval(path, data):
|
|
62
|
+
d = os.path.dirname(path)
|
|
63
|
+
os.makedirs(d, exist_ok=True)
|
|
64
|
+
fd, tmp = tempfile.mkstemp(dir=d, suffix=".tmp")
|
|
65
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
66
|
+
json.dump(data, f, indent=2, sort_keys=True)
|
|
67
|
+
f.write("\n")
|
|
68
|
+
os.replace(tmp, path)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def verify_data(data, key, plan_id=None):
|
|
72
|
+
"""Return (ok, reason)."""
|
|
73
|
+
if not isinstance(data, dict):
|
|
74
|
+
return False, "approval file is not a JSON object"
|
|
75
|
+
mac = data.get("mac")
|
|
76
|
+
hashes = data.get("hashes")
|
|
77
|
+
if not isinstance(mac, str) or not isinstance(hashes, list) or not all(isinstance(h, str) for h in hashes):
|
|
78
|
+
return False, "approval file is malformed"
|
|
79
|
+
if key is None:
|
|
80
|
+
return False, "no approval key on this machine (run `dea approve` in your own terminal)"
|
|
81
|
+
if not hmac.compare_digest(mac, sign(data, key)):
|
|
82
|
+
return False, "bad MAC (approval file was edited, or signed with a different key)"
|
|
83
|
+
if plan_id is not None and data.get("plan_id") != plan_id:
|
|
84
|
+
return False, "approval is for a different plan id"
|
|
85
|
+
return True, ""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def read_approval(path, key, plan_id=None):
|
|
89
|
+
"""Return (data-or-None, reason). data is returned only when the MAC verifies."""
|
|
90
|
+
try:
|
|
91
|
+
with open(path, encoding="utf-8") as f:
|
|
92
|
+
data = json.load(f)
|
|
93
|
+
except (OSError, ValueError) as ex:
|
|
94
|
+
return None, f"unreadable approval file: {ex}"
|
|
95
|
+
ok, why = verify_data(data, key, plan_id)
|
|
96
|
+
return (data if ok else None), why
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def approved_hashes(project_dir):
|
|
100
|
+
"""Union of hashes from every `.approved` file whose MAC verifies. Bad or unsigned files grant nothing."""
|
|
101
|
+
import glob
|
|
102
|
+
|
|
103
|
+
key = load_key()
|
|
104
|
+
out = set()
|
|
105
|
+
for path in glob.glob(os.path.join(project_dir, ".dea", "plans", "*.approved")):
|
|
106
|
+
stem = os.path.basename(path)[: -len(".approved")]
|
|
107
|
+
data, _ = read_approval(path, key, stem)
|
|
108
|
+
if data:
|
|
109
|
+
out.update(data["hashes"])
|
|
110
|
+
return out
|