de-agent 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. de_agent/__init__.py +1 -0
  2. de_agent/applyplan.py +270 -0
  3. de_agent/approval.py +110 -0
  4. de_agent/cli.py +650 -0
  5. de_agent/cliclass.py +1651 -0
  6. de_agent/clients.py +74 -0
  7. de_agent/collectors/__init__.py +31 -0
  8. de_agent/collectors/_airflow_core.py +343 -0
  9. de_agent/collectors/_pack_http_core.py +219 -0
  10. de_agent/collectors/airflow.py +125 -0
  11. de_agent/collectors/aws/__init__.py +172 -0
  12. de_agent/collectors/aws/_aws_core.py +823 -0
  13. de_agent/collectors/base.py +66 -0
  14. de_agent/collectors/dbt.py +198 -0
  15. de_agent/collectors/pack_base.py +225 -0
  16. de_agent/collectors/pack_cli.py +93 -0
  17. de_agent/collectors/pack_http.py +137 -0
  18. de_agent/collectors/pack_sql.py +217 -0
  19. de_agent/collectors/postgres.py +285 -0
  20. de_agent/collectors/redshift.py +815 -0
  21. de_agent/config.py +450 -0
  22. de_agent/configdoc.py +115 -0
  23. de_agent/data/sql_cases.yaml +1735 -0
  24. de_agent/diff.py +280 -0
  25. de_agent/extract.py +662 -0
  26. de_agent/find.py +218 -0
  27. de_agent/hubpath.py +32 -0
  28. de_agent/initcmd.py +49 -0
  29. de_agent/lint.py +297 -0
  30. de_agent/packs/__init__.py +5 -0
  31. de_agent/packs/argv.py +269 -0
  32. de_agent/packs/bigquery.yaml +67 -0
  33. de_agent/packs/check.py +228 -0
  34. de_agent/packs/cli.py +147 -0
  35. de_agent/packs/dagster.yaml +59 -0
  36. de_agent/packs/databricks-system.yaml +44 -0
  37. de_agent/packs/databricks.yaml +56 -0
  38. de_agent/packs/errors.py +6 -0
  39. de_agent/packs/expr.py +314 -0
  40. de_agent/packs/fivetran.yaml +30 -0
  41. de_agent/packs/gcp.yaml +53 -0
  42. de_agent/packs/kubernetes.yaml +64 -0
  43. de_agent/packs/loader.py +85 -0
  44. de_agent/packs/model.py +154 -0
  45. de_agent/packs/mssql.yaml +61 -0
  46. de_agent/packs/mysql.yaml +53 -0
  47. de_agent/packs/postgres-lite.yaml +65 -0
  48. de_agent/packs/scaffold.py +75 -0
  49. de_agent/packs/snowflake.yaml +91 -0
  50. de_agent/packs/template.py +111 -0
  51. de_agent/packs/view.py +64 -0
  52. de_agent/plans.py +621 -0
  53. de_agent/pyscan.py +214 -0
  54. de_agent/render.py +854 -0
  55. de_agent/runner.py +66 -0
  56. de_agent/scanners/__init__.py +1 -0
  57. de_agent/scanners/airflow.py +691 -0
  58. de_agent/scanners/catalog.py +205 -0
  59. de_agent/scanners/common.py +146 -0
  60. de_agent/scanners/dbt.py +99 -0
  61. de_agent/scanners/edges.py +358 -0
  62. de_agent/scanners/headers.py +117 -0
  63. de_agent/scanners/pycode.py +719 -0
  64. de_agent/scanners/pyexpr.py +308 -0
  65. de_agent/scanners/python.py +240 -0
  66. de_agent/scanners/run.py +288 -0
  67. de_agent/scanners/sql.py +131 -0
  68. de_agent/scanners/terraform.py +769 -0
  69. de_agent/secretscan.py +28 -0
  70. de_agent/snapshot.py +159 -0
  71. de_agent/sqlclass.py +333 -0
  72. de_agent/sqlexec.py +626 -0
  73. de_agent/stepexec.py +257 -0
  74. de_agent/templates/example/integrations/orders_etl--warehouse.md +10 -0
  75. de_agent/templates/example/map_row.txt +1 -0
  76. de_agent/templates/example/projects/orders_etl.md +21 -0
  77. de_agent/templates/hub/README.md +16 -0
  78. de_agent/templates/hub/conventions.md +15 -0
  79. de_agent/templates/hub/de-agent.yaml +20 -0
  80. de_agent/templates/hub/domains/_template.md +10 -0
  81. de_agent/templates/hub/gitignore.tmpl +1 -0
  82. de_agent/templates/hub/integrations/_template.md +12 -0
  83. de_agent/templates/hub/map.md +17 -0
  84. de_agent/templates/hub/projects/_template.md +24 -0
  85. de_agent/templates/hub/status.md +23 -0
  86. de_agent-0.2.0.dist-info/METADATA +94 -0
  87. de_agent-0.2.0.dist-info/RECORD +91 -0
  88. de_agent-0.2.0.dist-info/WHEEL +4 -0
  89. de_agent-0.2.0.dist-info/entry_points.txt +2 -0
  90. de_agent-0.2.0.dist-info/licenses/LICENSE +202 -0
  91. de_agent-0.2.0.dist-info/licenses/NOTICE +6 -0
de_agent/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.2.0"
de_agent/applyplan.py ADDED
@@ -0,0 +1,270 @@
1
+ """`dea apply`: run an approved plan (M2 section 1).
2
+
3
+ Order of operations: verify the HMAC, re-hash every statement and refuse anything not in the approved set, THEN run.
4
+ One transaction per step (per source) unless a block says `transaction: false`; verify blocks run after their step's
5
+ statements; the first failed statement or verify stops the run. Progress is written to <id>.applied.json after each step.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import re
11
+ import time
12
+ from dataclasses import dataclass, field
13
+ from datetime import datetime, timezone
14
+ from pathlib import Path
15
+ from typing import Any, Callable
16
+
17
+ from de_agent import approval, stepexec
18
+ from de_agent.collectors.base import CollectorError
19
+ from de_agent.config import Config
20
+ from de_agent.plans import (
21
+ Block,
22
+ Plan,
23
+ blocks_covered,
24
+ check_plan,
25
+ eval_expect,
26
+ plans_dir,
27
+ repo_path,
28
+ statements,
29
+ )
30
+
31
+
32
+ class ApplyRefused(Exception):
33
+ """Nothing was run: missing/bad approval, unapproved statement, plan errors, or already applied."""
34
+
35
+
36
+ @dataclass
37
+ class ApplyResult:
38
+ status: str # ok | failed
39
+ mode: str
40
+ steps: list[dict[str, Any]] = field(default_factory=list)
41
+ path: Path | None = None
42
+
43
+
44
+ @dataclass
45
+ class Unit:
46
+ kind: str # apply | verify
47
+ source: str
48
+ transaction: bool
49
+ texts: list[str]
50
+ block: Block
51
+ reverse: bool = False # diff units: reverse-apply (rollback)
52
+
53
+ @property
54
+ def bk(self) -> str:
55
+ return self.block.kind
56
+
57
+
58
+ def _now() -> str:
59
+ return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
60
+
61
+
62
+ def _atomic_json(path: Path, data: dict[str, Any]) -> None:
63
+ tmp = path.with_name(path.name + ".tmp")
64
+ tmp.write_text(json.dumps(data, indent=2, default=str) + "\n", encoding="utf-8")
65
+ tmp.replace(path)
66
+
67
+
68
+ def _units(blocks: list[Block], reverse: bool = False) -> list[Unit]:
69
+ """Execution units: the step's statements in plan order (consecutive transactional SQL blocks on one source share a
70
+ transaction; a diff or shell block ends the run), then its verify blocks."""
71
+ units: list[Unit] = []
72
+ for b in (x for x in blocks if not x.verify):
73
+ if b.kind != "sql":
74
+ units.append(Unit("apply", b.source or "", True, [b.sql], b, reverse and b.kind == "diff"))
75
+ continue
76
+ texts = [t for t, _ in statements(b.sql)]
77
+ last = units[-1] if units else None
78
+ if last and last.bk == "sql" and last.source == b.source and last.transaction and b.transaction:
79
+ last.texts.extend(texts)
80
+ else:
81
+ units.append(Unit("apply", b.source or "", b.transaction, texts, b))
82
+ units += [Unit("verify", b.source or "", True, [b.sql], b) for b in blocks if b.verify]
83
+ return units
84
+
85
+
86
+ def _groups(plan: Plan, rollback: bool, applied: set[tuple[int, int]] | None = None
87
+ ) -> list[tuple[int, str, list[Block], bool]]:
88
+ """(step number, title, blocks, reverse). Rollback = the diffs that were applied, reversed newest first, then the
89
+ Rollback section in plan order."""
90
+ if not rollback:
91
+ return [(s.n, s.title, s.blocks, False) for s in plan.steps]
92
+ out: list[tuple[int, str, list[Block], bool]] = []
93
+ for s in reversed(plan.steps):
94
+ diffs = [b for b in reversed(s.blocks) if b.kind == "diff" and (applied is None or (s.n, b.line) in applied)]
95
+ if diffs:
96
+ out.append((s.n, f"revert step {s.n}: {s.title}", diffs, True))
97
+ if plan.rollback:
98
+ out.append((0, "rollback", plan.rollback, False))
99
+ return out
100
+
101
+
102
+ def _applied_diffs(path: Path) -> set[tuple[int, int]]:
103
+ """(step, line) of every diff the earlier apply actually applied (a failed run may have applied only some)."""
104
+ prev = _load(path) or {}
105
+ return {(st["step"], a["line"]) for st in prev.get("steps", []) for a in st.get("actions", [])
106
+ if a.get("kind") == "diff" and a.get("mode") == "applied"}
107
+
108
+
109
+ def authorize(hub: Path, cfg: Config, plan: Plan, rollback: bool, key: bytes | None) -> dict[str, Any]:
110
+ """Raise ApplyRefused unless the plan is checked clean, signed, and every statement to run is approved."""
111
+ rep = check_plan(plan, cfg, filename_id=plan.path.stem if plan.path else None, hub=hub, tree=not rollback)
112
+ if not rep.ok:
113
+ raise ApplyRefused("plan has check errors (run `dea plan check`): " + "; ".join(rep.errors[:3]))
114
+ path = plans_dir(hub) / f"{plan.id}.approved"
115
+ if not path.is_file():
116
+ raise ApplyRefused(
117
+ f"plan {plan.id} is not approved: the human must run `dea approve {plan.id}` in their own terminal")
118
+ data, why = approval.read_approval(str(path), key, plan.id)
119
+ if data is None:
120
+ raise ApplyRefused(f"approval rejected: {why}")
121
+ ok = set(data["hashes"])
122
+ blocks = ([b for s in plan.steps for b in s.blocks if b.kind == "diff"] + plan.rollback) if rollback else \
123
+ [b for s in plan.steps for b in s.blocks]
124
+ if rollback and not blocks:
125
+ raise ApplyRefused("plan has no Rollback section")
126
+ bad = blocks_covered(blocks, ok)
127
+ if bad:
128
+ raise ApplyRefused(f"{len(bad)} statement(s) not in the approved set (plan edited after approval?): "
129
+ f"{bad[0][0].kind} {bad[0][1][:12]}")
130
+ return data
131
+
132
+
133
+ def dry_run_lines(plan: Plan, rollback: bool) -> list[str]:
134
+ out: list[str] = []
135
+ for n, title, blocks, rev in _groups(plan, rollback):
136
+ out.append(f"== {title if rollback else f'step {n}: {title}'}")
137
+ for u in _units(blocks, rev):
138
+ if u.bk == "diff":
139
+ out.append(f" [{'reverse ' if u.reverse else ''}git apply {u.block.repo}/{u.block.file}]")
140
+ elif u.bk == "shell":
141
+ out.append(f" [{u.kind} shell on {u.block.exec}]")
142
+ out.append(f" {u.texts[0][:100]}")
143
+ else:
144
+ out.append(f" [{u.kind} on {u.source}{'' if u.transaction else ', no transaction'}]")
145
+ out += [f" {t[:100]}" for t in u.texts]
146
+ return out
147
+
148
+
149
+ def _load(p: Path) -> dict[str, Any] | None:
150
+ try:
151
+ d = json.loads(p.read_text(encoding="utf-8"))
152
+ return d if isinstance(d, dict) else None
153
+ except (OSError, ValueError):
154
+ return None
155
+
156
+
157
+ def apply_plan(
158
+ hub: Path, cfg: Config, plan: Plan, rollback: bool = False, key: bytes | None = None,
159
+ write_fn: Callable[..., list[int | None]] | None = None,
160
+ read_fn: Callable[..., list[dict[str, Any]]] | None = None,
161
+ ) -> ApplyResult:
162
+ from de_agent import sqlexec
163
+
164
+ write_fn = write_fn or sqlexec.run_write
165
+ read_fn = read_fn or sqlexec.run_query
166
+ approval_data = authorize(hub, cfg, plan, rollback, key)
167
+ out_path = plans_dir(hub) / (f"{plan.id}.rolledback.json" if rollback else f"{plan.id}.applied.json")
168
+ if not rollback:
169
+ prev = _load(out_path)
170
+ if prev and prev.get("status") == "ok":
171
+ raise ApplyRefused(f"plan {plan.id} was already applied; write a new plan instead of re-running it")
172
+
173
+ groups = _groups(plan, rollback, _applied_diffs(plans_dir(hub) / f"{plan.id}.applied.json") if rollback else None)
174
+ record: dict[str, Any] = {
175
+ "plan_id": plan.id, "mode": "rollback" if rollback else "apply", "status": "running",
176
+ "approved_by": approval_data.get("approver"), "approved_at": approval_data.get("approved_at"),
177
+ "started_at": _now(), "finished_at": None,
178
+ "steps": [{"step": n, "title": t, "status": "pending", "rowcount": None, "duration_s": None,
179
+ "verify": [], "actions": [], "error": None} for n, t, _, _ in groups],
180
+ }
181
+ _atomic_json(out_path, record)
182
+ failed = False
183
+ for idx, (_n, _title, blocks, reverse) in enumerate(groups):
184
+ rec = record["steps"][idx]
185
+ if failed:
186
+ rec["status"] = "skipped"
187
+ continue
188
+ t0 = time.monotonic()
189
+ total, known = 0, False
190
+ try:
191
+ for u in _units(blocks, reverse):
192
+ if u.bk == "diff":
193
+ _run_diff(hub, cfg, u, rec, out_path, record)
194
+ elif u.bk == "shell":
195
+ _run_shell(hub, plan, u, rec)
196
+ else:
197
+ src = cfg.source(u.source)
198
+ if src.conn is None:
199
+ raise CollectorError(f"source {u.source} has no conn")
200
+ timeout = float(plan.statement_timeout_s)
201
+ if u.kind == "apply":
202
+ for c in write_fn(src.conn, u.texts, src.exec, timeout, u.transaction):
203
+ if c is not None:
204
+ total, known = total + c, True
205
+ else:
206
+ rows = read_fn(src.conn, u.texts[0], src.exec, timeout)
207
+ value = next(iter(rows[0].values()), None) if rows else None
208
+ ok = True if u.block.expect is None else eval_expect(u.block.expect, value)
209
+ rec["verify"].append({"source": u.source, "sql": u.texts[0][:200], "expect": u.block.expect,
210
+ "value": value, "ok": ok})
211
+ if not ok:
212
+ raise CollectorError(f"verify failed: got {value!r}, expected {u.block.expect}")
213
+ rec["status"] = "ok"
214
+ except Exception as ex:
215
+ rec["status"] = "failed"
216
+ rec["error"] = stepexec.redact_output(str(ex))[:500]
217
+ failed = True
218
+ rec["rowcount"] = total if known else None
219
+ rec["duration_s"] = round(time.monotonic() - t0, 3)
220
+ _atomic_json(out_path, record)
221
+ record["status"] = "failed" if failed else "ok"
222
+ record["finished_at"] = _now()
223
+ _atomic_json(out_path, record)
224
+ return ApplyResult(record["status"], record["mode"], record["steps"], out_path)
225
+
226
+
227
+ def _run_diff(hub: Path, cfg: Config, u: Unit, rec: dict[str, Any], out_path: Path, record: dict[str, Any]) -> None:
228
+ b = u.block
229
+ repo = repo_path(hub, cfg, b.repo)
230
+ if repo is None:
231
+ raise CollectorError(f"unknown repo {b.repo!r}")
232
+ act = {"kind": "diff", "mode": "pending", "repo": b.repo, "file": b.file, "line": b.line, "diff_stat": None}
233
+ rec["actions"].append(act)
234
+ try:
235
+ act["diff_stat"] = stepexec.git_apply(repo, b.sql, b.file or "", reverse=u.reverse)
236
+ except RuntimeError as ex:
237
+ act["mode"] = "failed"
238
+ raise CollectorError(str(ex)) from None
239
+ act["mode"] = "reverted" if u.reverse else "applied"
240
+ _atomic_json(out_path, record)
241
+
242
+
243
+ def _run_shell(hub: Path, plan: Plan, u: Unit, rec: dict[str, Any]) -> None:
244
+ b = u.block
245
+ cwd = None
246
+ if b.exec == "local":
247
+ cwd = (hub / b.cwd).resolve() if b.cwd else hub
248
+ if not cwd.is_dir():
249
+ raise CollectorError(f"cwd {b.cwd!r} does not exist")
250
+ timeout = float(b.timeout or plan.statement_timeout_s)
251
+ try:
252
+ res = stepexec.run_shell(b.sql, b.exec, cwd, timeout, remote_cwd=b.cwd)
253
+ except RuntimeError as ex:
254
+ raise CollectorError(str(ex)) from None
255
+ if u.kind == "apply":
256
+ rec["actions"].append({"kind": "shell", "command": b.sql, "exec": b.exec, "exit_code": res.exit_code,
257
+ "stdout_tail": res.stdout_tail, "stderr_tail": res.stderr_tail, "line": b.line})
258
+ if res.exit_code != 0:
259
+ raise CollectorError(f"command exited {res.exit_code}: {(res.stderr_tail or res.stdout_tail)[-300:]}")
260
+ return
261
+ want_exit = b.expect_exit if b.expect_exit is not None or b.expect_stdout is not None else 0
262
+ ok = (want_exit is None or res.exit_code == want_exit) and \
263
+ (b.expect_stdout is None or re.search(b.expect_stdout, res.stdout_full) is not None)
264
+ exp = " and ".join(x for x in (None if want_exit is None else f"exit={want_exit}",
265
+ None if b.expect_stdout is None else f"stdout~={b.expect_stdout}") if x)
266
+ rec["verify"].append({"source": f"shell:{b.exec}", "sql": b.sql[:200], "expect": exp,
267
+ "value": res.exit_code if b.expect_stdout is None else res.stdout_tail[-200:],
268
+ "ok": ok, "exit_code": res.exit_code, "stdout_tail": res.stdout_tail})
269
+ if not ok:
270
+ raise CollectorError(f"verify failed: exit {res.exit_code}, expected {exp}")
de_agent/approval.py ADDED
@@ -0,0 +1,110 @@
1
+ """HMAC-signed plan approvals (M2 section 1). Stdlib only and Python 3.9 compatible: the guard hooks import this.
2
+
3
+ Threat model (docs/SAFETY.md): the key file stops accidents and casual bypass by a model that cannot read it
4
+ (secret_guard) or run `dea approve` (approval_guard). It is not a sandbox against a hostile local process that runs as you.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ import hashlib
9
+ import hmac
10
+ import json
11
+ import os
12
+ import tempfile
13
+
14
+ KEY_NAME = "approve.key"
15
+
16
+
17
+ def key_path():
18
+ base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config")
19
+ return os.path.join(base, "de-agent", KEY_NAME)
20
+
21
+
22
+ def load_key(create=False):
23
+ """Return the key bytes, or None when it does not exist (and create is False)."""
24
+ path = key_path()
25
+ try:
26
+ with open(path, "rb") as f:
27
+ data = f.read().strip()
28
+ return data or None
29
+ except OSError:
30
+ pass
31
+ if not create:
32
+ return None
33
+ os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
34
+ key = os.urandom(32).hex().encode()
35
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
36
+ with os.fdopen(fd, "wb") as f:
37
+ f.write(key + b"\n")
38
+ return key
39
+
40
+
41
+ def _canonical(payload):
42
+ body = {
43
+ "plan_id": payload.get("plan_id"),
44
+ "hashes": payload.get("hashes"),
45
+ "approver": payload.get("approver"),
46
+ "approved_at": payload.get("approved_at"),
47
+ }
48
+ return json.dumps(body, sort_keys=True, separators=(",", ":")).encode()
49
+
50
+
51
+ def sign(payload, key):
52
+ return hmac.new(key, _canonical(payload), hashlib.sha256).hexdigest()
53
+
54
+
55
+ def make_approval(plan_id, hashes, approver, approved_at, key):
56
+ data = {"plan_id": plan_id, "hashes": list(hashes), "approver": approver, "approved_at": approved_at}
57
+ data["mac"] = sign(data, key)
58
+ return data
59
+
60
+
61
+ def write_approval(path, data):
62
+ d = os.path.dirname(path)
63
+ os.makedirs(d, exist_ok=True)
64
+ fd, tmp = tempfile.mkstemp(dir=d, suffix=".tmp")
65
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
66
+ json.dump(data, f, indent=2, sort_keys=True)
67
+ f.write("\n")
68
+ os.replace(tmp, path)
69
+
70
+
71
+ def verify_data(data, key, plan_id=None):
72
+ """Return (ok, reason)."""
73
+ if not isinstance(data, dict):
74
+ return False, "approval file is not a JSON object"
75
+ mac = data.get("mac")
76
+ hashes = data.get("hashes")
77
+ if not isinstance(mac, str) or not isinstance(hashes, list) or not all(isinstance(h, str) for h in hashes):
78
+ return False, "approval file is malformed"
79
+ if key is None:
80
+ return False, "no approval key on this machine (run `dea approve` in your own terminal)"
81
+ if not hmac.compare_digest(mac, sign(data, key)):
82
+ return False, "bad MAC (approval file was edited, or signed with a different key)"
83
+ if plan_id is not None and data.get("plan_id") != plan_id:
84
+ return False, "approval is for a different plan id"
85
+ return True, ""
86
+
87
+
88
+ def read_approval(path, key, plan_id=None):
89
+ """Return (data-or-None, reason). data is returned only when the MAC verifies."""
90
+ try:
91
+ with open(path, encoding="utf-8") as f:
92
+ data = json.load(f)
93
+ except (OSError, ValueError) as ex:
94
+ return None, f"unreadable approval file: {ex}"
95
+ ok, why = verify_data(data, key, plan_id)
96
+ return (data if ok else None), why
97
+
98
+
99
+ def approved_hashes(project_dir):
100
+ """Union of hashes from every `.approved` file whose MAC verifies. Bad or unsigned files grant nothing."""
101
+ import glob
102
+
103
+ key = load_key()
104
+ out = set()
105
+ for path in glob.glob(os.path.join(project_dir, ".dea", "plans", "*.approved")):
106
+ stem = os.path.basename(path)[: -len(".approved")]
107
+ data, _ = read_approval(path, key, stem)
108
+ if data:
109
+ out.update(data["hashes"])
110
+ return out