de-agent 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. de_agent/__init__.py +1 -0
  2. de_agent/applyplan.py +270 -0
  3. de_agent/approval.py +110 -0
  4. de_agent/cli.py +650 -0
  5. de_agent/cliclass.py +1651 -0
  6. de_agent/clients.py +74 -0
  7. de_agent/collectors/__init__.py +31 -0
  8. de_agent/collectors/_airflow_core.py +343 -0
  9. de_agent/collectors/_pack_http_core.py +219 -0
  10. de_agent/collectors/airflow.py +125 -0
  11. de_agent/collectors/aws/__init__.py +172 -0
  12. de_agent/collectors/aws/_aws_core.py +823 -0
  13. de_agent/collectors/base.py +66 -0
  14. de_agent/collectors/dbt.py +198 -0
  15. de_agent/collectors/pack_base.py +225 -0
  16. de_agent/collectors/pack_cli.py +93 -0
  17. de_agent/collectors/pack_http.py +137 -0
  18. de_agent/collectors/pack_sql.py +217 -0
  19. de_agent/collectors/postgres.py +285 -0
  20. de_agent/collectors/redshift.py +815 -0
  21. de_agent/config.py +450 -0
  22. de_agent/configdoc.py +115 -0
  23. de_agent/data/sql_cases.yaml +1735 -0
  24. de_agent/diff.py +280 -0
  25. de_agent/extract.py +662 -0
  26. de_agent/find.py +218 -0
  27. de_agent/hubpath.py +32 -0
  28. de_agent/initcmd.py +49 -0
  29. de_agent/lint.py +297 -0
  30. de_agent/packs/__init__.py +5 -0
  31. de_agent/packs/argv.py +269 -0
  32. de_agent/packs/bigquery.yaml +67 -0
  33. de_agent/packs/check.py +228 -0
  34. de_agent/packs/cli.py +147 -0
  35. de_agent/packs/dagster.yaml +59 -0
  36. de_agent/packs/databricks-system.yaml +44 -0
  37. de_agent/packs/databricks.yaml +56 -0
  38. de_agent/packs/errors.py +6 -0
  39. de_agent/packs/expr.py +314 -0
  40. de_agent/packs/fivetran.yaml +30 -0
  41. de_agent/packs/gcp.yaml +53 -0
  42. de_agent/packs/kubernetes.yaml +64 -0
  43. de_agent/packs/loader.py +85 -0
  44. de_agent/packs/model.py +154 -0
  45. de_agent/packs/mssql.yaml +61 -0
  46. de_agent/packs/mysql.yaml +53 -0
  47. de_agent/packs/postgres-lite.yaml +65 -0
  48. de_agent/packs/scaffold.py +75 -0
  49. de_agent/packs/snowflake.yaml +91 -0
  50. de_agent/packs/template.py +111 -0
  51. de_agent/packs/view.py +64 -0
  52. de_agent/plans.py +621 -0
  53. de_agent/pyscan.py +214 -0
  54. de_agent/render.py +854 -0
  55. de_agent/runner.py +66 -0
  56. de_agent/scanners/__init__.py +1 -0
  57. de_agent/scanners/airflow.py +691 -0
  58. de_agent/scanners/catalog.py +205 -0
  59. de_agent/scanners/common.py +146 -0
  60. de_agent/scanners/dbt.py +99 -0
  61. de_agent/scanners/edges.py +358 -0
  62. de_agent/scanners/headers.py +117 -0
  63. de_agent/scanners/pycode.py +719 -0
  64. de_agent/scanners/pyexpr.py +308 -0
  65. de_agent/scanners/python.py +240 -0
  66. de_agent/scanners/run.py +288 -0
  67. de_agent/scanners/sql.py +131 -0
  68. de_agent/scanners/terraform.py +769 -0
  69. de_agent/secretscan.py +28 -0
  70. de_agent/snapshot.py +159 -0
  71. de_agent/sqlclass.py +333 -0
  72. de_agent/sqlexec.py +626 -0
  73. de_agent/stepexec.py +257 -0
  74. de_agent/templates/example/integrations/orders_etl--warehouse.md +10 -0
  75. de_agent/templates/example/map_row.txt +1 -0
  76. de_agent/templates/example/projects/orders_etl.md +21 -0
  77. de_agent/templates/hub/README.md +16 -0
  78. de_agent/templates/hub/conventions.md +15 -0
  79. de_agent/templates/hub/de-agent.yaml +20 -0
  80. de_agent/templates/hub/domains/_template.md +10 -0
  81. de_agent/templates/hub/gitignore.tmpl +1 -0
  82. de_agent/templates/hub/integrations/_template.md +12 -0
  83. de_agent/templates/hub/map.md +17 -0
  84. de_agent/templates/hub/projects/_template.md +24 -0
  85. de_agent/templates/hub/status.md +23 -0
  86. de_agent-0.2.0.dist-info/METADATA +94 -0
  87. de_agent-0.2.0.dist-info/RECORD +91 -0
  88. de_agent-0.2.0.dist-info/WHEEL +4 -0
  89. de_agent-0.2.0.dist-info/entry_points.txt +2 -0
  90. de_agent-0.2.0.dist-info/licenses/LICENSE +202 -0
  91. de_agent-0.2.0.dist-info/licenses/NOTICE +6 -0
de_agent/cli.py ADDED
@@ -0,0 +1,650 @@
1
+ """dea command-line entry point."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import os
6
+ import shutil
7
+ import subprocess
8
+ import sys
9
+ from pathlib import Path
10
+
11
+ import typer
12
+
13
+ from de_agent.sqlclass import classify
14
+
15
+ app = typer.Typer(help="de-agent: ground-truth context and guardrails for data engineering.")
16
+
17
+
18
+ def _require_hub() -> Path:
19
+ from de_agent.hubpath import HINT, find_hub
20
+
21
+ hub = find_hub()
22
+ if hub is None:
23
+ typer.echo(HINT, err=True)
24
+ raise typer.Exit(2)
25
+ return hub
26
+
27
+
28
+ def _load(hub: Path): # noqa: ANN202
29
+ from de_agent.config import ConfigError, load_config
30
+
31
+ try:
32
+ return load_config(hub / "de-agent.yaml")
33
+ except ConfigError as ex:
34
+ typer.echo(f"config error: {ex}", err=True)
35
+ raise typer.Exit(2) from None
36
+
37
+
38
+ @app.command("init")
39
+ def init(
40
+ dir: Path = typer.Argument(Path("."), help="Directory to scaffold the hub in"),
41
+ name: str = typer.Option("My data platform", "--name", help="Platform name"),
42
+ with_example: bool = typer.Option(False, "--with-example", help="Add a small generic example project"),
43
+ ) -> None:
44
+ """Scaffold a hub (de-agent.yaml, map, status, conventions, templates). Never overwrites."""
45
+ from de_agent.initcmd import init_hub
46
+
47
+ created, skipped = init_hub(dir, name, with_example)
48
+ for rel in created:
49
+ typer.echo(f"created {rel}")
50
+ if skipped:
51
+ typer.echo("skipped (already exist): " + ", ".join(skipped))
52
+ typer.echo(f"hub ready at {dir.resolve()}")
53
+
54
+
55
+ @app.command("snapshot")
56
+ def snapshot(
57
+ source: list[str] | None = typer.Option(None, "--source", help="Only this source (repeatable)"),
58
+ no_render: bool = typer.Option(False, "--no-render", help="Write JSON only, skip platform/*.md"),
59
+ deep: bool = typer.Option(False, "--deep", help="Recompute heavy sections instead of reusing recent ones (redshift)"),
60
+ ) -> None:
61
+ """Collect every source concurrently and write .dea/snapshots + platform/*.md."""
62
+ from de_agent.config import ConfigError
63
+ from de_agent.render import signals_for
64
+ from de_agent.runner import result_status
65
+ from de_agent.snapshot import take_snapshot
66
+
67
+ hub = _require_hub()
68
+ cfg = _load(hub)
69
+ if not cfg.sources:
70
+ typer.echo("no sources configured in de-agent.yaml", err=True)
71
+ raise typer.Exit(2)
72
+ try:
73
+ out = take_snapshot(hub, cfg, only=source or None, do_render=not no_render, deep=deep)
74
+ except ConfigError as ex:
75
+ typer.echo(str(ex), err=True)
76
+ raise typer.Exit(2) from None
77
+ for r in out.results:
78
+ typer.echo(f"{r['source']}: {result_status(r)} ({len(signals_for(r))} signals, {r['duration_s']}s)")
79
+ for e in r["errors"]:
80
+ typer.echo(f" error: {e}", err=True)
81
+ typer.echo(f"snapshot {out.snapshot_id}")
82
+ raise typer.Exit(1 if out.failed else 0)
83
+
84
+
85
+ @app.command("diff")
86
+ def diff(
87
+ source: str | None = typer.Option(None, "--source"),
88
+ since: str | None = typer.Option(None, "--since", help="Snapshot id to compare against"),
89
+ as_json: bool = typer.Option(False, "--json"),
90
+ ) -> None:
91
+ """What changed between the latest snapshot and the previous one."""
92
+ from de_agent.diff import DiffError, build_diff, render_markdown
93
+
94
+ hub = _require_hub()
95
+ try:
96
+ report = build_diff(hub, source, since)
97
+ except DiffError as ex:
98
+ typer.echo(str(ex), err=True)
99
+ raise typer.Exit(2) from None
100
+ typer.echo(json.dumps(report, indent=2, default=str) if as_json else render_markdown(report).rstrip("\n"))
101
+
102
+
103
+ @app.command("lint")
104
+ def lint(as_json: bool = typer.Option(False, "--json")) -> None:
105
+ """Check the hub: budgets, links, frontmatter, staleness, secrets. Exit 1 on any error."""
106
+ from de_agent.lint import run_lint
107
+
108
+ hub = _require_hub()
109
+ findings = run_lint(hub)
110
+ errors = [f for f in findings if f.level == "error"]
111
+ if as_json:
112
+ typer.echo(json.dumps([f.as_dict() for f in findings], indent=2))
113
+ else:
114
+ for f in findings:
115
+ typer.echo(f"{f.level.upper():7} {f.rule:19} {f.path}: {f.message}")
116
+ typer.echo(f"{len(errors)} error(s), {len(findings) - len(errors)} warning(s)")
117
+ raise typer.Exit(1 if errors else 0)
118
+
119
+
120
+ @app.command("find")
121
+ def find(
122
+ query: str = typer.Argument(..., help="Identifier or words, e.g. analytics.fct_orders"),
123
+ n: int = typer.Option(10, "-n", help="Max results"),
124
+ as_json: bool = typer.Option(False, "--json"),
125
+ rebuild: bool = typer.Option(False, "--rebuild", help="Force a full reindex"),
126
+ ) -> None:
127
+ """Search hub pages and platform snapshots (exact identifiers first)."""
128
+ from de_agent.find import find as do_find
129
+
130
+ hub = _require_hub()
131
+ hits = do_find(hub, query, n, rebuild)
132
+ if as_json:
133
+ typer.echo(json.dumps([h.as_dict() for h in hits], indent=2))
134
+ else:
135
+ for h in hits:
136
+ typer.echo(f"{h.path}:{h.line} {h.snippet}")
137
+ if not hits:
138
+ typer.echo(f"no hits for {query}")
139
+ raise typer.Exit(0 if hits else 1)
140
+
141
+
142
+ @app.command("scan")
143
+ def scan(
144
+ name: list[str] | None = typer.Option(None, "--name", help="Only this scan (repeatable)"),
145
+ full: bool = typer.Option(False, "--full", help="Ignore the per-file cache and re-parse everything"),
146
+ ) -> None:
147
+ """Statically scan configured repos into catalog/, refresh profile headers and the find index."""
148
+ from de_agent.scanners.run import run_scan
149
+
150
+ hub = _require_hub()
151
+ cfg = _load(hub)
152
+ if not cfg.scans:
153
+ typer.echo("no scans configured in de-agent.yaml (add a `scans:` list)", err=True)
154
+ raise typer.Exit(2)
155
+ out = run_scan(hub, cfg, only=name or None, full=full)
156
+ for s in out.scans:
157
+ if s["status"] == "ok":
158
+ st = s["stats"]
159
+ extra = ", ".join(f"{k}={v}" for k, v in st.items() if k.startswith(("dags", "models", "objects", "owners", "resources", "unresolved")))
160
+ typer.echo(f"{s['name']}: {s['type']} ok, {s['files']} files ({s['parsed']} parsed, {s['cached']} cached), "
161
+ f"{s['duration_s']}s, rev {s['revision']}; {extra}")
162
+ elif s["status"] == "skipped":
163
+ typer.echo(f"warning: {s['name']}: {s['type']} {s['error']}", err=True)
164
+ else:
165
+ typer.echo(f"{s['name']}: {s['type']} ERROR {s['error']}", err=True)
166
+ for proj, status in sorted(out.profiles.items()):
167
+ typer.echo(f"profile {proj}: {status}")
168
+ st = out.edges.get("stats", {})
169
+ typer.echo(f"catalog: {out.pages} files; tables: {st.get('tables', 0)}, cross-project edges: {st.get('cross_project_edges', 0)}")
170
+ shown = {s.get("error") for s in out.scans}
171
+ for e in out.errors:
172
+ if e not in shown:
173
+ typer.echo(f"error: {e}", err=True)
174
+ raise typer.Exit(1 if out.failed else 0)
175
+
176
+
177
+ def _load_edges(hub: Path) -> dict:
178
+ p = hub / "catalog" / "edges.json"
179
+ if not p.is_file():
180
+ typer.echo("catalog/edges.json not found: run `dea scan` first", err=True)
181
+ raise typer.Exit(2)
182
+ return json.loads(p.read_text(encoding="utf-8"))
183
+
184
+
185
+ def _load_symbols(hub: Path) -> dict:
186
+ p = hub / "catalog" / "symbols.json"
187
+ if not p.is_file():
188
+ return {}
189
+ flat: dict = {}
190
+ for scan_name, syms in json.loads(p.read_text(encoding="utf-8")).get("scans", {}).items():
191
+ for q, s in syms.items():
192
+ flat[q] = {**s, "scan": scan_name}
193
+ return flat
194
+
195
+
196
+ def _unknown(kind: str, name: str, pool: list[str]) -> None:
197
+ import difflib
198
+
199
+ names = {p.lower(): p for p in pool}
200
+ close = difflib.get_close_matches(name.lower(), list(names), n=5, cutoff=0.4)
201
+ typer.echo(f"unknown {kind}: {name}", err=True)
202
+ if close:
203
+ typer.echo("closest: " + ", ".join(names[c] for c in close), err=True)
204
+ raise typer.Exit(1)
205
+
206
+
207
+ def _resolve_table(data: dict, name: str) -> str | None:
208
+ """`schema.table` as given, or an unqualified name that matches exactly one table. Ambiguity exits 1 with the candidates."""
209
+ key = name.lower()
210
+ if key in data["tables"]:
211
+ return key
212
+ if "." in key:
213
+ return None
214
+ cands = sorted(t for t in data["tables"] if t.rsplit(".", 1)[-1] == key)
215
+ if len(cands) == 1:
216
+ return cands[0]
217
+ if len(cands) > 1:
218
+ typer.echo(f"ambiguous table {name}: matches {', '.join(cands)}; qualify it as schema.table", err=True)
219
+ raise typer.Exit(1)
220
+ return None
221
+
222
+
223
+ @app.command("edges")
224
+ def edges(table: str = typer.Argument(..., help="schema.table, e.g. analytics.fct_orders")) -> None:
225
+ """Who writes and reads a table (from catalog/edges.json)."""
226
+ hub = _require_hub()
227
+ data = _load_edges(hub)
228
+ key = _resolve_table(data, table)
229
+ if key is None:
230
+ _unknown("table", table, sorted(data["tables"]))
231
+ t = data["tables"][key]
232
+ typer.echo(key)
233
+ for label, items in (("writers", t["writers"]), ("readers", t["readers"])):
234
+ typer.echo(f"{label}:" + ("" if items else " none"))
235
+ for e in items:
236
+ extra = f", proc {e['proc']}" if e.get("proc") else ""
237
+ extra += ", unrolled" if e.get("unrolled") else ""
238
+ extra += ", heuristic" if e.get("heuristic") else ""
239
+ extra += ", pattern" if e.get("pattern") else ""
240
+ extra += ", direction unclear" if e.get("unclear") else ""
241
+ typer.echo(f" {e['id']} [{e['project']}] via {e['via']}{extra} {e.get('file', '')}")
242
+
243
+
244
+ @app.command("impact")
245
+ def impact(
246
+ target: str = typer.Argument(..., help="util symbol (module.func) or schema.table"),
247
+ depth: int = typer.Option(5, "--depth", min=1, max=10, help="Max downstream hops for a table (1-10)"),
248
+ ) -> None:
249
+ """Symbol: the DAGs that use it. Table: downstream readers, transitively, with the path."""
250
+ from de_agent.scanners.edges import downstream
251
+
252
+ hub = _require_hub()
253
+ key = target.lower()
254
+ data = _load_edges(hub)
255
+ syms = _load_symbols(hub)
256
+ if key not in data["tables"] and not any(q == target or q.endswith("." + target) for q in syms):
257
+ key = _resolve_table(data, target) or key
258
+ if key in data["tables"]:
259
+ rows = downstream(data, key, depth)
260
+ typer.echo(f"impact of table {key} (downstream readers, depth <= {depth}): {len(rows)}")
261
+ for r in sorted(rows, key=lambda r: (r["depth"], r["reader"])):
262
+ typer.echo(f" depth {r['depth']} {r['reader']} [{r['project']}] {' -> '.join(r['path'])}")
263
+ return
264
+ match = [q for q in syms if q == target or q.endswith("." + target)]
265
+ if match:
266
+ for q in sorted(match):
267
+ s = syms[q]
268
+ typer.echo(f"impact of symbol {q} ({s['file']}:{s['line']}): used by {len({d['dag_id'] for d in s['dags']})} DAGs")
269
+ for d in s["dags"]:
270
+ typer.echo(f" {d['dag_id']} {d['file']}")
271
+ return
272
+ _unknown("table or symbol", target, [*data["tables"], *syms])
273
+
274
+
275
+ def _fmt_cell(v: object) -> str:
276
+ return "" if v is None else (json.dumps(v, default=str) if isinstance(v, (dict, list)) else str(v))
277
+
278
+
279
+ def _print_table(rows: list[dict]) -> None:
280
+ if not rows:
281
+ typer.echo("(0 rows)")
282
+ return
283
+ cols = list(rows[0])
284
+ cells = [[_fmt_cell(r.get(c))[:60] for c in cols] for r in rows]
285
+ widths = [max(len(c), *(len(r[i]) for r in cells)) for i, c in enumerate(cols)]
286
+ typer.echo(" ".join(c.ljust(w) for c, w in zip(cols, widths)).rstrip())
287
+ typer.echo(" ".join("-" * w for w in widths))
288
+ for r in cells:
289
+ typer.echo(" ".join(v.ljust(w) for v, w in zip(r, widths)).rstrip())
290
+
291
+
292
+ def _source_dialect(source: str) -> str:
293
+ """SQL dialect for classifying `dea query` input: redshift for redshift sources, else postgres."""
294
+ from de_agent.hubpath import find_hub
295
+
296
+ try:
297
+ hub = find_hub()
298
+ if hub is not None:
299
+ from de_agent.config import load_config
300
+
301
+ src = load_config(hub / "de-agent.yaml").source(source)
302
+ return "redshift" if src.type == "redshift" else "postgres"
303
+ except Exception:
304
+ pass
305
+ return "postgres"
306
+
307
+
308
+ @app.command("query")
309
+ def query(
310
+ source: str = typer.Argument(..., help="Source name from de_agent.yaml"),
311
+ sql: str = typer.Argument(..., help="One read-only statement"),
312
+ fmt: str = typer.Option("table", "--format", help="table | json | csv"),
313
+ limit: int = typer.Option(200, "--limit", min=1, help="Max rows returned"),
314
+ ) -> None:
315
+ """Run one read-only statement on a configured source (read-only transaction, timeouts, row cap)."""
316
+ import csv
317
+
318
+ from de_agent.collectors.base import CollectorError
319
+ from de_agent.config import ConfigError
320
+ from de_agent.sqlexec import query_limited
321
+
322
+ if fmt not in ("table", "json", "csv"):
323
+ typer.echo("--format must be table, json or csv", err=True)
324
+ raise typer.Exit(2)
325
+ dialect = _source_dialect(source)
326
+ stmts = classify(sql, dialect)
327
+ writes = [st for st in stmts if st.kind != "read"]
328
+ if writes or not stmts:
329
+ why = f"{writes[0].label} is a write" if writes else "empty statement"
330
+ typer.echo(f"refused: {why}; dea query is read-only (writes go through a plan: dea plan check, dea approve, dea apply)",
331
+ err=True)
332
+ raise typer.Exit(2)
333
+ if len(stmts) != 1:
334
+ typer.echo(f"refused: dea query runs exactly one statement (got {len(stmts)})", err=True)
335
+ raise typer.Exit(2)
336
+ hub = _require_hub()
337
+ cfg = _load(hub)
338
+ try:
339
+ src = cfg.source(source)
340
+ except ConfigError:
341
+ typer.echo(f"unknown source {source!r}; configured: {', '.join(s.name for s in cfg.sources if s.conn or s.data_api) or 'none'}",
342
+ err=True)
343
+ raise typer.Exit(2) from None
344
+ if src.type in ("sql", "cli", "http"):
345
+ typer.echo(f"source {source!r} is a pack source: use `dea pack test {source}` (dea query is for postgres and redshift sources)", err=True)
346
+ raise typer.Exit(2)
347
+ if src.conn is None and src.data_api is None:
348
+ typer.echo(f"source {source!r} is not a SQL source (type {src.type})", err=True)
349
+ raise typer.Exit(2)
350
+ if src.writable:
351
+ typer.echo(f"source {source!r} is a write-capable source for plans; query a read-only source instead", err=True)
352
+ raise typer.Exit(2)
353
+ try:
354
+ if src.type == "redshift":
355
+ from de_agent.sqlexec import query_limited_redshift
356
+
357
+ rows, total = query_limited_redshift(src, sql, float(src.timeout_seconds), limit)
358
+ else:
359
+ rows, total = query_limited(src.conn, sql, src.exec, float(src.timeout_seconds), limit)
360
+ except CollectorError as ex:
361
+ typer.echo(f"query failed: {ex}", err=True)
362
+ raise typer.Exit(1) from None
363
+ except Exception as ex: # never a traceback for the caller (an agent reads this output)
364
+ typer.echo(f"query failed: {type(ex).__name__}: {ex}", err=True)
365
+ raise typer.Exit(1) from None
366
+ more = total - len(rows)
367
+ if fmt == "json":
368
+ typer.echo(json.dumps({"rows": rows, "total": total, "more_rows": max(more, 0)}, default=str, indent=2))
369
+ return
370
+ if fmt == "csv":
371
+ import io
372
+
373
+ buf = io.StringIO()
374
+ if rows:
375
+ w = csv.DictWriter(buf, fieldnames=list(rows[0]))
376
+ w.writeheader()
377
+ for r in rows:
378
+ w.writerow({k: _fmt_cell(v) for k, v in r.items()})
379
+ typer.echo(buf.getvalue().rstrip("\n"))
380
+ else:
381
+ _print_table(rows)
382
+ if more > 0:
383
+ typer.echo(f"({more} more rows not shown; raise --limit)", err=fmt == "csv")
384
+
385
+
386
+ plan_app = typer.Typer(help="Plan files: .dea/plans/<id>.md")
387
+ app.add_typer(plan_app, name="plan")
388
+
389
+
390
+ def _plan_and_cfg(plan_id: str): # noqa: ANN202
391
+ from de_agent.plans import PlanError, load_plan
392
+
393
+ hub = _require_hub()
394
+ cfg = _load(hub)
395
+ try:
396
+ return hub, cfg, load_plan(hub, plan_id)
397
+ except PlanError as ex:
398
+ typer.echo(str(ex), err=True)
399
+ raise typer.Exit(2) from None
400
+
401
+
402
+ @plan_app.command("check")
403
+ def plan_check(plan_id: str = typer.Argument(..., help="Plan id (file name without .md)"),
404
+ as_json: bool = typer.Option(False, "--json")) -> None:
405
+ """Classify every statement in a plan and list errors. Exit 1 on any error."""
406
+ from datetime import datetime, timezone
407
+
408
+ from de_agent.plans import check_plan, render_report, write_checked
409
+
410
+ hub, cfg, plan = _plan_and_cfg(plan_id)
411
+ rep = check_plan(plan, cfg, filename_id=plan_id.removesuffix(".md"), hub=hub)
412
+ if as_json:
413
+ typer.echo(json.dumps({**rep.as_dict(), "hashes": rep.hashes}, indent=2))
414
+ else:
415
+ typer.echo(render_report(plan, rep))
416
+ if rep.ok:
417
+ write_checked(hub, plan, rep, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))
418
+ raise typer.Exit(0 if rep.ok else 1)
419
+
420
+
421
+ @app.command("approve")
422
+ def approve(plan_id: str = typer.Argument(..., help="Plan id")) -> None:
423
+ """HUMAN ONLY: review the plan check, type the plan id, sign the statement hashes (needs a TTY)."""
424
+ from datetime import datetime, timezone
425
+
426
+ from de_agent import approval
427
+ from de_agent.plans import check_plan, plan_hashes, plans_dir, render_report
428
+
429
+ if not (sys.stdin.isatty() and sys.stdout.isatty()):
430
+ typer.echo("dea approve is human-only and needs an interactive terminal: run this in your own terminal.", err=True)
431
+ raise typer.Exit(2)
432
+ hub, cfg, plan = _plan_and_cfg(plan_id)
433
+ rep = check_plan(plan, cfg, filename_id=plan_id.removesuffix(".md"), hub=hub)
434
+ typer.echo(render_report(plan, rep))
435
+ if not rep.ok:
436
+ typer.echo("\nrefusing to approve a plan with errors", err=True)
437
+ raise typer.Exit(1)
438
+ typer.echo("\n" + plan.text.split("\n## Rollback")[0][:6000])
439
+ typed = input(f"\nType the plan id ({plan.id}) to approve it, anything else to cancel: ").strip()
440
+ if typed != plan.id:
441
+ typer.echo("cancelled; nothing written", err=True)
442
+ raise typer.Exit(1)
443
+ try:
444
+ who = subprocess.run(["git", "config", "user.email"], capture_output=True, text=True, cwd=hub,
445
+ timeout=10).stdout.strip()
446
+ except (OSError, subprocess.TimeoutExpired):
447
+ who = ""
448
+ who = who or os.environ.get("USER") or os.environ.get("USERNAME") or "unknown"
449
+ key = approval.load_key(create=True)
450
+ data = approval.make_approval(plan.id, plan_hashes(plan), who,
451
+ datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), key)
452
+ out = plans_dir(hub) / f"{plan.id}.approved"
453
+ approval.write_approval(str(out), data)
454
+ typer.echo(f"approved {plan.id} ({len(data['hashes'])} statement hashes) by {who}: {out}")
455
+
456
+
457
+ @app.command("apply")
458
+ def apply(
459
+ plan_id: str = typer.Argument(..., help="Plan id"),
460
+ rollback: bool = typer.Option(False, "--rollback", help="Run the Rollback section instead"),
461
+ dry_run: bool = typer.Option(False, "--dry-run", help="Print what would run"),
462
+ ) -> None:
463
+ """Run an approved plan step by step (executor agent only; verifies the HMAC and every statement hash first)."""
464
+ from de_agent import approval
465
+ from de_agent.applyplan import ApplyRefused, apply_plan, authorize, dry_run_lines
466
+
467
+ hub, cfg, plan = _plan_and_cfg(plan_id)
468
+ key = approval.load_key()
469
+ try:
470
+ if dry_run:
471
+ authorize(hub, cfg, plan, rollback, key)
472
+ typer.echo("\n".join(dry_run_lines(plan, rollback)))
473
+ typer.echo("dry run: nothing executed")
474
+ return
475
+ res = apply_plan(hub, cfg, plan, rollback=rollback, key=key)
476
+ except ApplyRefused as ex:
477
+ typer.echo(f"refused: {ex}", err=True)
478
+ raise typer.Exit(2) from None
479
+ for st in res.steps:
480
+ line = f"step {st['step']} {st['status']} rowcount={st['rowcount']} {st['duration_s']}s"
481
+ typer.echo(line + (f" error: {st['error']}" if st["error"] else ""))
482
+ for act in st.get("actions", []):
483
+ if act["kind"] == "diff":
484
+ typer.echo(f" {act['mode']} diff {act['repo']}/{act['file']}: {act.get('diff_stat') or '(no stat)'}".replace("\n", " | "))
485
+ else:
486
+ typer.echo(f" shell exit={act['exit_code']}: {act['command'][:100]}")
487
+ for v in st["verify"]:
488
+ typer.echo(f" verify {'ok' if v['ok'] else 'FAILED'}: value={v['value']!r} expect={v['expect']}")
489
+ typer.echo(f"{res.mode} {res.status}: {res.path}")
490
+ raise typer.Exit(0 if res.status == "ok" else 1)
491
+
492
+
493
+ @app.command("plans")
494
+ def plans_cmd() -> None:
495
+ """List plans with their state: draft / checked / approved / applied / failed / rolled back."""
496
+ from de_agent import approval
497
+ from de_agent.plans import load_plan, plan_state, plans_dir
498
+
499
+ hub = _require_hub()
500
+ key = approval.load_key()
501
+ paths = sorted(plans_dir(hub).glob("*.md")) if plans_dir(hub).is_dir() else []
502
+ if not paths:
503
+ typer.echo("no plans (.dea/plans/ is empty)")
504
+ return
505
+ for p in paths:
506
+ try:
507
+ pl = load_plan(hub, p.stem)
508
+ meta = f"{pl.risk or '?':6} {pl.title}"
509
+ except Exception:
510
+ meta = "(unreadable)"
511
+ typer.echo(f"{p.stem} {plan_state(hub, p.stem, key):12} {meta}")
512
+
513
+
514
+ def _hook_python() -> str:
515
+ """The python3 the hooks will run: DEA_HOOK_PYTHON, else the first `python3` on PATH once this venv's bin dir is
516
+ removed (a pipx/uv tool venv is not what Claude Code's hook command resolves)."""
517
+ env = os.environ.get("DEA_HOOK_PYTHON")
518
+ if env:
519
+ return env
520
+ skip = {os.path.realpath(os.path.dirname(sys.executable))} if sys.prefix != getattr(sys, "base_prefix", sys.prefix) else set()
521
+ if os.environ.get("VIRTUAL_ENV"):
522
+ skip.add(os.path.realpath(os.path.join(os.environ["VIRTUAL_ENV"], "bin")))
523
+ path = os.pathsep.join(d for d in os.environ.get("PATH", "").split(os.pathsep) if d and os.path.realpath(d) not in skip)
524
+ return shutil.which("python3", path=path) or "python3"
525
+
526
+
527
+ def _plugin_root() -> Path | None:
528
+ env = os.environ.get("CLAUDE_PLUGIN_ROOT")
529
+ cands = [Path(env)] if env else []
530
+ cands.append(Path(__file__).resolve().parents[3])
531
+ return next((c for c in cands if (c / "scripts" / "_vendor").is_dir()), None)
532
+
533
+
534
+ @app.command("doctor")
535
+ def doctor() -> None:
536
+ """Check the environment: python, hub, config, sources, psql, FTS5, hook python + vendored sqlglot."""
537
+ from de_agent.collectors import get_collector
538
+ from de_agent.collectors.base import RunContext
539
+ from de_agent.config import ConfigError, load_config
540
+ from de_agent.find import fts5_available
541
+ from de_agent.hubpath import find_hub
542
+
543
+ rows: list[tuple[str, str, str]] = [] # (mark, name, detail)
544
+
545
+ def add(ok: bool | None, name: str, detail: str) -> None:
546
+ rows.append(("✓" if ok else ("✗" if ok is False else "-"), name, detail))
547
+
548
+ v = sys.version_info
549
+ add(v >= (3, 10), "python", f"{v.major}.{v.minor}.{v.micro} (CLI needs >= 3.10)")
550
+ hub = find_hub()
551
+ add(hub is not None, "hub", str(hub) if hub else "not found (dea init, or set DEA_HUB)")
552
+ cfg = None
553
+ if hub:
554
+ try:
555
+ cfg = load_config(hub / "de-agent.yaml")
556
+ add(True, "config", f"valid, {len(cfg.sources)} source(s)")
557
+ except ConfigError as ex:
558
+ add(False, "config", str(ex))
559
+ for s in cfg.sources if cfg else []:
560
+ try:
561
+ detail = get_collector(s.type).ping(s, RunContext(timeout_seconds=15))
562
+ add(True, f"source {s.name}", f"{s.type} reachable via {s.exec} ({detail})")
563
+ except Exception as ex:
564
+ add(False, f"source {s.name}", str(ex).splitlines()[-1][:120] if str(ex) else type(ex).__name__)
565
+ if hub:
566
+ from de_agent.packs.cli import doctor_rows
567
+
568
+ for ok, name, detail in doctor_rows():
569
+ add(ok, name, detail)
570
+ psql = shutil.which("psql")
571
+ add(psql is not None, "psql", psql or "not on PATH (needed for ssh sources and the no-psycopg fallback)")
572
+ add(fts5_available(), "fts5", "SQLite FTS5 available" if fts5_available() else "missing: dea find falls back to a slow scan")
573
+ py = _hook_python()
574
+ try:
575
+ out = subprocess.run([py, "-c", "import sys;print('%d.%d' % sys.version_info[:2])"],
576
+ capture_output=True, text=True, timeout=10).stdout.strip()
577
+ major, minor = (int(x) for x in out.split("."))
578
+ add((major, minor) >= (3, 9), "hook python", f"{py} is {out} (the python3 the hooks run; needs >= 3.9)")
579
+ except Exception as ex:
580
+ add(False, "hook python", f"cannot run {py}: {ex}")
581
+ py = ""
582
+ root = _plugin_root()
583
+ if root and py:
584
+ vend = root / "scripts" / "_vendor"
585
+ r = subprocess.run([py, "-c", "import sys;sys.path.insert(0, sys.argv[1]);import sqlglot;print(sqlglot.__version__)", str(vend)],
586
+ capture_output=True, text=True, timeout=30)
587
+ add(r.returncode == 0, "vendored sqlglot", f"v{r.stdout.strip()} under {py}" if r.returncode == 0 else r.stderr.strip()[-100:])
588
+ else:
589
+ add(None, "vendored sqlglot", "not checked: set CLAUDE_PLUGIN_ROOT=<your clone of the plugin> to check it")
590
+ width = max(len(n) for _, n, _ in rows)
591
+ for mark, name, detail in rows:
592
+ typer.echo(f"{mark} {name.ljust(width)} {detail}")
593
+ raise typer.Exit(1 if any(m == "✗" for m, _, _ in rows) else 0)
594
+
595
+
596
+ config_app = typer.Typer(help="de-agent.yaml tooling.", no_args_is_help=True)
597
+ app.add_typer(config_app, name="config")
598
+
599
+
600
+ @config_app.command("schema")
601
+ def config_schema(
602
+ markdown: bool = typer.Option(False, "--markdown", help="Markdown reference (docs/configuration.md) instead of JSON Schema"),
603
+ out: Path | None = typer.Option(None, "--out", "-o", help="Write to this file instead of stdout"),
604
+ ) -> None:
605
+ """Print the de-agent.yaml JSON Schema (default) or its markdown reference, generated from the config models."""
606
+ from de_agent.configdoc import json_text
607
+ from de_agent.configdoc import markdown as md_text
608
+
609
+ text = md_text() if markdown else json_text()
610
+ if out:
611
+ out.write_text(text if text.endswith("\n") else text + "\n", encoding="utf-8")
612
+ typer.echo(f"wrote {out}")
613
+ else:
614
+ typer.echo(text.rstrip("\n"))
615
+
616
+
617
+ @app.command("guard-test")
618
+ def guard_test(
619
+ corpus: Path | None = typer.Option(None, help="YAML corpus of SQL classification cases (default: the one shipped in the package)"),
620
+ ) -> None:
621
+ """Run the SQL classifier corpus (the packaged one unless --corpus is given)."""
622
+ import yaml
623
+
624
+ if corpus is None:
625
+ corpus = Path(__file__).resolve().parent / "data" / "sql_cases.yaml"
626
+ cases = yaml.safe_load(corpus.read_text())
627
+ failed = 0
628
+ for c in cases:
629
+ got = classify(c["sql"], c.get("dialect", "postgres"))
630
+ kinds = [s.kind for s in got]
631
+ want = c["kind"] if isinstance(c["kind"], list) else [c["kind"]]
632
+ if kinds != want:
633
+ failed += 1
634
+ typer.echo(f"FAIL {json.dumps(c['sql'])[:80]}: want {want} got {kinds}")
635
+ typer.echo(f"{len(cases) - failed}/{len(cases)} passed")
636
+ raise typer.Exit(1 if failed else 0)
637
+
638
+
639
+ def _register_packs() -> None:
640
+ from de_agent.packs.cli import pack_app
641
+
642
+ app.add_typer(pack_app, name="pack")
643
+
644
+
645
+ _register_packs()
646
+
647
+
648
+ @app.callback()
649
+ def main() -> None:
650
+ """de-agent CLI."""