de-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- de_agent/__init__.py +1 -0
- de_agent/applyplan.py +270 -0
- de_agent/approval.py +110 -0
- de_agent/cli.py +650 -0
- de_agent/cliclass.py +1651 -0
- de_agent/clients.py +74 -0
- de_agent/collectors/__init__.py +31 -0
- de_agent/collectors/_airflow_core.py +343 -0
- de_agent/collectors/_pack_http_core.py +219 -0
- de_agent/collectors/airflow.py +125 -0
- de_agent/collectors/aws/__init__.py +172 -0
- de_agent/collectors/aws/_aws_core.py +823 -0
- de_agent/collectors/base.py +66 -0
- de_agent/collectors/dbt.py +198 -0
- de_agent/collectors/pack_base.py +225 -0
- de_agent/collectors/pack_cli.py +93 -0
- de_agent/collectors/pack_http.py +137 -0
- de_agent/collectors/pack_sql.py +217 -0
- de_agent/collectors/postgres.py +285 -0
- de_agent/collectors/redshift.py +815 -0
- de_agent/config.py +450 -0
- de_agent/configdoc.py +115 -0
- de_agent/data/sql_cases.yaml +1735 -0
- de_agent/diff.py +280 -0
- de_agent/extract.py +662 -0
- de_agent/find.py +218 -0
- de_agent/hubpath.py +32 -0
- de_agent/initcmd.py +49 -0
- de_agent/lint.py +297 -0
- de_agent/packs/__init__.py +5 -0
- de_agent/packs/argv.py +269 -0
- de_agent/packs/bigquery.yaml +67 -0
- de_agent/packs/check.py +228 -0
- de_agent/packs/cli.py +147 -0
- de_agent/packs/dagster.yaml +59 -0
- de_agent/packs/databricks-system.yaml +44 -0
- de_agent/packs/databricks.yaml +56 -0
- de_agent/packs/errors.py +6 -0
- de_agent/packs/expr.py +314 -0
- de_agent/packs/fivetran.yaml +30 -0
- de_agent/packs/gcp.yaml +53 -0
- de_agent/packs/kubernetes.yaml +64 -0
- de_agent/packs/loader.py +85 -0
- de_agent/packs/model.py +154 -0
- de_agent/packs/mssql.yaml +61 -0
- de_agent/packs/mysql.yaml +53 -0
- de_agent/packs/postgres-lite.yaml +65 -0
- de_agent/packs/scaffold.py +75 -0
- de_agent/packs/snowflake.yaml +91 -0
- de_agent/packs/template.py +111 -0
- de_agent/packs/view.py +64 -0
- de_agent/plans.py +621 -0
- de_agent/pyscan.py +214 -0
- de_agent/render.py +854 -0
- de_agent/runner.py +66 -0
- de_agent/scanners/__init__.py +1 -0
- de_agent/scanners/airflow.py +691 -0
- de_agent/scanners/catalog.py +205 -0
- de_agent/scanners/common.py +146 -0
- de_agent/scanners/dbt.py +99 -0
- de_agent/scanners/edges.py +358 -0
- de_agent/scanners/headers.py +117 -0
- de_agent/scanners/pycode.py +719 -0
- de_agent/scanners/pyexpr.py +308 -0
- de_agent/scanners/python.py +240 -0
- de_agent/scanners/run.py +288 -0
- de_agent/scanners/sql.py +131 -0
- de_agent/scanners/terraform.py +769 -0
- de_agent/secretscan.py +28 -0
- de_agent/snapshot.py +159 -0
- de_agent/sqlclass.py +333 -0
- de_agent/sqlexec.py +626 -0
- de_agent/stepexec.py +257 -0
- de_agent/templates/example/integrations/orders_etl--warehouse.md +10 -0
- de_agent/templates/example/map_row.txt +1 -0
- de_agent/templates/example/projects/orders_etl.md +21 -0
- de_agent/templates/hub/README.md +16 -0
- de_agent/templates/hub/conventions.md +15 -0
- de_agent/templates/hub/de-agent.yaml +20 -0
- de_agent/templates/hub/domains/_template.md +10 -0
- de_agent/templates/hub/gitignore.tmpl +1 -0
- de_agent/templates/hub/integrations/_template.md +12 -0
- de_agent/templates/hub/map.md +17 -0
- de_agent/templates/hub/projects/_template.md +24 -0
- de_agent/templates/hub/status.md +23 -0
- de_agent-0.2.0.dist-info/METADATA +94 -0
- de_agent-0.2.0.dist-info/RECORD +91 -0
- de_agent-0.2.0.dist-info/WHEEL +4 -0
- de_agent-0.2.0.dist-info/entry_points.txt +2 -0
- de_agent-0.2.0.dist-info/licenses/LICENSE +202 -0
- de_agent-0.2.0.dist-info/licenses/NOTICE +6 -0
de_agent/cli.py
ADDED
|
@@ -0,0 +1,650 @@
|
|
|
1
|
+
"""dea command-line entry point."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import subprocess
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import typer
|
|
12
|
+
|
|
13
|
+
from de_agent.sqlclass import classify
|
|
14
|
+
|
|
15
|
+
app = typer.Typer(help="de-agent: ground-truth context and guardrails for data engineering.")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _require_hub() -> Path:
|
|
19
|
+
from de_agent.hubpath import HINT, find_hub
|
|
20
|
+
|
|
21
|
+
hub = find_hub()
|
|
22
|
+
if hub is None:
|
|
23
|
+
typer.echo(HINT, err=True)
|
|
24
|
+
raise typer.Exit(2)
|
|
25
|
+
return hub
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _load(hub: Path): # noqa: ANN202
|
|
29
|
+
from de_agent.config import ConfigError, load_config
|
|
30
|
+
|
|
31
|
+
try:
|
|
32
|
+
return load_config(hub / "de-agent.yaml")
|
|
33
|
+
except ConfigError as ex:
|
|
34
|
+
typer.echo(f"config error: {ex}", err=True)
|
|
35
|
+
raise typer.Exit(2) from None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@app.command("init")
|
|
39
|
+
def init(
|
|
40
|
+
dir: Path = typer.Argument(Path("."), help="Directory to scaffold the hub in"),
|
|
41
|
+
name: str = typer.Option("My data platform", "--name", help="Platform name"),
|
|
42
|
+
with_example: bool = typer.Option(False, "--with-example", help="Add a small generic example project"),
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Scaffold a hub (de-agent.yaml, map, status, conventions, templates). Never overwrites."""
|
|
45
|
+
from de_agent.initcmd import init_hub
|
|
46
|
+
|
|
47
|
+
created, skipped = init_hub(dir, name, with_example)
|
|
48
|
+
for rel in created:
|
|
49
|
+
typer.echo(f"created {rel}")
|
|
50
|
+
if skipped:
|
|
51
|
+
typer.echo("skipped (already exist): " + ", ".join(skipped))
|
|
52
|
+
typer.echo(f"hub ready at {dir.resolve()}")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@app.command("snapshot")
|
|
56
|
+
def snapshot(
|
|
57
|
+
source: list[str] | None = typer.Option(None, "--source", help="Only this source (repeatable)"),
|
|
58
|
+
no_render: bool = typer.Option(False, "--no-render", help="Write JSON only, skip platform/*.md"),
|
|
59
|
+
deep: bool = typer.Option(False, "--deep", help="Recompute heavy sections instead of reusing recent ones (redshift)"),
|
|
60
|
+
) -> None:
|
|
61
|
+
"""Collect every source concurrently and write .dea/snapshots + platform/*.md."""
|
|
62
|
+
from de_agent.config import ConfigError
|
|
63
|
+
from de_agent.render import signals_for
|
|
64
|
+
from de_agent.runner import result_status
|
|
65
|
+
from de_agent.snapshot import take_snapshot
|
|
66
|
+
|
|
67
|
+
hub = _require_hub()
|
|
68
|
+
cfg = _load(hub)
|
|
69
|
+
if not cfg.sources:
|
|
70
|
+
typer.echo("no sources configured in de-agent.yaml", err=True)
|
|
71
|
+
raise typer.Exit(2)
|
|
72
|
+
try:
|
|
73
|
+
out = take_snapshot(hub, cfg, only=source or None, do_render=not no_render, deep=deep)
|
|
74
|
+
except ConfigError as ex:
|
|
75
|
+
typer.echo(str(ex), err=True)
|
|
76
|
+
raise typer.Exit(2) from None
|
|
77
|
+
for r in out.results:
|
|
78
|
+
typer.echo(f"{r['source']}: {result_status(r)} ({len(signals_for(r))} signals, {r['duration_s']}s)")
|
|
79
|
+
for e in r["errors"]:
|
|
80
|
+
typer.echo(f" error: {e}", err=True)
|
|
81
|
+
typer.echo(f"snapshot {out.snapshot_id}")
|
|
82
|
+
raise typer.Exit(1 if out.failed else 0)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@app.command("diff")
|
|
86
|
+
def diff(
|
|
87
|
+
source: str | None = typer.Option(None, "--source"),
|
|
88
|
+
since: str | None = typer.Option(None, "--since", help="Snapshot id to compare against"),
|
|
89
|
+
as_json: bool = typer.Option(False, "--json"),
|
|
90
|
+
) -> None:
|
|
91
|
+
"""What changed between the latest snapshot and the previous one."""
|
|
92
|
+
from de_agent.diff import DiffError, build_diff, render_markdown
|
|
93
|
+
|
|
94
|
+
hub = _require_hub()
|
|
95
|
+
try:
|
|
96
|
+
report = build_diff(hub, source, since)
|
|
97
|
+
except DiffError as ex:
|
|
98
|
+
typer.echo(str(ex), err=True)
|
|
99
|
+
raise typer.Exit(2) from None
|
|
100
|
+
typer.echo(json.dumps(report, indent=2, default=str) if as_json else render_markdown(report).rstrip("\n"))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@app.command("lint")
|
|
104
|
+
def lint(as_json: bool = typer.Option(False, "--json")) -> None:
|
|
105
|
+
"""Check the hub: budgets, links, frontmatter, staleness, secrets. Exit 1 on any error."""
|
|
106
|
+
from de_agent.lint import run_lint
|
|
107
|
+
|
|
108
|
+
hub = _require_hub()
|
|
109
|
+
findings = run_lint(hub)
|
|
110
|
+
errors = [f for f in findings if f.level == "error"]
|
|
111
|
+
if as_json:
|
|
112
|
+
typer.echo(json.dumps([f.as_dict() for f in findings], indent=2))
|
|
113
|
+
else:
|
|
114
|
+
for f in findings:
|
|
115
|
+
typer.echo(f"{f.level.upper():7} {f.rule:19} {f.path}: {f.message}")
|
|
116
|
+
typer.echo(f"{len(errors)} error(s), {len(findings) - len(errors)} warning(s)")
|
|
117
|
+
raise typer.Exit(1 if errors else 0)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@app.command("find")
|
|
121
|
+
def find(
|
|
122
|
+
query: str = typer.Argument(..., help="Identifier or words, e.g. analytics.fct_orders"),
|
|
123
|
+
n: int = typer.Option(10, "-n", help="Max results"),
|
|
124
|
+
as_json: bool = typer.Option(False, "--json"),
|
|
125
|
+
rebuild: bool = typer.Option(False, "--rebuild", help="Force a full reindex"),
|
|
126
|
+
) -> None:
|
|
127
|
+
"""Search hub pages and platform snapshots (exact identifiers first)."""
|
|
128
|
+
from de_agent.find import find as do_find
|
|
129
|
+
|
|
130
|
+
hub = _require_hub()
|
|
131
|
+
hits = do_find(hub, query, n, rebuild)
|
|
132
|
+
if as_json:
|
|
133
|
+
typer.echo(json.dumps([h.as_dict() for h in hits], indent=2))
|
|
134
|
+
else:
|
|
135
|
+
for h in hits:
|
|
136
|
+
typer.echo(f"{h.path}:{h.line} {h.snippet}")
|
|
137
|
+
if not hits:
|
|
138
|
+
typer.echo(f"no hits for {query}")
|
|
139
|
+
raise typer.Exit(0 if hits else 1)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@app.command("scan")
|
|
143
|
+
def scan(
|
|
144
|
+
name: list[str] | None = typer.Option(None, "--name", help="Only this scan (repeatable)"),
|
|
145
|
+
full: bool = typer.Option(False, "--full", help="Ignore the per-file cache and re-parse everything"),
|
|
146
|
+
) -> None:
|
|
147
|
+
"""Statically scan configured repos into catalog/, refresh profile headers and the find index."""
|
|
148
|
+
from de_agent.scanners.run import run_scan
|
|
149
|
+
|
|
150
|
+
hub = _require_hub()
|
|
151
|
+
cfg = _load(hub)
|
|
152
|
+
if not cfg.scans:
|
|
153
|
+
typer.echo("no scans configured in de-agent.yaml (add a `scans:` list)", err=True)
|
|
154
|
+
raise typer.Exit(2)
|
|
155
|
+
out = run_scan(hub, cfg, only=name or None, full=full)
|
|
156
|
+
for s in out.scans:
|
|
157
|
+
if s["status"] == "ok":
|
|
158
|
+
st = s["stats"]
|
|
159
|
+
extra = ", ".join(f"{k}={v}" for k, v in st.items() if k.startswith(("dags", "models", "objects", "owners", "resources", "unresolved")))
|
|
160
|
+
typer.echo(f"{s['name']}: {s['type']} ok, {s['files']} files ({s['parsed']} parsed, {s['cached']} cached), "
|
|
161
|
+
f"{s['duration_s']}s, rev {s['revision']}; {extra}")
|
|
162
|
+
elif s["status"] == "skipped":
|
|
163
|
+
typer.echo(f"warning: {s['name']}: {s['type']} {s['error']}", err=True)
|
|
164
|
+
else:
|
|
165
|
+
typer.echo(f"{s['name']}: {s['type']} ERROR {s['error']}", err=True)
|
|
166
|
+
for proj, status in sorted(out.profiles.items()):
|
|
167
|
+
typer.echo(f"profile {proj}: {status}")
|
|
168
|
+
st = out.edges.get("stats", {})
|
|
169
|
+
typer.echo(f"catalog: {out.pages} files; tables: {st.get('tables', 0)}, cross-project edges: {st.get('cross_project_edges', 0)}")
|
|
170
|
+
shown = {s.get("error") for s in out.scans}
|
|
171
|
+
for e in out.errors:
|
|
172
|
+
if e not in shown:
|
|
173
|
+
typer.echo(f"error: {e}", err=True)
|
|
174
|
+
raise typer.Exit(1 if out.failed else 0)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _load_edges(hub: Path) -> dict:
|
|
178
|
+
p = hub / "catalog" / "edges.json"
|
|
179
|
+
if not p.is_file():
|
|
180
|
+
typer.echo("catalog/edges.json not found: run `dea scan` first", err=True)
|
|
181
|
+
raise typer.Exit(2)
|
|
182
|
+
return json.loads(p.read_text(encoding="utf-8"))
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _load_symbols(hub: Path) -> dict:
|
|
186
|
+
p = hub / "catalog" / "symbols.json"
|
|
187
|
+
if not p.is_file():
|
|
188
|
+
return {}
|
|
189
|
+
flat: dict = {}
|
|
190
|
+
for scan_name, syms in json.loads(p.read_text(encoding="utf-8")).get("scans", {}).items():
|
|
191
|
+
for q, s in syms.items():
|
|
192
|
+
flat[q] = {**s, "scan": scan_name}
|
|
193
|
+
return flat
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _unknown(kind: str, name: str, pool: list[str]) -> None:
|
|
197
|
+
import difflib
|
|
198
|
+
|
|
199
|
+
names = {p.lower(): p for p in pool}
|
|
200
|
+
close = difflib.get_close_matches(name.lower(), list(names), n=5, cutoff=0.4)
|
|
201
|
+
typer.echo(f"unknown {kind}: {name}", err=True)
|
|
202
|
+
if close:
|
|
203
|
+
typer.echo("closest: " + ", ".join(names[c] for c in close), err=True)
|
|
204
|
+
raise typer.Exit(1)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _resolve_table(data: dict, name: str) -> str | None:
|
|
208
|
+
"""`schema.table` as given, or an unqualified name that matches exactly one table. Ambiguity exits 1 with the candidates."""
|
|
209
|
+
key = name.lower()
|
|
210
|
+
if key in data["tables"]:
|
|
211
|
+
return key
|
|
212
|
+
if "." in key:
|
|
213
|
+
return None
|
|
214
|
+
cands = sorted(t for t in data["tables"] if t.rsplit(".", 1)[-1] == key)
|
|
215
|
+
if len(cands) == 1:
|
|
216
|
+
return cands[0]
|
|
217
|
+
if len(cands) > 1:
|
|
218
|
+
typer.echo(f"ambiguous table {name}: matches {', '.join(cands)}; qualify it as schema.table", err=True)
|
|
219
|
+
raise typer.Exit(1)
|
|
220
|
+
return None
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
@app.command("edges")
|
|
224
|
+
def edges(table: str = typer.Argument(..., help="schema.table, e.g. analytics.fct_orders")) -> None:
|
|
225
|
+
"""Who writes and reads a table (from catalog/edges.json)."""
|
|
226
|
+
hub = _require_hub()
|
|
227
|
+
data = _load_edges(hub)
|
|
228
|
+
key = _resolve_table(data, table)
|
|
229
|
+
if key is None:
|
|
230
|
+
_unknown("table", table, sorted(data["tables"]))
|
|
231
|
+
t = data["tables"][key]
|
|
232
|
+
typer.echo(key)
|
|
233
|
+
for label, items in (("writers", t["writers"]), ("readers", t["readers"])):
|
|
234
|
+
typer.echo(f"{label}:" + ("" if items else " none"))
|
|
235
|
+
for e in items:
|
|
236
|
+
extra = f", proc {e['proc']}" if e.get("proc") else ""
|
|
237
|
+
extra += ", unrolled" if e.get("unrolled") else ""
|
|
238
|
+
extra += ", heuristic" if e.get("heuristic") else ""
|
|
239
|
+
extra += ", pattern" if e.get("pattern") else ""
|
|
240
|
+
extra += ", direction unclear" if e.get("unclear") else ""
|
|
241
|
+
typer.echo(f" {e['id']} [{e['project']}] via {e['via']}{extra} {e.get('file', '')}")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
@app.command("impact")
|
|
245
|
+
def impact(
|
|
246
|
+
target: str = typer.Argument(..., help="util symbol (module.func) or schema.table"),
|
|
247
|
+
depth: int = typer.Option(5, "--depth", min=1, max=10, help="Max downstream hops for a table (1-10)"),
|
|
248
|
+
) -> None:
|
|
249
|
+
"""Symbol: the DAGs that use it. Table: downstream readers, transitively, with the path."""
|
|
250
|
+
from de_agent.scanners.edges import downstream
|
|
251
|
+
|
|
252
|
+
hub = _require_hub()
|
|
253
|
+
key = target.lower()
|
|
254
|
+
data = _load_edges(hub)
|
|
255
|
+
syms = _load_symbols(hub)
|
|
256
|
+
if key not in data["tables"] and not any(q == target or q.endswith("." + target) for q in syms):
|
|
257
|
+
key = _resolve_table(data, target) or key
|
|
258
|
+
if key in data["tables"]:
|
|
259
|
+
rows = downstream(data, key, depth)
|
|
260
|
+
typer.echo(f"impact of table {key} (downstream readers, depth <= {depth}): {len(rows)}")
|
|
261
|
+
for r in sorted(rows, key=lambda r: (r["depth"], r["reader"])):
|
|
262
|
+
typer.echo(f" depth {r['depth']} {r['reader']} [{r['project']}] {' -> '.join(r['path'])}")
|
|
263
|
+
return
|
|
264
|
+
match = [q for q in syms if q == target or q.endswith("." + target)]
|
|
265
|
+
if match:
|
|
266
|
+
for q in sorted(match):
|
|
267
|
+
s = syms[q]
|
|
268
|
+
typer.echo(f"impact of symbol {q} ({s['file']}:{s['line']}): used by {len({d['dag_id'] for d in s['dags']})} DAGs")
|
|
269
|
+
for d in s["dags"]:
|
|
270
|
+
typer.echo(f" {d['dag_id']} {d['file']}")
|
|
271
|
+
return
|
|
272
|
+
_unknown("table or symbol", target, [*data["tables"], *syms])
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _fmt_cell(v: object) -> str:
|
|
276
|
+
return "" if v is None else (json.dumps(v, default=str) if isinstance(v, (dict, list)) else str(v))
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _print_table(rows: list[dict]) -> None:
|
|
280
|
+
if not rows:
|
|
281
|
+
typer.echo("(0 rows)")
|
|
282
|
+
return
|
|
283
|
+
cols = list(rows[0])
|
|
284
|
+
cells = [[_fmt_cell(r.get(c))[:60] for c in cols] for r in rows]
|
|
285
|
+
widths = [max(len(c), *(len(r[i]) for r in cells)) for i, c in enumerate(cols)]
|
|
286
|
+
typer.echo(" ".join(c.ljust(w) for c, w in zip(cols, widths)).rstrip())
|
|
287
|
+
typer.echo(" ".join("-" * w for w in widths))
|
|
288
|
+
for r in cells:
|
|
289
|
+
typer.echo(" ".join(v.ljust(w) for v, w in zip(r, widths)).rstrip())
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _source_dialect(source: str) -> str:
|
|
293
|
+
"""SQL dialect for classifying `dea query` input: redshift for redshift sources, else postgres."""
|
|
294
|
+
from de_agent.hubpath import find_hub
|
|
295
|
+
|
|
296
|
+
try:
|
|
297
|
+
hub = find_hub()
|
|
298
|
+
if hub is not None:
|
|
299
|
+
from de_agent.config import load_config
|
|
300
|
+
|
|
301
|
+
src = load_config(hub / "de-agent.yaml").source(source)
|
|
302
|
+
return "redshift" if src.type == "redshift" else "postgres"
|
|
303
|
+
except Exception:
|
|
304
|
+
pass
|
|
305
|
+
return "postgres"
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
@app.command("query")
|
|
309
|
+
def query(
|
|
310
|
+
source: str = typer.Argument(..., help="Source name from de_agent.yaml"),
|
|
311
|
+
sql: str = typer.Argument(..., help="One read-only statement"),
|
|
312
|
+
fmt: str = typer.Option("table", "--format", help="table | json | csv"),
|
|
313
|
+
limit: int = typer.Option(200, "--limit", min=1, help="Max rows returned"),
|
|
314
|
+
) -> None:
|
|
315
|
+
"""Run one read-only statement on a configured source (read-only transaction, timeouts, row cap)."""
|
|
316
|
+
import csv
|
|
317
|
+
|
|
318
|
+
from de_agent.collectors.base import CollectorError
|
|
319
|
+
from de_agent.config import ConfigError
|
|
320
|
+
from de_agent.sqlexec import query_limited
|
|
321
|
+
|
|
322
|
+
if fmt not in ("table", "json", "csv"):
|
|
323
|
+
typer.echo("--format must be table, json or csv", err=True)
|
|
324
|
+
raise typer.Exit(2)
|
|
325
|
+
dialect = _source_dialect(source)
|
|
326
|
+
stmts = classify(sql, dialect)
|
|
327
|
+
writes = [st for st in stmts if st.kind != "read"]
|
|
328
|
+
if writes or not stmts:
|
|
329
|
+
why = f"{writes[0].label} is a write" if writes else "empty statement"
|
|
330
|
+
typer.echo(f"refused: {why}; dea query is read-only (writes go through a plan: dea plan check, dea approve, dea apply)",
|
|
331
|
+
err=True)
|
|
332
|
+
raise typer.Exit(2)
|
|
333
|
+
if len(stmts) != 1:
|
|
334
|
+
typer.echo(f"refused: dea query runs exactly one statement (got {len(stmts)})", err=True)
|
|
335
|
+
raise typer.Exit(2)
|
|
336
|
+
hub = _require_hub()
|
|
337
|
+
cfg = _load(hub)
|
|
338
|
+
try:
|
|
339
|
+
src = cfg.source(source)
|
|
340
|
+
except ConfigError:
|
|
341
|
+
typer.echo(f"unknown source {source!r}; configured: {', '.join(s.name for s in cfg.sources if s.conn or s.data_api) or 'none'}",
|
|
342
|
+
err=True)
|
|
343
|
+
raise typer.Exit(2) from None
|
|
344
|
+
if src.type in ("sql", "cli", "http"):
|
|
345
|
+
typer.echo(f"source {source!r} is a pack source: use `dea pack test {source}` (dea query is for postgres and redshift sources)", err=True)
|
|
346
|
+
raise typer.Exit(2)
|
|
347
|
+
if src.conn is None and src.data_api is None:
|
|
348
|
+
typer.echo(f"source {source!r} is not a SQL source (type {src.type})", err=True)
|
|
349
|
+
raise typer.Exit(2)
|
|
350
|
+
if src.writable:
|
|
351
|
+
typer.echo(f"source {source!r} is a write-capable source for plans; query a read-only source instead", err=True)
|
|
352
|
+
raise typer.Exit(2)
|
|
353
|
+
try:
|
|
354
|
+
if src.type == "redshift":
|
|
355
|
+
from de_agent.sqlexec import query_limited_redshift
|
|
356
|
+
|
|
357
|
+
rows, total = query_limited_redshift(src, sql, float(src.timeout_seconds), limit)
|
|
358
|
+
else:
|
|
359
|
+
rows, total = query_limited(src.conn, sql, src.exec, float(src.timeout_seconds), limit)
|
|
360
|
+
except CollectorError as ex:
|
|
361
|
+
typer.echo(f"query failed: {ex}", err=True)
|
|
362
|
+
raise typer.Exit(1) from None
|
|
363
|
+
except Exception as ex: # never a traceback for the caller (an agent reads this output)
|
|
364
|
+
typer.echo(f"query failed: {type(ex).__name__}: {ex}", err=True)
|
|
365
|
+
raise typer.Exit(1) from None
|
|
366
|
+
more = total - len(rows)
|
|
367
|
+
if fmt == "json":
|
|
368
|
+
typer.echo(json.dumps({"rows": rows, "total": total, "more_rows": max(more, 0)}, default=str, indent=2))
|
|
369
|
+
return
|
|
370
|
+
if fmt == "csv":
|
|
371
|
+
import io
|
|
372
|
+
|
|
373
|
+
buf = io.StringIO()
|
|
374
|
+
if rows:
|
|
375
|
+
w = csv.DictWriter(buf, fieldnames=list(rows[0]))
|
|
376
|
+
w.writeheader()
|
|
377
|
+
for r in rows:
|
|
378
|
+
w.writerow({k: _fmt_cell(v) for k, v in r.items()})
|
|
379
|
+
typer.echo(buf.getvalue().rstrip("\n"))
|
|
380
|
+
else:
|
|
381
|
+
_print_table(rows)
|
|
382
|
+
if more > 0:
|
|
383
|
+
typer.echo(f"({more} more rows not shown; raise --limit)", err=fmt == "csv")
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
plan_app = typer.Typer(help="Plan files: .dea/plans/<id>.md")
|
|
387
|
+
app.add_typer(plan_app, name="plan")
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def _plan_and_cfg(plan_id: str): # noqa: ANN202
|
|
391
|
+
from de_agent.plans import PlanError, load_plan
|
|
392
|
+
|
|
393
|
+
hub = _require_hub()
|
|
394
|
+
cfg = _load(hub)
|
|
395
|
+
try:
|
|
396
|
+
return hub, cfg, load_plan(hub, plan_id)
|
|
397
|
+
except PlanError as ex:
|
|
398
|
+
typer.echo(str(ex), err=True)
|
|
399
|
+
raise typer.Exit(2) from None
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
@plan_app.command("check")
|
|
403
|
+
def plan_check(plan_id: str = typer.Argument(..., help="Plan id (file name without .md)"),
|
|
404
|
+
as_json: bool = typer.Option(False, "--json")) -> None:
|
|
405
|
+
"""Classify every statement in a plan and list errors. Exit 1 on any error."""
|
|
406
|
+
from datetime import datetime, timezone
|
|
407
|
+
|
|
408
|
+
from de_agent.plans import check_plan, render_report, write_checked
|
|
409
|
+
|
|
410
|
+
hub, cfg, plan = _plan_and_cfg(plan_id)
|
|
411
|
+
rep = check_plan(plan, cfg, filename_id=plan_id.removesuffix(".md"), hub=hub)
|
|
412
|
+
if as_json:
|
|
413
|
+
typer.echo(json.dumps({**rep.as_dict(), "hashes": rep.hashes}, indent=2))
|
|
414
|
+
else:
|
|
415
|
+
typer.echo(render_report(plan, rep))
|
|
416
|
+
if rep.ok:
|
|
417
|
+
write_checked(hub, plan, rep, datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"))
|
|
418
|
+
raise typer.Exit(0 if rep.ok else 1)
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
@app.command("approve")
|
|
422
|
+
def approve(plan_id: str = typer.Argument(..., help="Plan id")) -> None:
|
|
423
|
+
"""HUMAN ONLY: review the plan check, type the plan id, sign the statement hashes (needs a TTY)."""
|
|
424
|
+
from datetime import datetime, timezone
|
|
425
|
+
|
|
426
|
+
from de_agent import approval
|
|
427
|
+
from de_agent.plans import check_plan, plan_hashes, plans_dir, render_report
|
|
428
|
+
|
|
429
|
+
if not (sys.stdin.isatty() and sys.stdout.isatty()):
|
|
430
|
+
typer.echo("dea approve is human-only and needs an interactive terminal: run this in your own terminal.", err=True)
|
|
431
|
+
raise typer.Exit(2)
|
|
432
|
+
hub, cfg, plan = _plan_and_cfg(plan_id)
|
|
433
|
+
rep = check_plan(plan, cfg, filename_id=plan_id.removesuffix(".md"), hub=hub)
|
|
434
|
+
typer.echo(render_report(plan, rep))
|
|
435
|
+
if not rep.ok:
|
|
436
|
+
typer.echo("\nrefusing to approve a plan with errors", err=True)
|
|
437
|
+
raise typer.Exit(1)
|
|
438
|
+
typer.echo("\n" + plan.text.split("\n## Rollback")[0][:6000])
|
|
439
|
+
typed = input(f"\nType the plan id ({plan.id}) to approve it, anything else to cancel: ").strip()
|
|
440
|
+
if typed != plan.id:
|
|
441
|
+
typer.echo("cancelled; nothing written", err=True)
|
|
442
|
+
raise typer.Exit(1)
|
|
443
|
+
try:
|
|
444
|
+
who = subprocess.run(["git", "config", "user.email"], capture_output=True, text=True, cwd=hub,
|
|
445
|
+
timeout=10).stdout.strip()
|
|
446
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
447
|
+
who = ""
|
|
448
|
+
who = who or os.environ.get("USER") or os.environ.get("USERNAME") or "unknown"
|
|
449
|
+
key = approval.load_key(create=True)
|
|
450
|
+
data = approval.make_approval(plan.id, plan_hashes(plan), who,
|
|
451
|
+
datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), key)
|
|
452
|
+
out = plans_dir(hub) / f"{plan.id}.approved"
|
|
453
|
+
approval.write_approval(str(out), data)
|
|
454
|
+
typer.echo(f"approved {plan.id} ({len(data['hashes'])} statement hashes) by {who}: {out}")
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
@app.command("apply")
|
|
458
|
+
def apply(
|
|
459
|
+
plan_id: str = typer.Argument(..., help="Plan id"),
|
|
460
|
+
rollback: bool = typer.Option(False, "--rollback", help="Run the Rollback section instead"),
|
|
461
|
+
dry_run: bool = typer.Option(False, "--dry-run", help="Print what would run"),
|
|
462
|
+
) -> None:
|
|
463
|
+
"""Run an approved plan step by step (executor agent only; verifies the HMAC and every statement hash first)."""
|
|
464
|
+
from de_agent import approval
|
|
465
|
+
from de_agent.applyplan import ApplyRefused, apply_plan, authorize, dry_run_lines
|
|
466
|
+
|
|
467
|
+
hub, cfg, plan = _plan_and_cfg(plan_id)
|
|
468
|
+
key = approval.load_key()
|
|
469
|
+
try:
|
|
470
|
+
if dry_run:
|
|
471
|
+
authorize(hub, cfg, plan, rollback, key)
|
|
472
|
+
typer.echo("\n".join(dry_run_lines(plan, rollback)))
|
|
473
|
+
typer.echo("dry run: nothing executed")
|
|
474
|
+
return
|
|
475
|
+
res = apply_plan(hub, cfg, plan, rollback=rollback, key=key)
|
|
476
|
+
except ApplyRefused as ex:
|
|
477
|
+
typer.echo(f"refused: {ex}", err=True)
|
|
478
|
+
raise typer.Exit(2) from None
|
|
479
|
+
for st in res.steps:
|
|
480
|
+
line = f"step {st['step']} {st['status']} rowcount={st['rowcount']} {st['duration_s']}s"
|
|
481
|
+
typer.echo(line + (f" error: {st['error']}" if st["error"] else ""))
|
|
482
|
+
for act in st.get("actions", []):
|
|
483
|
+
if act["kind"] == "diff":
|
|
484
|
+
typer.echo(f" {act['mode']} diff {act['repo']}/{act['file']}: {act.get('diff_stat') or '(no stat)'}".replace("\n", " | "))
|
|
485
|
+
else:
|
|
486
|
+
typer.echo(f" shell exit={act['exit_code']}: {act['command'][:100]}")
|
|
487
|
+
for v in st["verify"]:
|
|
488
|
+
typer.echo(f" verify {'ok' if v['ok'] else 'FAILED'}: value={v['value']!r} expect={v['expect']}")
|
|
489
|
+
typer.echo(f"{res.mode} {res.status}: {res.path}")
|
|
490
|
+
raise typer.Exit(0 if res.status == "ok" else 1)
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
@app.command("plans")
|
|
494
|
+
def plans_cmd() -> None:
|
|
495
|
+
"""List plans with their state: draft / checked / approved / applied / failed / rolled back."""
|
|
496
|
+
from de_agent import approval
|
|
497
|
+
from de_agent.plans import load_plan, plan_state, plans_dir
|
|
498
|
+
|
|
499
|
+
hub = _require_hub()
|
|
500
|
+
key = approval.load_key()
|
|
501
|
+
paths = sorted(plans_dir(hub).glob("*.md")) if plans_dir(hub).is_dir() else []
|
|
502
|
+
if not paths:
|
|
503
|
+
typer.echo("no plans (.dea/plans/ is empty)")
|
|
504
|
+
return
|
|
505
|
+
for p in paths:
|
|
506
|
+
try:
|
|
507
|
+
pl = load_plan(hub, p.stem)
|
|
508
|
+
meta = f"{pl.risk or '?':6} {pl.title}"
|
|
509
|
+
except Exception:
|
|
510
|
+
meta = "(unreadable)"
|
|
511
|
+
typer.echo(f"{p.stem} {plan_state(hub, p.stem, key):12} {meta}")
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def _hook_python() -> str:
|
|
515
|
+
"""The python3 the hooks will run: DEA_HOOK_PYTHON, else the first `python3` on PATH once this venv's bin dir is
|
|
516
|
+
removed (a pipx/uv tool venv is not what Claude Code's hook command resolves)."""
|
|
517
|
+
env = os.environ.get("DEA_HOOK_PYTHON")
|
|
518
|
+
if env:
|
|
519
|
+
return env
|
|
520
|
+
skip = {os.path.realpath(os.path.dirname(sys.executable))} if sys.prefix != getattr(sys, "base_prefix", sys.prefix) else set()
|
|
521
|
+
if os.environ.get("VIRTUAL_ENV"):
|
|
522
|
+
skip.add(os.path.realpath(os.path.join(os.environ["VIRTUAL_ENV"], "bin")))
|
|
523
|
+
path = os.pathsep.join(d for d in os.environ.get("PATH", "").split(os.pathsep) if d and os.path.realpath(d) not in skip)
|
|
524
|
+
return shutil.which("python3", path=path) or "python3"
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _plugin_root() -> Path | None:
|
|
528
|
+
env = os.environ.get("CLAUDE_PLUGIN_ROOT")
|
|
529
|
+
cands = [Path(env)] if env else []
|
|
530
|
+
cands.append(Path(__file__).resolve().parents[3])
|
|
531
|
+
return next((c for c in cands if (c / "scripts" / "_vendor").is_dir()), None)
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
@app.command("doctor")
|
|
535
|
+
def doctor() -> None:
|
|
536
|
+
"""Check the environment: python, hub, config, sources, psql, FTS5, hook python + vendored sqlglot."""
|
|
537
|
+
from de_agent.collectors import get_collector
|
|
538
|
+
from de_agent.collectors.base import RunContext
|
|
539
|
+
from de_agent.config import ConfigError, load_config
|
|
540
|
+
from de_agent.find import fts5_available
|
|
541
|
+
from de_agent.hubpath import find_hub
|
|
542
|
+
|
|
543
|
+
rows: list[tuple[str, str, str]] = [] # (mark, name, detail)
|
|
544
|
+
|
|
545
|
+
def add(ok: bool | None, name: str, detail: str) -> None:
|
|
546
|
+
rows.append(("✓" if ok else ("✗" if ok is False else "-"), name, detail))
|
|
547
|
+
|
|
548
|
+
v = sys.version_info
|
|
549
|
+
add(v >= (3, 10), "python", f"{v.major}.{v.minor}.{v.micro} (CLI needs >= 3.10)")
|
|
550
|
+
hub = find_hub()
|
|
551
|
+
add(hub is not None, "hub", str(hub) if hub else "not found (dea init, or set DEA_HUB)")
|
|
552
|
+
cfg = None
|
|
553
|
+
if hub:
|
|
554
|
+
try:
|
|
555
|
+
cfg = load_config(hub / "de-agent.yaml")
|
|
556
|
+
add(True, "config", f"valid, {len(cfg.sources)} source(s)")
|
|
557
|
+
except ConfigError as ex:
|
|
558
|
+
add(False, "config", str(ex))
|
|
559
|
+
for s in cfg.sources if cfg else []:
|
|
560
|
+
try:
|
|
561
|
+
detail = get_collector(s.type).ping(s, RunContext(timeout_seconds=15))
|
|
562
|
+
add(True, f"source {s.name}", f"{s.type} reachable via {s.exec} ({detail})")
|
|
563
|
+
except Exception as ex:
|
|
564
|
+
add(False, f"source {s.name}", str(ex).splitlines()[-1][:120] if str(ex) else type(ex).__name__)
|
|
565
|
+
if hub:
|
|
566
|
+
from de_agent.packs.cli import doctor_rows
|
|
567
|
+
|
|
568
|
+
for ok, name, detail in doctor_rows():
|
|
569
|
+
add(ok, name, detail)
|
|
570
|
+
psql = shutil.which("psql")
|
|
571
|
+
add(psql is not None, "psql", psql or "not on PATH (needed for ssh sources and the no-psycopg fallback)")
|
|
572
|
+
add(fts5_available(), "fts5", "SQLite FTS5 available" if fts5_available() else "missing: dea find falls back to a slow scan")
|
|
573
|
+
py = _hook_python()
|
|
574
|
+
try:
|
|
575
|
+
out = subprocess.run([py, "-c", "import sys;print('%d.%d' % sys.version_info[:2])"],
|
|
576
|
+
capture_output=True, text=True, timeout=10).stdout.strip()
|
|
577
|
+
major, minor = (int(x) for x in out.split("."))
|
|
578
|
+
add((major, minor) >= (3, 9), "hook python", f"{py} is {out} (the python3 the hooks run; needs >= 3.9)")
|
|
579
|
+
except Exception as ex:
|
|
580
|
+
add(False, "hook python", f"cannot run {py}: {ex}")
|
|
581
|
+
py = ""
|
|
582
|
+
root = _plugin_root()
|
|
583
|
+
if root and py:
|
|
584
|
+
vend = root / "scripts" / "_vendor"
|
|
585
|
+
r = subprocess.run([py, "-c", "import sys;sys.path.insert(0, sys.argv[1]);import sqlglot;print(sqlglot.__version__)", str(vend)],
|
|
586
|
+
capture_output=True, text=True, timeout=30)
|
|
587
|
+
add(r.returncode == 0, "vendored sqlglot", f"v{r.stdout.strip()} under {py}" if r.returncode == 0 else r.stderr.strip()[-100:])
|
|
588
|
+
else:
|
|
589
|
+
add(None, "vendored sqlglot", "not checked: set CLAUDE_PLUGIN_ROOT=<your clone of the plugin> to check it")
|
|
590
|
+
width = max(len(n) for _, n, _ in rows)
|
|
591
|
+
for mark, name, detail in rows:
|
|
592
|
+
typer.echo(f"{mark} {name.ljust(width)} {detail}")
|
|
593
|
+
raise typer.Exit(1 if any(m == "✗" for m, _, _ in rows) else 0)
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
config_app = typer.Typer(help="de-agent.yaml tooling.", no_args_is_help=True)
|
|
597
|
+
app.add_typer(config_app, name="config")
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
@config_app.command("schema")
|
|
601
|
+
def config_schema(
|
|
602
|
+
markdown: bool = typer.Option(False, "--markdown", help="Markdown reference (docs/configuration.md) instead of JSON Schema"),
|
|
603
|
+
out: Path | None = typer.Option(None, "--out", "-o", help="Write to this file instead of stdout"),
|
|
604
|
+
) -> None:
|
|
605
|
+
"""Print the de-agent.yaml JSON Schema (default) or its markdown reference, generated from the config models."""
|
|
606
|
+
from de_agent.configdoc import json_text
|
|
607
|
+
from de_agent.configdoc import markdown as md_text
|
|
608
|
+
|
|
609
|
+
text = md_text() if markdown else json_text()
|
|
610
|
+
if out:
|
|
611
|
+
out.write_text(text if text.endswith("\n") else text + "\n", encoding="utf-8")
|
|
612
|
+
typer.echo(f"wrote {out}")
|
|
613
|
+
else:
|
|
614
|
+
typer.echo(text.rstrip("\n"))
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
@app.command("guard-test")
|
|
618
|
+
def guard_test(
|
|
619
|
+
corpus: Path | None = typer.Option(None, help="YAML corpus of SQL classification cases (default: the one shipped in the package)"),
|
|
620
|
+
) -> None:
|
|
621
|
+
"""Run the SQL classifier corpus (the packaged one unless --corpus is given)."""
|
|
622
|
+
import yaml
|
|
623
|
+
|
|
624
|
+
if corpus is None:
|
|
625
|
+
corpus = Path(__file__).resolve().parent / "data" / "sql_cases.yaml"
|
|
626
|
+
cases = yaml.safe_load(corpus.read_text())
|
|
627
|
+
failed = 0
|
|
628
|
+
for c in cases:
|
|
629
|
+
got = classify(c["sql"], c.get("dialect", "postgres"))
|
|
630
|
+
kinds = [s.kind for s in got]
|
|
631
|
+
want = c["kind"] if isinstance(c["kind"], list) else [c["kind"]]
|
|
632
|
+
if kinds != want:
|
|
633
|
+
failed += 1
|
|
634
|
+
typer.echo(f"FAIL {json.dumps(c['sql'])[:80]}: want {want} got {kinds}")
|
|
635
|
+
typer.echo(f"{len(cases) - failed}/{len(cases)} passed")
|
|
636
|
+
raise typer.Exit(1 if failed else 0)
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def _register_packs() -> None:
|
|
640
|
+
from de_agent.packs.cli import pack_app
|
|
641
|
+
|
|
642
|
+
app.add_typer(pack_app, name="pack")
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
_register_packs()
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
@app.callback()
|
|
649
|
+
def main() -> None:
|
|
650
|
+
"""de-agent CLI."""
|