db-agent-kit 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- db_agent/__init__.py +3 -0
- db_agent/cli.py +32 -0
- db_agent/engines/__init__.py +36 -0
- db_agent/engines/databricks/__init__.py +1 -0
- db_agent/engines/databricks/checks.py +221 -0
- db_agent/engines/databricks/cli.py +61 -0
- db_agent/engines/databricks/client.py +86 -0
- db_agent/engines/databricks/cost.py +96 -0
- db_agent/engines/databricks/health.py +156 -0
- db_agent/engines/databricks/model.py +29 -0
- db_agent/engines/databricks/queries.py +98 -0
- db_agent/engines/databricks/render.py +78 -0
- db_agent/engines/databricks/runtimes.py +58 -0
- db_agent/engines/databricks/sqlguard.py +80 -0
- db_agent/engines/sqlserver/__init__.py +11 -0
- db_agent/engines/sqlserver/classify.py +102 -0
- db_agent/engines/sqlserver/cli.py +265 -0
- db_agent/engines/sqlserver/data/classification.yaml +29 -0
- db_agent/engines/sqlserver/db.py +183 -0
- db_agent/engines/sqlserver/guard.py +190 -0
- db_agent/engines/sqlserver/inventory.py +324 -0
- db_agent/engines/sqlserver/outputs.py +299 -0
- db_agent/engines/sqlserver/rdl.py +269 -0
- db_agent/engines/sqlserver/report.py +120 -0
- db_agent/engines/sqlserver/risks.py +184 -0
- db_agent/engines/sqlserver/rules.py +141 -0
- db_agent/engines/sqlserver/sqlrefs.py +103 -0
- db_agent/engines/sqlserver/ssrs.py +230 -0
- db_agent/engines/sqlserver/templates/summary.html.j2 +142 -0
- db_agent/engines/sqlserver/usage.py +127 -0
- db_agent/engines/sqlserver/validate.py +238 -0
- db_agent_kit-0.2.0.dist-info/METADATA +192 -0
- db_agent_kit-0.2.0.dist-info/RECORD +36 -0
- db_agent_kit-0.2.0.dist-info/WHEEL +4 -0
- db_agent_kit-0.2.0.dist-info/entry_points.txt +2 -0
- db_agent_kit-0.2.0.dist-info/licenses/LICENSE +201 -0
db_agent/__init__.py
ADDED
db_agent/cli.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""db-agent <engine> <verb>. Each available engine is a sub-command group; planned engines are only listed."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import typer
|
|
6
|
+
|
|
7
|
+
from . import __version__
|
|
8
|
+
from .engines import ENGINES
|
|
9
|
+
|
|
10
|
+
app = typer.Typer(add_completion=False, no_args_is_help=True,
|
|
11
|
+
help="db-agent: read-only DBA toolkit. Run `db-agent engines` to see what is supported.")
|
|
12
|
+
|
|
13
|
+
for _e in ENGINES:
|
|
14
|
+
if _e.status == "available":
|
|
15
|
+
app.add_typer(_e.load(), name=_e.name, help=_e.summary)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@app.command()
|
|
19
|
+
def engines() -> None:
|
|
20
|
+
"""List database engines and whether they are implemented."""
|
|
21
|
+
for e in ENGINES:
|
|
22
|
+
typer.echo(f"{e.name:<12} {e.status:<10} {e.title}: {e.summary}")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@app.command()
|
|
26
|
+
def version() -> None:
|
|
27
|
+
"""Print the db-agent version."""
|
|
28
|
+
typer.echo(__version__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
if __name__ == "__main__":
|
|
32
|
+
app()
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Engine registry. One entry per database engine the CLI knows about.
|
|
2
|
+
|
|
3
|
+
To add an engine, create `db_agent/engines/<name>/` with a `cli.py` that exposes a Typer `app`, then add
|
|
4
|
+
an `Engine(...)` below with status "available". See docs/adding-an-engine.md.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import importlib
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
import typer
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Engine:
|
|
17
|
+
name: str # the CLI verb: db-agent <name> ...
|
|
18
|
+
title: str
|
|
19
|
+
summary: str
|
|
20
|
+
status: str # "available" or "planned"
|
|
21
|
+
module: str | None = None # dotted path to a module exposing a Typer `app` (available engines only)
|
|
22
|
+
|
|
23
|
+
def load(self) -> typer.Typer:
|
|
24
|
+
if self.status != "available" or not self.module:
|
|
25
|
+
raise LookupError(f"engine {self.name!r} is {self.status}, not implemented")
|
|
26
|
+
return importlib.import_module(self.module).app
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
ENGINES: tuple[Engine, ...] = (
|
|
30
|
+
Engine("sqlserver", "SQL Server and SSRS", "Inventory, migration risk, SSRS report triage and validation.",
|
|
31
|
+
"available", "db_agent.engines.sqlserver.cli"),
|
|
32
|
+
Engine("postgres", "PostgreSQL", "Not implemented yet.", "planned"),
|
|
33
|
+
Engine("mysql", "MySQL", "Not implemented yet.", "planned"),
|
|
34
|
+
Engine("databricks", "Databricks", "Workspace health: runtimes, legacy dependencies, DBU spend, spend waste.",
|
|
35
|
+
"available", "db_agent.engines.databricks.cli"),
|
|
36
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""db-agent Databricks engine: read-only workspace checks (system tables plus the REST API)."""
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Pure checks: plain dicts in, Findings out. No network, no SDK."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections import Counter
|
|
6
|
+
from datetime import date
|
|
7
|
+
from typing import Iterable
|
|
8
|
+
|
|
9
|
+
from . import runtimes
|
|
10
|
+
from .model import Finding
|
|
11
|
+
|
|
12
|
+
ALL_PURPOSE_SOURCES = {"UI", "API", None, ""}
|
|
13
|
+
NO_ISOLATION = {"NONE", "LEGACY_TABLE_ACL", "LEGACY_PASSTHROUGH", "LEGACY_SINGLE_USER", "LEGACY_SINGLE_USER_STANDARD"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _num(v) -> float:
|
|
17
|
+
try:
|
|
18
|
+
return float(v)
|
|
19
|
+
except (TypeError, ValueError):
|
|
20
|
+
return 0.0
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def is_all_purpose(c: dict) -> bool:
|
|
24
|
+
return c.get("cluster_source") in ALL_PURPOSE_SOURCES
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _job_specs(jobs: Iterable[dict]) -> Iterable[dict]:
|
|
28
|
+
"""Every cluster definition inside job settings: {'job_id','job','where','spec'}."""
|
|
29
|
+
for j in jobs:
|
|
30
|
+
s = j.get("settings") or {}
|
|
31
|
+
base = {"job_id": j.get("job_id"), "job": s.get("name")}
|
|
32
|
+
for jc in s.get("job_clusters") or []:
|
|
33
|
+
yield {**base, "where": f"job cluster {jc.get('job_cluster_key')}", "spec": jc.get("new_cluster") or {}}
|
|
34
|
+
for t in s.get("tasks") or []:
|
|
35
|
+
if t.get("new_cluster"):
|
|
36
|
+
yield {**base, "where": f"task {t.get('task_key')}", "spec": t["new_cluster"]}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _all_specs(clusters: list[dict], jobs: list[dict]) -> Iterable[dict]:
|
|
40
|
+
for c in clusters:
|
|
41
|
+
if is_all_purpose(c):
|
|
42
|
+
yield {"job_id": None, "job": None, "where": f"cluster {c.get('cluster_name')} ({c.get('cluster_id')})",
|
|
43
|
+
"spec": c, "owner": c.get("creator_user_name")}
|
|
44
|
+
yield from _job_specs(jobs)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# ---- runtimes -------------------------------------------------------------------------------------
|
|
48
|
+
def runtime_findings(clusters, jobs, today: date, warn_days: int) -> list[Finding]:
|
|
49
|
+
rows, versions = [], Counter()
|
|
50
|
+
for s in _all_specs(clusters, jobs):
|
|
51
|
+
st = runtimes.classify(s["spec"].get("spark_version"), today, warn_days)
|
|
52
|
+
versions[st.label] += 1
|
|
53
|
+
if st.state in ("ended", "ending_soon"):
|
|
54
|
+
rows.append({"where": ("job " + str(s["job"]) + ": " if s["job"] else "") + s["where"],
|
|
55
|
+
"runtime": st.label, "state": st.state,
|
|
56
|
+
"end_of_support": st.eos.isoformat() if st.eos else "ended (before 10.5)",
|
|
57
|
+
"days_left": st.days_left})
|
|
58
|
+
out = []
|
|
59
|
+
ended = [r for r in rows if r["state"] == "ended"]
|
|
60
|
+
soon = [r for r in rows if r["state"] == "ending_soon"]
|
|
61
|
+
if ended:
|
|
62
|
+
out.append(Finding("runtime.ended", "high", f"{len(ended)} cluster definitions on runtimes past end of support",
|
|
63
|
+
"Databricks no longer supports these runtimes. Existing jobs may keep running, but they get no fixes.",
|
|
64
|
+
ended, "Move to a supported LTS runtime (check the support table in the Databricks docs)."))
|
|
65
|
+
if soon:
|
|
66
|
+
out.append(Finding("runtime.ending_soon", "medium", f"{len(soon)} cluster definitions on runtimes that end support within {warn_days} days",
|
|
67
|
+
"Plan the upgrade before the end-of-support date.", soon, "Test the workload on the next LTS runtime."))
|
|
68
|
+
unknown = sum(n for lbl, n in versions.items() if lbl.startswith("custom") or lbl == "unknown")
|
|
69
|
+
detail = ", ".join(f"{k}: {v}" for k, v in sorted(versions.items())) or "none found"
|
|
70
|
+
if unknown:
|
|
71
|
+
detail += f". {unknown} could not be classified (custom image or version newer than this tool's table)."
|
|
72
|
+
out.append(Finding("runtime.summary", "info", "Runtime versions in use", detail, [], ""))
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# ---- legacy dependencies ----------------------------------------------------------------------------
|
|
77
|
+
def init_script_findings(clusters, jobs) -> list[Finding]:
|
|
78
|
+
rows = []
|
|
79
|
+
for s in _all_specs(clusters, jobs):
|
|
80
|
+
for sc in s["spec"].get("init_scripts") or []:
|
|
81
|
+
dest = (sc.get("dbfs") or {}).get("destination")
|
|
82
|
+
if dest:
|
|
83
|
+
rows.append({"where": ("job " + str(s["job"]) + ": " if s["job"] else "") + s["where"], "script": dest})
|
|
84
|
+
if not rows:
|
|
85
|
+
return []
|
|
86
|
+
return [Finding("legacy.dbfs_init_scripts", "high", f"{len(rows)} init script references on DBFS",
|
|
87
|
+
"DBFS root is local to one workspace, so these scripts do not exist in a new workspace.",
|
|
88
|
+
rows, "Move the scripts to workspace files or a Unity Catalog volume and update the cluster definitions.")]
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def isolation_findings(clusters, jobs) -> list[Finding]:
|
|
92
|
+
rows, unset = [], 0
|
|
93
|
+
for s in _all_specs(clusters, jobs):
|
|
94
|
+
mode = s["spec"].get("data_security_mode")
|
|
95
|
+
if mode in NO_ISOLATION:
|
|
96
|
+
rows.append({"where": ("job " + str(s["job"]) + ": " if s["job"] else "") + s["where"], "data_security_mode": mode})
|
|
97
|
+
elif mode is None:
|
|
98
|
+
unset += 1
|
|
99
|
+
out = []
|
|
100
|
+
if rows:
|
|
101
|
+
out.append(Finding("legacy.no_isolation", "medium", f"{len(rows)} cluster definitions without Unity Catalog access mode",
|
|
102
|
+
"No-isolation and legacy access modes cannot use Unity Catalog governance.", rows,
|
|
103
|
+
"Switch to standard (shared) or dedicated (single user) access mode."))
|
|
104
|
+
if unset:
|
|
105
|
+
out.append(Finding("legacy.access_mode_unset", "info", f"{unset} cluster definitions do not set an access mode",
|
|
106
|
+
"The effective mode then comes from workspace or policy defaults. Check them in the UI.", [], ""))
|
|
107
|
+
return out
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def hms_findings(schemas: dict[str, int] | None) -> list[Finding]:
|
|
111
|
+
if not schemas:
|
|
112
|
+
return []
|
|
113
|
+
total = sum(schemas.values())
|
|
114
|
+
if total == 0:
|
|
115
|
+
return []
|
|
116
|
+
rows = [{"schema": k, "tables": v} for k, v in sorted(schemas.items(), key=lambda kv: -kv[1]) if v][:25]
|
|
117
|
+
return [Finding("legacy.hive_metastore", "medium", f"{total} tables in the Hive metastore across {len(rows)} schemas",
|
|
118
|
+
"Hive metastore tables belong to one workspace and are not governed by Unity Catalog. A new workspace will not see them.",
|
|
119
|
+
rows, "Upgrade them to Unity Catalog (SYNC for external tables, DEEP CLONE or CTAS for managed ones).")]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def mount_findings(events: list[dict] | None, lookback_days: int) -> list[Finding]:
|
|
123
|
+
if not events:
|
|
124
|
+
return []
|
|
125
|
+
return [Finding("legacy.dbfs_mounts", "medium", f"{len(events)} DBFS mount events in the last {lookback_days} days",
|
|
126
|
+
"Audit logs show mount creation only. They do not list the mounts that exist today; run dbutils.fs.mounts() "
|
|
127
|
+
"in a notebook for that. Mounts are workspace-local and deprecated in favour of Unity Catalog external locations.",
|
|
128
|
+
[{"date": str(e.get("event_date")), "user": e.get("user_email"), "params": e.get("params")} for e in events[:25]],
|
|
129
|
+
"Replace /mnt paths with external locations or volumes.")]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# ---- waste -------------------------------------------------------------------------------------------
|
|
133
|
+
def _scheduled(settings: dict) -> bool:
|
|
134
|
+
for key in ("schedule", "trigger", "continuous"):
|
|
135
|
+
v = settings.get(key)
|
|
136
|
+
if v and v.get("pause_status") != "PAUSED":
|
|
137
|
+
return True
|
|
138
|
+
return False
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def jobs_on_all_purpose(jobs, clusters, spend: dict[str, dict] | None, days: int) -> list[Finding]:
|
|
142
|
+
names = {c.get("cluster_id"): c for c in clusters}
|
|
143
|
+
rows = []
|
|
144
|
+
for j in jobs:
|
|
145
|
+
s = j.get("settings") or {}
|
|
146
|
+
if not _scheduled(s):
|
|
147
|
+
continue
|
|
148
|
+
ids = {t.get("existing_cluster_id") for t in s.get("tasks") or [] if t.get("existing_cluster_id")}
|
|
149
|
+
if s.get("existing_cluster_id"):
|
|
150
|
+
ids.add(s["existing_cluster_id"])
|
|
151
|
+
for cid in sorted(ids):
|
|
152
|
+
sp = (spend or {}).get(cid)
|
|
153
|
+
rows.append({"job": s.get("name"), "job_id": j.get("job_id"), "cluster": (names.get(cid) or {}).get("cluster_name", cid),
|
|
154
|
+
"cluster_id": cid, f"cluster_spend_{days}d": round(_num(sp["cost"]), 2) if sp and sp.get("cost") is not None else None})
|
|
155
|
+
if not rows:
|
|
156
|
+
return []
|
|
157
|
+
rows.sort(key=lambda r: -(r[f"cluster_spend_{days}d"] or 0))
|
|
158
|
+
return [Finding("waste.jobs_on_all_purpose", "high", f"{len(rows)} scheduled jobs run on all-purpose clusters",
|
|
159
|
+
"All-purpose compute costs more per DBU than job compute. Spend shown is the whole cluster's list-price spend, "
|
|
160
|
+
"which includes anything else that ran on it (billing does not tag all-purpose usage with the job).",
|
|
161
|
+
rows, "Give each job its own job cluster, or use serverless jobs.")]
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def no_autotermination(clusters, spend: dict[str, dict] | None, days: int) -> list[Finding]:
|
|
165
|
+
rows = []
|
|
166
|
+
for c in clusters:
|
|
167
|
+
if not is_all_purpose(c) or c.get("autotermination_minutes") != 0: # only an explicit 0 means disabled
|
|
168
|
+
continue
|
|
169
|
+
sp = (spend or {}).get(c.get("cluster_id"))
|
|
170
|
+
rows.append({"cluster": c.get("cluster_name"), "cluster_id": c.get("cluster_id"), "state": c.get("state"),
|
|
171
|
+
"owner": c.get("creator_user_name"), f"spend_{days}d": round(_num(sp["cost"]), 2) if sp and sp.get("cost") is not None else None})
|
|
172
|
+
if not rows:
|
|
173
|
+
return []
|
|
174
|
+
running = sum(1 for r in rows if r["state"] == "RUNNING")
|
|
175
|
+
rows.sort(key=lambda r: (r["state"] != "RUNNING", -(r[f"spend_{days}d"] or 0)))
|
|
176
|
+
return [Finding("waste.no_autotermination", "high" if running else "medium",
|
|
177
|
+
f"{len(rows)} all-purpose clusters never auto-terminate ({running} running now)",
|
|
178
|
+
"A cluster with auto-termination off bills until someone stops it.", rows,
|
|
179
|
+
"Set auto-termination (for example 30 to 60 minutes) or enforce it with a cluster policy.")]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def warehouse_findings(warehouses, spend: dict[str, dict] | None, queries: dict[str, dict] | None, days: int) -> list[Finding]:
|
|
183
|
+
billed_idle, never_stop = [], []
|
|
184
|
+
for w in warehouses:
|
|
185
|
+
wid = w.get("id")
|
|
186
|
+
sp = (spend or {}).get(wid)
|
|
187
|
+
q = (queries or {}).get(wid)
|
|
188
|
+
cost = _num(sp["cost"]) if sp and sp.get("cost") is not None else None
|
|
189
|
+
nq = int(q["queries"]) if q else (0 if queries is not None else None)
|
|
190
|
+
row = {"warehouse": w.get("name"), "id": wid, "state": w.get("state"), "size": w.get("cluster_size"),
|
|
191
|
+
"auto_stop_mins": w.get("auto_stop_mins"), f"spend_{days}d": round(cost, 2) if cost is not None else None,
|
|
192
|
+
f"queries_{days}d": nq, "last_query": q.get("last_query") if q else None}
|
|
193
|
+
if nq == 0 and sp and _num(sp.get("dbus")) > 0:
|
|
194
|
+
billed_idle.append(row)
|
|
195
|
+
if w.get("auto_stop_mins") == 0:
|
|
196
|
+
never_stop.append(row)
|
|
197
|
+
out = []
|
|
198
|
+
if billed_idle:
|
|
199
|
+
out.append(Finding("waste.idle_warehouses", "high", f"{len(billed_idle)} SQL warehouses billed DBUs but ran no queries in {days} days",
|
|
200
|
+
"These warehouses were started and kept running without serving any query.", billed_idle,
|
|
201
|
+
"Stop or delete them, and lower auto-stop."))
|
|
202
|
+
if never_stop:
|
|
203
|
+
out.append(Finding("waste.warehouse_no_autostop", "medium", f"{len(never_stop)} SQL warehouses with auto-stop disabled",
|
|
204
|
+
"They run until someone stops them.", never_stop, "Set auto-stop to 10 minutes or less."))
|
|
205
|
+
return out
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def untagged_findings(rows: list[dict] | None, days: int) -> list[Finding]:
|
|
209
|
+
if not rows:
|
|
210
|
+
return []
|
|
211
|
+
total = sum(_num(r.get("cost")) for r in rows)
|
|
212
|
+
untagged = sum(_num(r.get("untagged_cost")) for r in rows)
|
|
213
|
+
if total <= 0:
|
|
214
|
+
return []
|
|
215
|
+
share = untagged / total * 100
|
|
216
|
+
items = [{"product": r["product"], "untagged_cost": round(_num(r.get("untagged_cost")), 2),
|
|
217
|
+
"total_cost": round(_num(r.get("cost")), 2)} for r in rows if _num(r.get("untagged_cost")) > 0][:15]
|
|
218
|
+
sev = "high" if share >= 50 else "medium" if share >= 20 else "low"
|
|
219
|
+
return [Finding("waste.untagged_spend", sev, f"{share:.0f}% of spend has no tags (${untagged:,.0f} of ${total:,.0f} in {days} days)",
|
|
220
|
+
"Spend without tags cannot be charged back to a team or project. List price, before discounts.",
|
|
221
|
+
items, "Add custom tags on clusters, jobs and warehouses, and enforce them with a cluster policy.")]
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""db-agent databricks <verb>."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from datetime import date
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated, Optional
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
|
|
11
|
+
from . import cost as cost_mod, health as health_mod, render
|
|
12
|
+
|
|
13
|
+
app = typer.Typer(add_completion=False, no_args_is_help=True, help="Read-only checks of a Databricks workspace.")
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@app.callback()
|
|
17
|
+
def _root() -> None:
|
|
18
|
+
"""Databricks engine."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@app.command()
|
|
22
|
+
def health(
|
|
23
|
+
profile: Annotated[Optional[str], typer.Option("--profile", help="Profile in ~/.databrickscfg. Without it, DATABRICKS_HOST/DATABRICKS_TOKEN or the default profile is used. Tokens are never taken on the command line.")] = None,
|
|
24
|
+
host: Annotated[Optional[str], typer.Option("--host", help="Workspace URL.")] = None,
|
|
25
|
+
warehouse_id: Annotated[Optional[str], typer.Option("--warehouse-id", help="SQL warehouse for system table queries. Default: a warehouse that is already running. A stopped warehouse starts (and bills) when it is used.")] = None,
|
|
26
|
+
no_sql: Annotated[bool, typer.Option("--no-sql", help="REST API checks only. Skips every system table query.")] = False,
|
|
27
|
+
lookback_days: Annotated[int, typer.Option("--lookback-days", min=1, max=365, help="Window for waste checks.")] = 30,
|
|
28
|
+
cost_from: Annotated[Optional[str], typer.Option("--cost-from", help="First month for the cost comparison, YYYY-MM. Default: two months before the current one.")] = None,
|
|
29
|
+
cost_to: Annotated[Optional[str], typer.Option("--cost-to", help="Last month, YYYY-MM. Default: the current month.")] = None,
|
|
30
|
+
eos_warning_days: Annotated[int, typer.Option("--eos-warning-days", help="Flag runtimes that end support within this many days.")] = 180,
|
|
31
|
+
all_workspaces: Annotated[bool, typer.Option("--all-workspaces", help="Include every workspace in the account in billing queries.")] = False,
|
|
32
|
+
out: Annotated[Path, typer.Option("--out", help="Output folder.")] = Path("db-agent-databricks-output"),
|
|
33
|
+
format: Annotated[str, typer.Option("--format", help="Comma list of: md, json.")] = "md,json",
|
|
34
|
+
) -> None:
|
|
35
|
+
"""Workspace health: runtimes, legacy dependencies, DBU spend over time, spend waste."""
|
|
36
|
+
formats = {f.strip().lower() for f in format.split(",") if f.strip()}
|
|
37
|
+
if formats - {"md", "json"}:
|
|
38
|
+
raise typer.BadParameter("--format accepts md and json")
|
|
39
|
+
if bool(cost_from) != bool(cost_to):
|
|
40
|
+
raise typer.BadParameter("give both --cost-from and --cost-to, or neither")
|
|
41
|
+
if cost_from:
|
|
42
|
+
try:
|
|
43
|
+
cost_mod.month_list(cost_from, cost_to)
|
|
44
|
+
except (ValueError, TypeError) as e:
|
|
45
|
+
raise typer.BadParameter(f"--cost-from/--cost-to: {e}")
|
|
46
|
+
from .client import DatabricksError, connect # imports the SDK only when the command runs
|
|
47
|
+
try:
|
|
48
|
+
reader = connect(profile, host, warehouse_id, use_sql=not no_sql)
|
|
49
|
+
except Exception as e: # noqa: BLE001
|
|
50
|
+
typer.echo(f"Could not connect to Databricks: {e}\nCheck `databricks auth login` or DATABRICKS_HOST/DATABRICKS_TOKEN.", err=True)
|
|
51
|
+
raise typer.Exit(2)
|
|
52
|
+
if not no_sql and not reader.has_sql:
|
|
53
|
+
typer.echo("No running SQL warehouse found. Pass --warehouse-id (it will start if stopped) or --no-sql.", err=True)
|
|
54
|
+
try:
|
|
55
|
+
rep = health_mod.run(reader, health_mod.Options(date.today(), lookback_days, cost_from, cost_to, eos_warning_days, all_workspaces))
|
|
56
|
+
except DatabricksError as e:
|
|
57
|
+
typer.echo(str(e), err=True)
|
|
58
|
+
raise typer.Exit(2)
|
|
59
|
+
typer.echo(render.console(rep))
|
|
60
|
+
for p in render.write(rep, out, formats):
|
|
61
|
+
typer.echo(f"wrote {p}")
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""The only module that imports the Databricks SDK. Read-only by construction:
|
|
2
|
+
|
|
3
|
+
* REST: list/get calls only (clusters, jobs, warehouses, workspace id).
|
|
4
|
+
* SQL: statements go through sqlguard.check() and are SELECT/WITH or SHOW ... IN hive_metastore only.
|
|
5
|
+
tests/databricks/test_architecture.py enforces both rules.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import time
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from . import sqlguard
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DatabricksError(Exception):
|
|
17
|
+
pass
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Reader:
|
|
21
|
+
def __init__(self, ws: Any, warehouse_id: str | None, sql_timeout_s: int = 300, sql_skipped: bool = False):
|
|
22
|
+
self._ws = ws
|
|
23
|
+
self.sql_skipped = sql_skipped
|
|
24
|
+
self.warehouse_id = warehouse_id
|
|
25
|
+
self._timeout = sql_timeout_s
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def has_sql(self) -> bool:
|
|
29
|
+
return bool(self.warehouse_id)
|
|
30
|
+
|
|
31
|
+
# ---- REST -------------------------------------------------------------------------------------
|
|
32
|
+
def workspace_id(self) -> str:
|
|
33
|
+
return str(self._ws.get_workspace_id())
|
|
34
|
+
|
|
35
|
+
def host(self) -> str:
|
|
36
|
+
return str(self._ws.config.host)
|
|
37
|
+
|
|
38
|
+
def clusters(self) -> list[dict]:
|
|
39
|
+
return [c.as_dict() for c in self._ws.clusters.list()]
|
|
40
|
+
|
|
41
|
+
def jobs(self) -> list[dict]:
|
|
42
|
+
return [j.as_dict() for j in self._ws.jobs.list(expand_tasks=True)]
|
|
43
|
+
|
|
44
|
+
def warehouses(self) -> list[dict]:
|
|
45
|
+
return [w.as_dict() for w in self._ws.warehouses.list()]
|
|
46
|
+
|
|
47
|
+
# ---- SQL --------------------------------------------------------------------------------------
|
|
48
|
+
def sql(self, statement: str) -> list[dict]:
|
|
49
|
+
if not self.warehouse_id:
|
|
50
|
+
raise DatabricksError("no SQL warehouse selected")
|
|
51
|
+
sqlguard.check(statement)
|
|
52
|
+
r = self._ws.statement_execution.execute_statement(
|
|
53
|
+
statement=statement, warehouse_id=self.warehouse_id, wait_timeout="30s", row_limit=10000)
|
|
54
|
+
deadline = time.monotonic() + self._timeout
|
|
55
|
+
while r.status and r.status.state.value in ("PENDING", "RUNNING"):
|
|
56
|
+
if time.monotonic() > deadline:
|
|
57
|
+
self._ws.statement_execution.cancel_execution(r.statement_id)
|
|
58
|
+
raise DatabricksError(f"query still running after {self._timeout}s, cancelled")
|
|
59
|
+
time.sleep(2)
|
|
60
|
+
r = self._ws.statement_execution.get_statement(r.statement_id)
|
|
61
|
+
state = r.status.state.value if r.status and r.status.state else "UNKNOWN"
|
|
62
|
+
if state != "SUCCEEDED":
|
|
63
|
+
err = r.status.error.message if r.status and r.status.error else state
|
|
64
|
+
raise DatabricksError(str(err)[:400])
|
|
65
|
+
cols = [c.name for c in (r.manifest.schema.columns if r.manifest and r.manifest.schema else [])]
|
|
66
|
+
rows = (r.result.data_array if r.result and r.result.data_array else [])
|
|
67
|
+
return [dict(zip(cols, row)) for row in rows]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def connect(profile: str | None, host: str | None, warehouse_id: str | None, use_sql: bool = True) -> Reader:
|
|
71
|
+
"""Unified auth: --profile, DATABRICKS_HOST/DATABRICKS_TOKEN, OAuth from `databricks auth login`.
|
|
72
|
+
A token is never accepted on the command line."""
|
|
73
|
+
from databricks.sdk import WorkspaceClient # lazy: keeps `db-agent --help` fast
|
|
74
|
+
|
|
75
|
+
kw: dict[str, Any] = {}
|
|
76
|
+
if profile:
|
|
77
|
+
kw["profile"] = profile
|
|
78
|
+
if host:
|
|
79
|
+
kw["host"] = host
|
|
80
|
+
ws = WorkspaceClient(**kw)
|
|
81
|
+
chosen = warehouse_id
|
|
82
|
+
if use_sql and not chosen:
|
|
83
|
+
running = [w for w in ws.warehouses.list() if w.state and w.state.value == "RUNNING"]
|
|
84
|
+
if running:
|
|
85
|
+
chosen = running[0].id
|
|
86
|
+
return Reader(ws, chosen if use_sql else None, sql_skipped=not use_sql)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Month-over-month DBU spend and a split of the change into volume and price."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from calendar import monthrange
|
|
6
|
+
from datetime import date
|
|
7
|
+
|
|
8
|
+
TIERS = ("STANDARD", "PREMIUM", "ENTERPRISE")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def month_list(start: str, end: str) -> list[date]:
|
|
12
|
+
"""'2026-09' .. '2026-11' -> first days of each month."""
|
|
13
|
+
def p(s: str) -> date:
|
|
14
|
+
y, m = s.split("-")
|
|
15
|
+
return date(int(y), int(m), 1)
|
|
16
|
+
a, b = p(start), p(end)
|
|
17
|
+
if b < a:
|
|
18
|
+
raise ValueError("--cost-to is before --cost-from")
|
|
19
|
+
out, cur = [], a
|
|
20
|
+
while cur <= b:
|
|
21
|
+
out.append(cur)
|
|
22
|
+
cur = date(cur.year + (cur.month == 12), cur.month % 12 + 1, 1)
|
|
23
|
+
if len(out) > 24:
|
|
24
|
+
raise ValueError("at most 24 months")
|
|
25
|
+
return out
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def default_range(today: date, n: int = 3) -> tuple[str, str]:
|
|
29
|
+
y, m = today.year, today.month - (n - 1)
|
|
30
|
+
while m < 1:
|
|
31
|
+
m += 12
|
|
32
|
+
y -= 1
|
|
33
|
+
return f"{y:04d}-{m:02d}", f"{today.year:04d}-{today.month:02d}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def tier_of(sku: str) -> str:
|
|
37
|
+
head = sku.split("_", 1)[0]
|
|
38
|
+
return head if head in TIERS else "OTHER"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def normalized(sku: str) -> str:
|
|
42
|
+
return sku.split("_", 1)[1] if tier_of(sku) != "OTHER" else sku
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _f(v) -> float | None:
|
|
46
|
+
return None if v is None else float(v)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def analyze(rows: list[dict], today: date) -> dict:
|
|
50
|
+
"""rows: month, sku_name, dbus, cost, days. Returns per-month figures and a volume/price split per day,
|
|
51
|
+
measured against the first month that has data."""
|
|
52
|
+
by_month: dict[str, dict] = {}
|
|
53
|
+
for r in rows:
|
|
54
|
+
key = str(r["month"])[:7]
|
|
55
|
+
m = by_month.setdefault(key, {"month": key, "dbus": 0.0, "cost": 0.0, "unpriced_dbus": 0.0, "days": 0,
|
|
56
|
+
"tiers": {}, "skus": {}})
|
|
57
|
+
dbus, cost = float(r["dbus"] or 0), _f(r.get("cost"))
|
|
58
|
+
m["dbus"] += dbus
|
|
59
|
+
m["days"] = max(m["days"], int(r.get("days") or 0))
|
|
60
|
+
if cost is None:
|
|
61
|
+
m["unpriced_dbus"] += dbus
|
|
62
|
+
else:
|
|
63
|
+
m["cost"] += cost
|
|
64
|
+
t = tier_of(r["sku_name"])
|
|
65
|
+
m["tiers"][t] = m["tiers"].get(t, 0.0) + dbus
|
|
66
|
+
n = normalized(r["sku_name"])
|
|
67
|
+
s = m["skus"].setdefault(n, {"dbus": 0.0, "cost": 0.0})
|
|
68
|
+
s["dbus"] += dbus
|
|
69
|
+
s["cost"] += cost or 0.0
|
|
70
|
+
months = [by_month[k] for k in sorted(by_month)]
|
|
71
|
+
for m in months:
|
|
72
|
+
y, mo = map(int, m["month"].split("-"))
|
|
73
|
+
m["partial"] = (y, mo) == (today.year, today.month)
|
|
74
|
+
d = max(m["days"], 1)
|
|
75
|
+
m["dbus_per_day"] = m["dbus"] / d
|
|
76
|
+
m["cost_per_day"] = m["cost"] / d
|
|
77
|
+
m["rate"] = (m["cost"] / (m["dbus"] - m["unpriced_dbus"])) if m["dbus"] - m["unpriced_dbus"] > 0 else None
|
|
78
|
+
m["dominant_tier"] = max(m["tiers"], key=m["tiers"].get) if m["tiers"] else None
|
|
79
|
+
if months:
|
|
80
|
+
base = months[0]
|
|
81
|
+
base_rate = {k: v["cost"] / v["dbus"] for k, v in base["skus"].items() if v["dbus"] > 0 and v["cost"] > 0}
|
|
82
|
+
for m in months[1:]:
|
|
83
|
+
d, bd = max(m["days"], 1), max(base["days"], 1)
|
|
84
|
+
at_base = 0.0
|
|
85
|
+
for k, v in m["skus"].items():
|
|
86
|
+
rate = base_rate.get(k) or (v["cost"] / v["dbus"] if v["dbus"] else 0.0)
|
|
87
|
+
at_base += v["dbus"] / d * rate
|
|
88
|
+
base_pd = base["cost"] / bd
|
|
89
|
+
m["vs_first"] = {"cost_per_day_change": m["cost_per_day"] - base_pd,
|
|
90
|
+
"pct": ((m["cost_per_day"] / base_pd - 1) * 100) if base_pd else None,
|
|
91
|
+
"volume_effect_per_day": at_base - base_pd,
|
|
92
|
+
"price_effect_per_day": m["cost_per_day"] - at_base}
|
|
93
|
+
tiers = [m["dominant_tier"] for m in months if m["dominant_tier"]]
|
|
94
|
+
changed = len(set(tiers)) > 1
|
|
95
|
+
return {"months": months, "tier_changed": changed,
|
|
96
|
+
"tier_sequence": [(m["month"], m["dominant_tier"]) for m in months]}
|