db-agent-kit 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- db_agent/__init__.py +3 -0
- db_agent/cli.py +32 -0
- db_agent/engines/__init__.py +36 -0
- db_agent/engines/databricks/__init__.py +1 -0
- db_agent/engines/databricks/checks.py +221 -0
- db_agent/engines/databricks/cli.py +61 -0
- db_agent/engines/databricks/client.py +86 -0
- db_agent/engines/databricks/cost.py +96 -0
- db_agent/engines/databricks/health.py +156 -0
- db_agent/engines/databricks/model.py +29 -0
- db_agent/engines/databricks/queries.py +98 -0
- db_agent/engines/databricks/render.py +78 -0
- db_agent/engines/databricks/runtimes.py +58 -0
- db_agent/engines/databricks/sqlguard.py +80 -0
- db_agent/engines/sqlserver/__init__.py +11 -0
- db_agent/engines/sqlserver/classify.py +102 -0
- db_agent/engines/sqlserver/cli.py +265 -0
- db_agent/engines/sqlserver/data/classification.yaml +29 -0
- db_agent/engines/sqlserver/db.py +183 -0
- db_agent/engines/sqlserver/guard.py +190 -0
- db_agent/engines/sqlserver/inventory.py +324 -0
- db_agent/engines/sqlserver/outputs.py +299 -0
- db_agent/engines/sqlserver/rdl.py +269 -0
- db_agent/engines/sqlserver/report.py +120 -0
- db_agent/engines/sqlserver/risks.py +184 -0
- db_agent/engines/sqlserver/rules.py +141 -0
- db_agent/engines/sqlserver/sqlrefs.py +103 -0
- db_agent/engines/sqlserver/ssrs.py +230 -0
- db_agent/engines/sqlserver/templates/summary.html.j2 +142 -0
- db_agent/engines/sqlserver/usage.py +127 -0
- db_agent/engines/sqlserver/validate.py +238 -0
- db_agent_kit-0.2.0.dist-info/METADATA +192 -0
- db_agent_kit-0.2.0.dist-info/RECORD +36 -0
- db_agent_kit-0.2.0.dist-info/WHEEL +4 -0
- db_agent_kit-0.2.0.dist-info/entry_points.txt +2 -0
- db_agent_kit-0.2.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Report usage from the SSRS execution log (ExecutionLog3) or an exported CSV.
|
|
2
|
+
|
|
3
|
+
Only counts are kept. User names are used to count distinct users and are then dropped.
|
|
4
|
+
|
|
5
|
+
Important: SSRS keeps the execution log for 60 days by default (ExecutionLogDaysKept). A report
|
|
6
|
+
that runs once a year looks unused in a 60-day log. So every result carries the log window, and
|
|
7
|
+
the classifier refuses to say "retire" unless the window covers the retire period.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import csv
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from datetime import datetime, timedelta
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from .db import SafeConnection
|
|
18
|
+
|
|
19
|
+
Q_USAGE = """
|
|
20
|
+
SELECT ItemPath,
|
|
21
|
+
MAX(TimeStart) AS last_run,
|
|
22
|
+
SUM(CASE WHEN TimeStart >= DATEADD(day, -90, GETDATE()) THEN 1 ELSE 0 END) AS runs_90,
|
|
23
|
+
SUM(CASE WHEN TimeStart >= DATEADD(day, -365, GETDATE()) THEN 1 ELSE 0 END) AS runs_365,
|
|
24
|
+
COUNT(DISTINCT UserName) AS distinct_users,
|
|
25
|
+
AVG(CAST(DATEDIFF(millisecond, TimeStart, TimeEnd) AS bigint)) AS avg_ms,
|
|
26
|
+
SUM(CASE WHEN Status <> 'rsSuccess' THEN 1 ELSE 0 END) AS errors
|
|
27
|
+
FROM dbo.ExecutionLog3
|
|
28
|
+
WHERE RequestType IN ('Interactive', 'Subscription')
|
|
29
|
+
GROUP BY ItemPath
|
|
30
|
+
"""
|
|
31
|
+
Q_WINDOW = "SELECT MIN(TimeStart) AS oldest, MAX(TimeStart) AS newest, COUNT(*) AS entries FROM dbo.ExecutionLog3"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class ReportUsage:
|
|
36
|
+
last_run: datetime | None = None
|
|
37
|
+
runs_90: int = 0
|
|
38
|
+
runs_365: int = 0
|
|
39
|
+
distinct_users: int = 0
|
|
40
|
+
avg_ms: int | None = None
|
|
41
|
+
errors: int = 0
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class UsageData:
|
|
46
|
+
by_path: dict[str, ReportUsage]
|
|
47
|
+
oldest: datetime | None
|
|
48
|
+
newest: datetime | None
|
|
49
|
+
as_of: datetime
|
|
50
|
+
source: str # "executionlog3" | "csv"
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def window_days(self) -> int:
|
|
54
|
+
"""How many days back the log reaches. 0 when the log is empty."""
|
|
55
|
+
if self.oldest is None:
|
|
56
|
+
return 0
|
|
57
|
+
return max(0, (self.as_of - self.oldest).days)
|
|
58
|
+
|
|
59
|
+
def get(self, path: str) -> ReportUsage:
|
|
60
|
+
return self.by_path.get(path.lower(), ReportUsage())
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _parse_dt(value: str) -> datetime | None:
|
|
64
|
+
value = (value or "").strip()
|
|
65
|
+
if not value:
|
|
66
|
+
return None
|
|
67
|
+
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d", "%Y-%m-%d %H:%M:%S.%f", "%m/%d/%Y %H:%M:%S", "%m/%d/%Y"):
|
|
68
|
+
try:
|
|
69
|
+
return datetime.strptime(value[:26] if "." in value else value, fmt)
|
|
70
|
+
except ValueError:
|
|
71
|
+
continue
|
|
72
|
+
try:
|
|
73
|
+
return datetime.fromisoformat(value.replace("Z", "")).replace(tzinfo=None)
|
|
74
|
+
except ValueError:
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def load_csv(path: Path, as_of: datetime | None = None) -> UsageData:
|
|
79
|
+
"""Columns follow ExecutionLog3: ItemPath, UserName, TimeStart, TimeEnd, Status, RequestType.
|
|
80
|
+
Only ItemPath and TimeStart are required."""
|
|
81
|
+
as_of = as_of or datetime.now()
|
|
82
|
+
rows: dict[str, list[dict]] = {}
|
|
83
|
+
oldest = newest = None
|
|
84
|
+
with open(path, newline="", encoding="utf-8-sig") as fh:
|
|
85
|
+
reader = csv.DictReader(fh)
|
|
86
|
+
header = {h.lower(): h for h in (reader.fieldnames or [])}
|
|
87
|
+
for need in ("itempath", "timestart"):
|
|
88
|
+
if need not in header:
|
|
89
|
+
raise ValueError(f"usage CSV needs a column named {'ItemPath' if need == 'itempath' else 'TimeStart'}")
|
|
90
|
+
for r in reader:
|
|
91
|
+
lr = {k.lower(): v for k, v in r.items()}
|
|
92
|
+
rtype = (lr.get("requesttype") or "Interactive").strip()
|
|
93
|
+
ts = _parse_dt(lr.get("timestart", ""))
|
|
94
|
+
if ts is None:
|
|
95
|
+
continue
|
|
96
|
+
oldest = ts if oldest is None or ts < oldest else oldest
|
|
97
|
+
newest = ts if newest is None or ts > newest else newest
|
|
98
|
+
if rtype not in ("Interactive", "Subscription"):
|
|
99
|
+
continue
|
|
100
|
+
rows.setdefault((lr.get("itempath") or "").strip().lower(), []).append(
|
|
101
|
+
{"ts": ts, "end": _parse_dt(lr.get("timeend", "")), "user": lr.get("username") or "", "status": (lr.get("status") or "rsSuccess").strip()}
|
|
102
|
+
)
|
|
103
|
+
by_path: dict[str, ReportUsage] = {}
|
|
104
|
+
d90, d365 = as_of - timedelta(days=90), as_of - timedelta(days=365)
|
|
105
|
+
for key, items in rows.items():
|
|
106
|
+
durations = [(i["end"] - i["ts"]).total_seconds() * 1000 for i in items if i["end"]]
|
|
107
|
+
by_path[key] = ReportUsage(
|
|
108
|
+
last_run=max(i["ts"] for i in items),
|
|
109
|
+
runs_90=sum(1 for i in items if i["ts"] >= d90),
|
|
110
|
+
runs_365=sum(1 for i in items if i["ts"] >= d365),
|
|
111
|
+
distinct_users=len({i["user"] for i in items if i["user"]}),
|
|
112
|
+
avg_ms=int(sum(durations) / len(durations)) if durations else None,
|
|
113
|
+
errors=sum(1 for i in items if i["status"] != "rsSuccess"),
|
|
114
|
+
)
|
|
115
|
+
return UsageData(by_path, oldest, newest, as_of, "csv")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def load_executionlog(conn: SafeConnection) -> UsageData:
|
|
119
|
+
win = conn.query(Q_WINDOW)[0]
|
|
120
|
+
by_path: dict[str, ReportUsage] = {}
|
|
121
|
+
for r in conn.query(Q_USAGE):
|
|
122
|
+
by_path[(r["ItemPath"] or "").lower()] = ReportUsage(
|
|
123
|
+
last_run=r["last_run"], runs_90=int(r["runs_90"] or 0), runs_365=int(r["runs_365"] or 0),
|
|
124
|
+
distinct_users=int(r["distinct_users"] or 0), avg_ms=int(r["avg_ms"]) if r["avg_ms"] is not None else None,
|
|
125
|
+
errors=int(r["errors"] or 0),
|
|
126
|
+
)
|
|
127
|
+
return UsageData(by_path, win["oldest"], win["newest"], datetime.now(), "executionlog3")
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""Compare report dataset results between a source and a target server.
|
|
2
|
+
|
|
3
|
+
Rows are folded into a row count and per-column totals as they arrive and are then discarded.
|
|
4
|
+
Only counts and totals reach memory beyond one chunk, and only counts and totals reach disk.
|
|
5
|
+
Every statement passes through the QueryGuard. Stored procedures run only with allow_exec.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
import re
|
|
12
|
+
import time
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from datetime import date, datetime
|
|
15
|
+
from decimal import Decimal
|
|
16
|
+
from typing import Any, Callable
|
|
17
|
+
|
|
18
|
+
from .db import ConnectionConfig, DatabaseError, SafeConnection
|
|
19
|
+
from .guard import GuardError, QueryGuard, sanitize
|
|
20
|
+
|
|
21
|
+
_PARAM_RE = re.compile(r"(?<![\w@])@(\w+)")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Fold:
|
|
26
|
+
columns: list[str] = field(default_factory=list)
|
|
27
|
+
rows: int = 0
|
|
28
|
+
sums: dict[str, Decimal | float] = field(default_factory=dict)
|
|
29
|
+
nonnull: dict[str, int] = field(default_factory=dict)
|
|
30
|
+
ms: int = 0
|
|
31
|
+
truncated: bool = False
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _is_number(v: Any) -> bool:
|
|
35
|
+
return isinstance(v, (int, float, Decimal)) and not isinstance(v, bool)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def fold_chunks(chunks: Any, max_rows: int) -> Fold:
|
|
39
|
+
out = Fold()
|
|
40
|
+
started = time.monotonic()
|
|
41
|
+
for cols, rows in chunks:
|
|
42
|
+
if not out.columns:
|
|
43
|
+
out.columns = list(cols)
|
|
44
|
+
for row in rows:
|
|
45
|
+
out.rows += 1
|
|
46
|
+
for c, v in zip(cols, row):
|
|
47
|
+
if v is None:
|
|
48
|
+
continue
|
|
49
|
+
out.nonnull[c] = out.nonnull.get(c, 0) + 1
|
|
50
|
+
if _is_number(v):
|
|
51
|
+
cur = out.sums.get(c, Decimal(0) if not isinstance(v, float) else 0.0)
|
|
52
|
+
if isinstance(cur, Decimal) and isinstance(v, float):
|
|
53
|
+
cur = float(cur)
|
|
54
|
+
elif isinstance(cur, float) and isinstance(v, Decimal):
|
|
55
|
+
v = float(v)
|
|
56
|
+
out.sums[c] = cur + v
|
|
57
|
+
if out.rows > max_rows:
|
|
58
|
+
out.truncated = True
|
|
59
|
+
break
|
|
60
|
+
out.ms = int((time.monotonic() - started) * 1000)
|
|
61
|
+
return out
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def compare(src: Fold, tgt: Fold, rel_tol: float = 1e-9) -> tuple[str, list[str]]:
|
|
65
|
+
"""Return ("pass" | "fail", differences). Differences hold counts and totals only."""
|
|
66
|
+
diffs: list[str] = []
|
|
67
|
+
s_cols, t_cols = [c.lower() for c in src.columns], [c.lower() for c in tgt.columns]
|
|
68
|
+
if s_cols != t_cols:
|
|
69
|
+
diffs.append(f"columns differ: source {src.columns} target {tgt.columns}")
|
|
70
|
+
if src.rows != tgt.rows:
|
|
71
|
+
diffs.append(f"row count: source {src.rows}, target {tgt.rows}")
|
|
72
|
+
for c in src.columns:
|
|
73
|
+
a, b = src.sums.get(c), tgt.sums.get(c)
|
|
74
|
+
if a is None and b is None:
|
|
75
|
+
continue
|
|
76
|
+
if a is None or b is None:
|
|
77
|
+
diffs.append(f"{c}: numeric total present on one side only")
|
|
78
|
+
continue
|
|
79
|
+
if isinstance(a, float) or isinstance(b, float):
|
|
80
|
+
ok = math.isclose(float(a), float(b), rel_tol=rel_tol, abs_tol=1e-9)
|
|
81
|
+
else:
|
|
82
|
+
ok = a == b
|
|
83
|
+
if not ok:
|
|
84
|
+
diffs.append(f"{c} total: source {a}, target {b}")
|
|
85
|
+
if src.nonnull.get(c, 0) != tgt.nonnull.get(c, 0):
|
|
86
|
+
diffs.append(f"{c} non-null count: source {src.nonnull.get(c, 0)}, target {tgt.nonnull.get(c, 0)}")
|
|
87
|
+
return ("fail" if diffs else "pass"), diffs
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def coerce(value: str, dtype: str | None) -> Any:
|
|
91
|
+
t = (dtype or "String").lower()
|
|
92
|
+
if t in ("integer", "int", "int32", "int64"):
|
|
93
|
+
return int(value)
|
|
94
|
+
if t in ("float", "decimal", "double"):
|
|
95
|
+
return float(value)
|
|
96
|
+
if t == "boolean":
|
|
97
|
+
return value.strip().lower() in ("true", "1", "yes")
|
|
98
|
+
if t == "datetime":
|
|
99
|
+
s = value.strip()
|
|
100
|
+
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
|
|
101
|
+
try:
|
|
102
|
+
return datetime.strptime(s, fmt)
|
|
103
|
+
except ValueError:
|
|
104
|
+
continue
|
|
105
|
+
raise ValueError(f"bad date value {value!r}")
|
|
106
|
+
return value
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def bind_params(report: dict[str, Any], dataset: dict[str, Any], overrides: dict[str, Any]) -> tuple[str, dict[str, Any]] | str:
|
|
110
|
+
"""Return (sql_with_placeholders, params) or a string explaining why the dataset must be skipped."""
|
|
111
|
+
rp = {p["name"].lower(): p for p in report["parameters"]}
|
|
112
|
+
sql = dataset["command_text"]
|
|
113
|
+
wanted = {m.group(1) for m in _PARAM_RE.finditer(sanitize(sql, preserve_length=True))}
|
|
114
|
+
mapping: dict[str, str] = {} # sql parameter -> report parameter
|
|
115
|
+
for q in dataset["query_parameters"]:
|
|
116
|
+
m = re.fullmatch(r"=\s*Parameters!(\w+)\.Value", q["value"].strip(), re.IGNORECASE)
|
|
117
|
+
mapping[q["name"].lstrip("@").lower()] = m.group(1) if m else q["value"]
|
|
118
|
+
params: dict[str, Any] = {}
|
|
119
|
+
for name in wanted:
|
|
120
|
+
source = mapping.get(name.lower(), name)
|
|
121
|
+
p = rp.get(source.lower())
|
|
122
|
+
if p and p["multi_value"]:
|
|
123
|
+
return f"parameter {source} is multi-value; not supported in this version"
|
|
124
|
+
raw = overrides.get(source) if source in overrides else overrides.get(name)
|
|
125
|
+
if raw is None and p:
|
|
126
|
+
raw = p["default"]
|
|
127
|
+
if raw is None or (isinstance(raw, str) and raw.startswith("=")):
|
|
128
|
+
return f"no value for parameter {source}; set it in params.yaml"
|
|
129
|
+
try:
|
|
130
|
+
params[name] = coerce(str(raw), p["type"] if p else None)
|
|
131
|
+
except ValueError as exc:
|
|
132
|
+
return f"parameter {source}: {exc}"
|
|
133
|
+
if not params:
|
|
134
|
+
return sql, {}
|
|
135
|
+
text = sanitize(sql, preserve_length=True)
|
|
136
|
+
out, last = [], 0
|
|
137
|
+
for m in _PARAM_RE.finditer(text):
|
|
138
|
+
out.append(sql[last : m.start()].replace("%", "%%"))
|
|
139
|
+
out.append(f"%({m.group(1)})s")
|
|
140
|
+
last = m.end()
|
|
141
|
+
out.append(sql[last:].replace("%", "%%"))
|
|
142
|
+
return "".join(out), params
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _run(cfg: ConnectionConfig, guard: QueryGuard, sql: str, params: dict[str, Any] | None, max_rows: int) -> Fold:
|
|
146
|
+
with SafeConnection(cfg, guard) as c:
|
|
147
|
+
return fold_chunks(c.stream(sql, params or None), max_rows)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def validate_reports(
|
|
151
|
+
reports: list[dict[str, Any]],
|
|
152
|
+
source: ConnectionConfig,
|
|
153
|
+
target: ConnectionConfig,
|
|
154
|
+
overrides: dict[str, Any] | None = None,
|
|
155
|
+
allow_exec: bool = False,
|
|
156
|
+
max_rows: int = 1_000_000,
|
|
157
|
+
progress: Callable[[str], None] | None = None,
|
|
158
|
+
) -> list[dict[str, Any]]:
|
|
159
|
+
overrides = overrides or {}
|
|
160
|
+
say = progress or (lambda _m: None)
|
|
161
|
+
results: list[dict[str, Any]] = []
|
|
162
|
+
for rep in reports:
|
|
163
|
+
if rep["classification"] != "move":
|
|
164
|
+
continue
|
|
165
|
+
rep_over = {**overrides.get("defaults", {}), **overrides.get("reports", {}).get(rep["path"], {})}
|
|
166
|
+
for ds in rep["datasets"]:
|
|
167
|
+
row = {"report": rep["path"], "dataset": ds["name"], "status": "skipped", "reason": "", "source_rows": None,
|
|
168
|
+
"target_rows": None, "source_ms": None, "target_ms": None, "differences": ""}
|
|
169
|
+
results.append(row)
|
|
170
|
+
say(f"{rep['path']} / {ds['name']}")
|
|
171
|
+
if ds["shared_reference"] and not ds["command_text"]:
|
|
172
|
+
row["reason"] = "shared dataset was not supplied"
|
|
173
|
+
continue
|
|
174
|
+
if ds["confidence"] == "n/a" and ds["command_type"] != "StoredProcedure":
|
|
175
|
+
row["reason"] = ds.get("parse_error") or "nothing to run"
|
|
176
|
+
continue
|
|
177
|
+
if ds["command_type"] == "TableDirect":
|
|
178
|
+
row["reason"] = "TableDirect datasets are not run"
|
|
179
|
+
continue
|
|
180
|
+
guard = QueryGuard()
|
|
181
|
+
if ds["command_type"] == "StoredProcedure":
|
|
182
|
+
if not allow_exec:
|
|
183
|
+
row["reason"] = "stored procedure; run again with --allow-exec"
|
|
184
|
+
continue
|
|
185
|
+
proc = ds["command_text"].strip()
|
|
186
|
+
guard.allow_proc(proc)
|
|
187
|
+
args = ", ".join(f"@{q['name'].lstrip('@')}=%({q['name'].lstrip('@')})s" for q in ds["query_parameters"])
|
|
188
|
+
bound = bind_exec(rep, ds, rep_over)
|
|
189
|
+
if isinstance(bound, str):
|
|
190
|
+
row["reason"] = bound
|
|
191
|
+
continue
|
|
192
|
+
sql, params = f"EXEC {proc} {args}".strip(), bound
|
|
193
|
+
else:
|
|
194
|
+
bound = bind_params(rep, ds, rep_over)
|
|
195
|
+
if isinstance(bound, str):
|
|
196
|
+
row["reason"] = bound
|
|
197
|
+
continue
|
|
198
|
+
sql, params = bound
|
|
199
|
+
try:
|
|
200
|
+
src = _run(source, guard, sql, params, max_rows)
|
|
201
|
+
except GuardError as exc:
|
|
202
|
+
row["reason"] = f"not run: cannot verify read-only ({exc})"
|
|
203
|
+
continue
|
|
204
|
+
except DatabaseError as exc:
|
|
205
|
+
row["reason"] = f"source error: {str(exc)[:160]}"
|
|
206
|
+
continue
|
|
207
|
+
if src.truncated:
|
|
208
|
+
row["reason"] = f"more than {max_rows} rows; not compared"
|
|
209
|
+
continue
|
|
210
|
+
try:
|
|
211
|
+
tgt = _run(target, guard, sql, params, max_rows)
|
|
212
|
+
except DatabaseError as exc:
|
|
213
|
+
row.update(status="fail", reason=f"target error: {str(exc)[:160]}", source_rows=src.rows, source_ms=src.ms)
|
|
214
|
+
continue
|
|
215
|
+
status, diffs = compare(src, tgt)
|
|
216
|
+
row.update(status=status, reason="" if status == "pass" else "results differ", source_rows=src.rows, target_rows=tgt.rows,
|
|
217
|
+
source_ms=src.ms, target_ms=tgt.ms, differences="; ".join(diffs))
|
|
218
|
+
return results
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def bind_exec(report: dict[str, Any], dataset: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any] | str:
|
|
222
|
+
rp = {p["name"].lower(): p for p in report["parameters"]}
|
|
223
|
+
params: dict[str, Any] = {}
|
|
224
|
+
for q in dataset["query_parameters"]:
|
|
225
|
+
name = q["name"].lstrip("@")
|
|
226
|
+
m = re.fullmatch(r"=\s*Parameters!(\w+)\.Value", q["value"].strip(), re.IGNORECASE)
|
|
227
|
+
source = m.group(1) if m else name
|
|
228
|
+
p = rp.get(source.lower())
|
|
229
|
+
raw = overrides.get(source, overrides.get(name))
|
|
230
|
+
if raw is None and p:
|
|
231
|
+
raw = p["default"]
|
|
232
|
+
if raw is None or (isinstance(raw, str) and raw.startswith("=")):
|
|
233
|
+
return f"no value for parameter {source}; set it in params.yaml"
|
|
234
|
+
try:
|
|
235
|
+
params[name] = coerce(str(raw), p["type"] if p else None)
|
|
236
|
+
except ValueError as exc:
|
|
237
|
+
return f"parameter {source}: {exc}"
|
|
238
|
+
return params
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: db-agent-kit
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Read-only DBA toolkit CLI: SQL Server 2016/2017 upgrade and SSRS to Power BI inventory, Databricks workspace health and DBU cost checks.
|
|
5
|
+
Project-URL: Homepage, https://github.com/db-agent/db-agent-kit
|
|
6
|
+
Project-URL: Documentation, https://github.com/db-agent/db-agent-kit/tree/main/docs
|
|
7
|
+
Project-URL: Issues, https://github.com/db-agent/db-agent-kit/issues
|
|
8
|
+
Project-URL: Text-to-SQL agent, https://github.com/db-agent/db-agent
|
|
9
|
+
Project-URL: Fixed-price help, https://becloudready.com/sql-server
|
|
10
|
+
License-Expression: Apache-2.0
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: azure-sql,cli,database-inventory,database-migration,databricks,dba,dbu-cost,end-of-life,migration-assessment,power-bi,sql-server,sql-server-upgrade,ssrs,unity-catalog
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Information Technology
|
|
16
|
+
Classifier: Intended Audience :: System Administrators
|
|
17
|
+
Classifier: Operating System :: OS Independent
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Database
|
|
23
|
+
Classifier: Topic :: System :: Systems Administration
|
|
24
|
+
Classifier: Topic :: Utilities
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Requires-Dist: databricks-sdk>=0.40
|
|
27
|
+
Requires-Dist: jinja2>=3.1
|
|
28
|
+
Requires-Dist: lxml>=5
|
|
29
|
+
Requires-Dist: openpyxl>=3.1
|
|
30
|
+
Requires-Dist: pandas>=2.2
|
|
31
|
+
Requires-Dist: pymssql>=2.3
|
|
32
|
+
Requires-Dist: pyyaml>=6
|
|
33
|
+
Requires-Dist: sqlglot>=25
|
|
34
|
+
Requires-Dist: typer>=0.12
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
37
|
+
Provides-Extra: odbc
|
|
38
|
+
Requires-Dist: pyodbc>=5; extra == 'odbc'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# db-agent CLI: read-only DBA toolkit for SQL Server, SSRS and Databricks
|
|
42
|
+
|
|
43
|
+
Check a database before you upgrade, migrate or pay for another year of it. db-agent only reads, and writes a report you can hand to a manager: SQL Server 2016/2017 upgrade risks, an SSRS to Power BI report inventory, and Databricks workspace health with DBU spend. One command per engine. Nothing is changed on any server.
|
|
44
|
+
|
|
45
|
+
Looking for the text-to-SQL agent? See [db-agent/db-agent](https://github.com/db-agent/db-agent).
|
|
46
|
+
|
|
47
|
+
```
|
|
48
|
+
db-agent <engine> <verb>
|
|
49
|
+
db-agent engines # what is implemented and what is planned
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
| Engine | Status | Verbs |
|
|
53
|
+
|---|---|---|
|
|
54
|
+
| `sqlserver` | available | `inventory`, `ssrs`, `validate`, `report` |
|
|
55
|
+
| `databricks` | available; live-tested on one small AWS workspace only, see the docs | `health` ([docs](https://github.com/db-agent/db-agent-kit/blob/main/docs/databricks-health.md)) |
|
|
56
|
+
|
|
57
|
+
Each engine is described below. To add one, see [docs/adding-an-engine.md](https://github.com/db-agent/db-agent-kit/blob/main/docs/adding-an-engine.md).
|
|
58
|
+
|
|
59
|
+
## db-agent sqlserver: inventory SQL Server and SSRS reports before you upgrade or move to Power BI
|
|
60
|
+
|
|
61
|
+
SQL Server 2016 reached end of support on July 14, 2026. SQL Server 2017 follows on October 12, 2027.
|
|
62
|
+
If you run SSRS (SQL Server Reporting Services) on either, you have two jobs: upgrade the database
|
|
63
|
+
server, and decide what happens to every report. This command line tool does the **SSRS migration
|
|
64
|
+
inventory** and the upgrade check, **read-only**. It lists your databases and reports, flags what will
|
|
65
|
+
break, sorts each report into **move**, **review** or **retire**, checks results after a move, and writes
|
|
66
|
+
a report you can hand to a manager or a client. It is the first step of an **SSRS to Power BI** move.
|
|
67
|
+
|
|
68
|
+
*Part of the [db-agent](https://github.com/db-agent) project.* Apache-2.0.
|
|
69
|
+
|
|
70
|
+

|
|
71
|
+
|
|
72
|
+
## What it does
|
|
73
|
+
|
|
74
|
+
| Command | What you get |
|
|
75
|
+
|---|---|
|
|
76
|
+
| `sqlserver inventory` | Version, edition, databases, tables, row counts (from metadata), views, procedures with code, linked servers, Agent jobs. **Upgrade risks**, each with a link to the Microsoft page it comes from. |
|
|
77
|
+
| `sqlserver ssrs` | Every report: data sources, datasets, tables and procedures used, parameters, subreports, custom code, subscriptions, usage. Each report is **move**, **review** or **retire** with the reason. |
|
|
78
|
+
| `sqlserver validate` | Runs each "move" report's queries on the old and new server and compares row counts and totals. |
|
|
79
|
+
| `sqlserver report` | One printable `summary.html` and one Excel workbook with all of it. |
|
|
80
|
+
|
|
81
|
+
## Install
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
pip install db-agent-kit # after the first PyPI release
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
From source (Python 3.11 or newer):
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
git clone https://github.com/db-agent/db-agent-kit && cd db-agent-kit
|
|
91
|
+
python3 -m venv .venv && .venv/bin/pip install -e .
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
A Docker image and a Windows single-file `.exe` do not exist yet.
|
|
95
|
+
|
|
96
|
+
## 5-minute demo (no database needed)
|
|
97
|
+
|
|
98
|
+
Run this from the source checkout (see Install):
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
make demo-files
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
This reads 12 sample reports and a synthetic usage log (`examples/`, clearly labelled sample data) and
|
|
105
|
+
opens `summary.html`. You will see 5 move, 5 review and 2 retire, each with a reason.
|
|
106
|
+
|
|
107
|
+
To also see `inventory` and `validate`, start two SQL Server containers, restore the AdventureWorks
|
|
108
|
+
sample database and run `make demo`. See [docs/demo.md](https://github.com/db-agent/db-agent-kit/blob/main/docs/demo.md).
|
|
109
|
+
|
|
110
|
+
## Use it
|
|
111
|
+
|
|
112
|
+
Run it as `db-agent sqlserver <command>`.
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
db-agent sqlserver inventory --server sql01 --user dbagent_reader # password: env DBAGENT_SQL_PASSWORD or a prompt
|
|
116
|
+
db-agent sqlserver ssrs --reportserver-db ReportServer --server sql01 --user dbagent_reader
|
|
117
|
+
db-agent sqlserver ssrs --rdl-folder ./reports --usage-csv executionlog.csv # no server access needed
|
|
118
|
+
db-agent sqlserver validate --source "server=old,database=Sales,user=u" --target "server=new,database=Sales,user=u" --reports db-agent-sqlserver-output/reports.csv
|
|
119
|
+
db-agent sqlserver report
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Output goes to `./db-agent-sqlserver-output` (change with `--out`). Windows authentication needs `--driver odbc`,
|
|
123
|
+
`pip install ".[odbc]"` and a Microsoft ODBC driver for SQL Server. The tool finds driver 18, 17 or 13 on its own; set
|
|
124
|
+
`DBAGENT_ODBC_DRIVER` to force one. It was tested with driver 17 and a local Windows account (NTLM). Domain (Kerberos)
|
|
125
|
+
logins and Microsoft Entra ID were not tested.
|
|
126
|
+
|
|
127
|
+
## What your DBA needs to grant
|
|
128
|
+
|
|
129
|
+
A dedicated login with read access to metadata and the ReportServer catalog. Nothing that can write.
|
|
130
|
+
The exact T-SQL script is in [docs/permissions.md](https://github.com/db-agent/db-agent-kit/blob/main/docs/permissions.md).
|
|
131
|
+
|
|
132
|
+
## Sample output
|
|
133
|
+
|
|
134
|
+
`reports.csv`, one row per report (a few of the columns; run counts move with the date you run it):
|
|
135
|
+
|
|
136
|
+
| path | classification | runs_365 | last_run | reasons |
|
|
137
|
+
|---|---|---|---|---|
|
|
138
|
+
| /Inventory/BillOfMaterials | move | 41 | 2026-11-03 | Plain report, no hard-to-move features |
|
|
139
|
+
| /HR/EmployeeDirectory | review | 12 | 2026-11-02 | Custom code block (3 lines) |
|
|
140
|
+
| /Purchasing/VendorSpend | review | 50 | 2026-11-04 | Dataset SQL could not be fully parsed (Spend); table list is a guess |
|
|
141
|
+
| /Inventory/LegacyInventory | retire | 0 | | No runs in the last 365 days (log covers 522 days) |
|
|
142
|
+
|
|
143
|
+
## What it does not do
|
|
144
|
+
|
|
145
|
+
- It does **not convert reports.** It tells you what to move and what is hard. A layout hint says which reports look like Power BI paginated reports and which look like an interactive rebuild. It is a hint only.
|
|
146
|
+
- It does **not read business data** during `inventory` or `ssrs`. Row counts come from system views.
|
|
147
|
+
- `validate` is the one command that runs report queries. It keeps **only row counts and numeric totals**, never row values, and only on a login you choose to give read access to.
|
|
148
|
+
- It does **not change anything** on any server, and sends nothing over the network except to the SQL Servers you name.
|
|
149
|
+
|
|
150
|
+
## FAQ
|
|
151
|
+
|
|
152
|
+
**Is it safe?** Every statement goes through one guard that allows a single `SELECT` (or, with `--allow-exec`, a stored procedure you named). `SELECT ... INTO`, `OPENROWSET`, linked servers and anything it cannot parse are refused. Tests check that no command can bypass it. The login's permissions are the real boundary: give it only what [docs/permissions.md](https://github.com/db-agent/db-agent-kit/blob/main/docs/permissions.md) lists.
|
|
153
|
+
|
|
154
|
+
**What access does it need?** See above. For `ssrs` without database access, export the `.rdl` files and the execution log and run it on a laptop.
|
|
155
|
+
|
|
156
|
+
**Does it work with Azure SQL?** It reads Azure SQL Database and Managed Instance for `inventory` and flags what would not work there. Tested against one Azure SQL Database (AdventureWorksLT); Managed Instance is not tested.
|
|
157
|
+
|
|
158
|
+
**What about Power BI paginated reports?** Reports are classified by what is hard to move, not converted. The hint column says which look paginated. Check Microsoft's current paginated report limits for your reports.
|
|
159
|
+
|
|
160
|
+
**Why are no reports marked retire?** SSRS keeps its usage log for 60 days by default. A report that runs once a year looks unused in a 60-day log. The kit only calls a report retire when the log covers 365 days, and says so. See [docs/ssrs-inventory.md](https://github.com/db-agent/db-agent-kit/blob/main/docs/ssrs-inventory.md).
|
|
161
|
+
|
|
162
|
+
**Where does my data go?** Nowhere. Everything is written to a local folder. Object code (`modules.json`) can contain secrets: skip it with `--no-code`.
|
|
163
|
+
|
|
164
|
+
## db-agent databricks: workspace health and DBU spend
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
databricks auth login --host https://<workspace>.cloud.databricks.com
|
|
168
|
+
db-agent databricks health # uses a SQL warehouse that is already running
|
|
169
|
+
db-agent databricks health --profile <name> --warehouse-id <id> --out ./out
|
|
170
|
+
db-agent databricks health --no-sql # REST API checks only, no warehouse needed
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
It reads the REST API and system tables and changes nothing. It checks:
|
|
174
|
+
|
|
175
|
+
- Clusters and job clusters on runtimes past end of support, or within 180 days of it (`--eos-warning-days`). In the built-in table, 13.3 LTS ended on 2026-08-22 and 14.3 LTS ends on 2027-02-01.
|
|
176
|
+
- Legacy dependencies: Hive metastore tables, DBFS mount events, init scripts stored on DBFS, and clusters without a Unity Catalog access mode.
|
|
177
|
+
- DBU spend by month at list price, split into usage and price change, with a note when the SKU tier changes.
|
|
178
|
+
- Spend waste: scheduled jobs on all-purpose clusters, clusters with auto-termination off, SQL warehouses billed with no queries or with auto-stop off, and spend with no tags.
|
|
179
|
+
|
|
180
|
+
Live-tested on one small AWS workspace only. See [docs/databricks-health.md](https://github.com/db-agent/db-agent-kit/blob/main/docs/databricks-health.md) for options, access needed and limits.
|
|
181
|
+
|
|
182
|
+
## Status
|
|
183
|
+
|
|
184
|
+
Version 0.2.0. What was tested is listed in [docs/testing.md](https://github.com/db-agent/db-agent-kit/blob/main/docs/testing.md):
|
|
185
|
+
|
|
186
|
+
- `sqlserver`: real SQL Server 2017 and 2022 (Linux, Docker), real SQL Server 2016 SP3 with a real SSRS 2016 server (Windows VM), and `inventory` on one Azure SQL Database (AdventureWorksLT). ODBC with Windows authentication was tested with a local account.
|
|
187
|
+
- `databricks`: one small AWS workspace. See [docs/databricks-health.md](https://github.com/db-agent/db-agent-kit/blob/main/docs/databricks-health.md).
|
|
188
|
+
- Not tested: SSRS 2017 and later, Power BI Report Server, domain (Kerberos) authentication, Entra ID, Azure SQL Managed Instance.
|
|
189
|
+
|
|
190
|
+
[Upgrade risk rules](https://github.com/db-agent/db-agent-kit/blob/main/docs/risk-rules.md) · [SSRS inventory](https://github.com/db-agent/db-agent-kit/blob/main/docs/ssrs-inventory.md) · [Permissions](https://github.com/db-agent/db-agent-kit/blob/main/docs/permissions.md) · [Examples](https://github.com/db-agent/db-agent-kit/blob/main/examples/README.md)
|
|
191
|
+
|
|
192
|
+
Need the upgrade and report move done for you at a fixed price? [becloudready.com/sql-server](https://becloudready.com/sql-server)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
db_agent/__init__.py,sha256=j0-dWiRXV6ESuGrqeYBvlu8mO0MUwHwC__PNVJM55EY,100
|
|
2
|
+
db_agent/cli.py,sha256=7JtZPFtoMLYFWDa0kZrxBMjO2D1m_V9v5FMWjBcVx1s,852
|
|
3
|
+
db_agent/engines/__init__.py,sha256=WWsjznYgYopDYXtmHR8_y6s--5p998kclxzBXIsXQXI,1410
|
|
4
|
+
db_agent/engines/databricks/__init__.py,sha256=mp5iUZ2ZVR3_1nhzt4hvVD8A60JegdhSPJ1lPxZtkl4,96
|
|
5
|
+
db_agent/engines/databricks/checks.py,sha256=RFo0S0XICjpeUYyziVmxOzzs9wUPt8zfTtdb7pJWDWY,12235
|
|
6
|
+
db_agent/engines/databricks/cli.py,sha256=_s8vBYvB2UedkQM8lOGfAz7ApY6yi95EegcgHYQq8kM,3565
|
|
7
|
+
db_agent/engines/databricks/client.py,sha256=4l91AVduqhwk4wZVWu54UQX-SqIpu9c1Bs4z_EwwAkA,3548
|
|
8
|
+
db_agent/engines/databricks/cost.py,sha256=H0EweWS7C29WIRDQZRT0bJzkuPG08mKh4aR-4NroEzw,3821
|
|
9
|
+
db_agent/engines/databricks/health.py,sha256=JRyQRNJci4aSJOzx9RPdv4M6PW7lzUxvR-_77v7EaUU,7950
|
|
10
|
+
db_agent/engines/databricks/model.py,sha256=3lvw6puBvvRAOoRxuDI8s2r1QuD_Ep_diM76lptgC2g,633
|
|
11
|
+
db_agent/engines/databricks/queries.py,sha256=FaY9OpKIFrP-BzRNyXiBCTBcnrsgTi0S3M30x1WwLig,4313
|
|
12
|
+
db_agent/engines/databricks/render.py,sha256=BCyVZgHgL6ZaBc4fY6XayLsePwLka7tuRFhoXQZ3KhI,3399
|
|
13
|
+
db_agent/engines/databricks/runtimes.py,sha256=KLSACuJIdPM9NxgmW-l2_GNRXpe-6wMb3HwidJDwQ30,2826
|
|
14
|
+
db_agent/engines/databricks/sqlguard.py,sha256=XjfmDvBCvWkOVC-gYje_RyrxBJzPgErKEuyiezAVcR8,2878
|
|
15
|
+
db_agent/engines/sqlserver/__init__.py,sha256=I7SsCx3QCahLxVuQvYe0sp88_rZUQfqP6hHqnica674,424
|
|
16
|
+
db_agent/engines/sqlserver/classify.py,sha256=GGbHa7tKVunZJNd5G_ejeXFLKnhOk8n9mpCtY2RVO1A,5131
|
|
17
|
+
db_agent/engines/sqlserver/cli.py,sha256=h1yVDoCLkK1y8nTLaRt8m8MyXqSjGWSpDsfPhxUyssE,14810
|
|
18
|
+
db_agent/engines/sqlserver/db.py,sha256=EnyXSHbtPgibChP3fCBRu7a7BM1fy6FqL4mGSo0LNKc,7036
|
|
19
|
+
db_agent/engines/sqlserver/guard.py,sha256=p_DLTjDEzlJm7tvNjUFn03zfB4bO6f7lePvXpuINOsM,7669
|
|
20
|
+
db_agent/engines/sqlserver/inventory.py,sha256=znm-YgsJ0yVK3Oexl5uKSqC57u42e0C796_vqSKHZmM,15757
|
|
21
|
+
db_agent/engines/sqlserver/outputs.py,sha256=Ehnd6xziq4D110_uPbt7enJSrt1DrwfRJyf8mJAi1Ys,15577
|
|
22
|
+
db_agent/engines/sqlserver/rdl.py,sha256=57vHNpTJx5NUmgxeKFcxQvsEQFVl5y5KORZtOLoBiT8,11496
|
|
23
|
+
db_agent/engines/sqlserver/report.py,sha256=tEfzZMqOLxYWw1v05tpz3WFTMeim_6vY7ndU0E8Seeg,7359
|
|
24
|
+
db_agent/engines/sqlserver/risks.py,sha256=AgEnGtTq0ABCmSif8FZrkLNQHwjdBn8u01py9jEAeMk,12867
|
|
25
|
+
db_agent/engines/sqlserver/rules.py,sha256=xKdxTZlYFrTb2pDlXEgUs9RCGnUHWvMqGddPQrTuDTo,8723
|
|
26
|
+
db_agent/engines/sqlserver/sqlrefs.py,sha256=Yt49pjg2Qk-U-mfQdObC5CHzHzDoEQCrpfB4NSylE5g,4091
|
|
27
|
+
db_agent/engines/sqlserver/ssrs.py,sha256=CyVUCiOBsv-eQS3H2MLpgcHINn3KFtpP-WbEaF3gjS0,10205
|
|
28
|
+
db_agent/engines/sqlserver/usage.py,sha256=kePBG86DYQIFR-q1H37E9wwreLukYwUvvd7X5m3ZRiY,5301
|
|
29
|
+
db_agent/engines/sqlserver/validate.py,sha256=k_ie-XbtnCt_EDYW_9lpmV7NlgFbQ3wvKwp5GVE2jr0,10230
|
|
30
|
+
db_agent/engines/sqlserver/data/classification.yaml,sha256=PnHZiKfYV3IEhx68WFiV6jx4vtMbwIjMmVGiD7Dpsh8,1049
|
|
31
|
+
db_agent/engines/sqlserver/templates/summary.html.j2,sha256=D158rr9VSQY1HjBEnY8DiWa80VA9LBHEwCrarXUZomM,9830
|
|
32
|
+
db_agent_kit-0.2.0.dist-info/METADATA,sha256=BeXrzDkeaDrCp3ws_a8mhEcjuA4yKoaqIQlOeIRCG1A,11785
|
|
33
|
+
db_agent_kit-0.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
34
|
+
db_agent_kit-0.2.0.dist-info/entry_points.txt,sha256=Ts_abJe0exYQwFimnNb7-K_xCczRjHjTzwFNjf-Y0PU,46
|
|
35
|
+
db_agent_kit-0.2.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
36
|
+
db_agent_kit-0.2.0.dist-info/RECORD,,
|