db-agent-kit 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- db_agent/__init__.py +3 -0
- db_agent/cli.py +32 -0
- db_agent/engines/__init__.py +36 -0
- db_agent/engines/databricks/__init__.py +1 -0
- db_agent/engines/databricks/checks.py +221 -0
- db_agent/engines/databricks/cli.py +61 -0
- db_agent/engines/databricks/client.py +86 -0
- db_agent/engines/databricks/cost.py +96 -0
- db_agent/engines/databricks/health.py +156 -0
- db_agent/engines/databricks/model.py +29 -0
- db_agent/engines/databricks/queries.py +98 -0
- db_agent/engines/databricks/render.py +78 -0
- db_agent/engines/databricks/runtimes.py +58 -0
- db_agent/engines/databricks/sqlguard.py +80 -0
- db_agent/engines/sqlserver/__init__.py +11 -0
- db_agent/engines/sqlserver/classify.py +102 -0
- db_agent/engines/sqlserver/cli.py +265 -0
- db_agent/engines/sqlserver/data/classification.yaml +29 -0
- db_agent/engines/sqlserver/db.py +183 -0
- db_agent/engines/sqlserver/guard.py +190 -0
- db_agent/engines/sqlserver/inventory.py +324 -0
- db_agent/engines/sqlserver/outputs.py +299 -0
- db_agent/engines/sqlserver/rdl.py +269 -0
- db_agent/engines/sqlserver/report.py +120 -0
- db_agent/engines/sqlserver/risks.py +184 -0
- db_agent/engines/sqlserver/rules.py +141 -0
- db_agent/engines/sqlserver/sqlrefs.py +103 -0
- db_agent/engines/sqlserver/ssrs.py +230 -0
- db_agent/engines/sqlserver/templates/summary.html.j2 +142 -0
- db_agent/engines/sqlserver/usage.py +127 -0
- db_agent/engines/sqlserver/validate.py +238 -0
- db_agent_kit-0.2.0.dist-info/METADATA +192 -0
- db_agent_kit-0.2.0.dist-info/RECORD +36 -0
- db_agent_kit-0.2.0.dist-info/WHEEL +4 -0
- db_agent_kit-0.2.0.dist-info/entry_points.txt +2 -0
- db_agent_kit-0.2.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
"""Command line: db-agent sqlserver <inventory|ssrs|validate|report|version>."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import functools
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Annotated, Any, Callable, Optional
|
|
11
|
+
|
|
12
|
+
import typer
|
|
13
|
+
import yaml
|
|
14
|
+
|
|
15
|
+
from . import SUPPORTED_SQL_SERVER, __version__, classify, outputs, report as report_mod, risks as risks_mod, ssrs as ssrs_mod, usage as usage_mod, validate as validate_mod
|
|
16
|
+
from . import inventory as inventory_mod
|
|
17
|
+
from .db import ConnectionConfig, DatabaseError, SafeConnection
|
|
18
|
+
from .guard import GuardError, QueryGuard
|
|
19
|
+
|
|
20
|
+
app = typer.Typer(add_completion=False, no_args_is_help=True, help="Read-only inventory of SQL Server and SSRS reports before an upgrade or a move to Power BI.")
|
|
21
|
+
|
|
22
|
+
# ---- shared options -----------------------------------------------------------------------------
|
|
23
|
+
Server = Annotated[Optional[str], typer.Option("--server", envvar="DBAGENT_SQL_SERVER", help="SQL Server host name or address.")]
|
|
24
|
+
Port = Annotated[int, typer.Option("--port", envvar="DBAGENT_SQL_PORT", help="TCP port.")]
|
|
25
|
+
Database = Annotated[Optional[str], typer.Option("--database", envvar="DBAGENT_SQL_DATABASE", help="Database to connect to.")]
|
|
26
|
+
User = Annotated[Optional[str], typer.Option("--user", envvar="DBAGENT_SQL_USER", help="SQL login. The password comes from DBAGENT_SQL_PASSWORD or a hidden prompt.")]
|
|
27
|
+
Driver = Annotated[str, typer.Option("--driver", help="pymssql (default) or odbc (Windows authentication, Entra ID).")]
|
|
28
|
+
Trusted = Annotated[bool, typer.Option("--trusted", help="Windows authentication. Needs --driver odbc.")]
|
|
29
|
+
Encrypt = Annotated[bool, typer.Option("--encrypt/--no-encrypt", help="Encrypt the connection.")]
|
|
30
|
+
TrustCert = Annotated[bool, typer.Option("--trust-server-certificate", help="Accept a self-signed server certificate (odbc driver).")]
|
|
31
|
+
Out = Annotated[Path, typer.Option("--out", help="Folder for all output files.")]
|
|
32
|
+
Format = Annotated[str, typer.Option("--format", help="Comma list of: md, json, xlsx, html.")]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _formats(value: str) -> set[str]:
|
|
36
|
+
chosen = {f.strip().lower() for f in value.split(",") if f.strip()}
|
|
37
|
+
bad = chosen - set(outputs.FORMATS)
|
|
38
|
+
if bad:
|
|
39
|
+
raise typer.BadParameter(f"unknown format(s): {', '.join(sorted(bad))}. Choose from {', '.join(outputs.FORMATS)}.")
|
|
40
|
+
return chosen
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _password(user: str | None, trusted: bool, env: str = "DBAGENT_SQL_PASSWORD", label: str = "SQL Server") -> str | None:
|
|
44
|
+
if trusted or not user:
|
|
45
|
+
return None
|
|
46
|
+
pw = os.environ.get(env)
|
|
47
|
+
if pw is None:
|
|
48
|
+
pw = typer.prompt(f"Password for {user} on {label}", hide_input=True)
|
|
49
|
+
return pw
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _config(server, port, database, user, driver, trusted, encrypt, trust_cert) -> ConnectionConfig:
|
|
53
|
+
if not server:
|
|
54
|
+
raise typer.BadParameter("--server is required (or set DBAGENT_SQL_SERVER).")
|
|
55
|
+
if driver not in ("pymssql", "odbc"):
|
|
56
|
+
raise typer.BadParameter("--driver must be pymssql or odbc.")
|
|
57
|
+
if trusted and driver != "odbc":
|
|
58
|
+
raise typer.BadParameter("--trusted (Windows authentication) needs --driver odbc.")
|
|
59
|
+
if not trusted and not user:
|
|
60
|
+
raise typer.BadParameter("--user is required unless --trusted is used.")
|
|
61
|
+
return ConnectionConfig(server=server, port=port, database=database or "", user=user, password=_password(user, trusted),
|
|
62
|
+
driver=driver, trusted=trusted, encrypt=encrypt, trust_server_certificate=trust_cert)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def parse_conn_spec(spec: str, env_prefix: str, label: str) -> ConnectionConfig:
|
|
66
|
+
"""server=host,port=1433,database=db,user=u,driver=odbc,trusted=yes,encrypt=no,trust_cert=yes
|
|
67
|
+
|
|
68
|
+
The password is never part of the spec: set <env_prefix>_PASSWORD or you will be asked."""
|
|
69
|
+
kv: dict[str, str] = {}
|
|
70
|
+
for part in spec.split(","):
|
|
71
|
+
if not part.strip():
|
|
72
|
+
continue
|
|
73
|
+
k, sep, v = part.partition("=")
|
|
74
|
+
if not sep:
|
|
75
|
+
raise typer.BadParameter(f"{label}: expected key=value pairs, got {part!r}")
|
|
76
|
+
kv[k.strip().lower()] = v.strip()
|
|
77
|
+
if "password" in kv or "pwd" in kv:
|
|
78
|
+
raise typer.BadParameter(f"{label}: do not put the password on the command line. Set {env_prefix}_PASSWORD or wait for the prompt.")
|
|
79
|
+
unknown = set(kv) - {"server", "port", "database", "user", "driver", "trusted", "encrypt", "trust_cert"}
|
|
80
|
+
if unknown:
|
|
81
|
+
raise typer.BadParameter(f"{label}: unknown key(s): {', '.join(sorted(unknown))}")
|
|
82
|
+
yes = lambda v: v.lower() in ("1", "true", "yes") # noqa: E731
|
|
83
|
+
trusted = yes(kv.get("trusted", "no"))
|
|
84
|
+
user = kv.get("user")
|
|
85
|
+
return ConnectionConfig(
|
|
86
|
+
server=kv.get("server", ""), port=int(kv.get("port", 1433)), database=kv.get("database", ""), user=user,
|
|
87
|
+
password=_password(user, trusted, f"{env_prefix}_PASSWORD", label), driver=kv.get("driver", "pymssql"), trusted=trusted,
|
|
88
|
+
encrypt=yes(kv.get("encrypt", "yes")), trust_server_certificate=yes(kv.get("trust_cert", "no")),
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _guarded(fn: Callable[..., Any]) -> Callable[..., Any]:
|
|
93
|
+
@functools.wraps(fn)
|
|
94
|
+
def run(*a: Any, **kw: Any) -> Any:
|
|
95
|
+
try:
|
|
96
|
+
return fn(*a, **kw)
|
|
97
|
+
except (DatabaseError, GuardError, ValueError, FileNotFoundError) as exc:
|
|
98
|
+
typer.secho(f"Error: {exc}", fg=typer.colors.RED, err=True)
|
|
99
|
+
raise typer.Exit(1)
|
|
100
|
+
|
|
101
|
+
return run
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _say(msg: str) -> None:
|
|
105
|
+
typer.echo(msg, err=True)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _done(written: list[str], out: Path) -> None:
|
|
109
|
+
typer.echo(f"Wrote to {out}:")
|
|
110
|
+
for w in written:
|
|
111
|
+
typer.echo(f" {w}")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# ---- commands -------------------------------------------------------------------------------------
|
|
115
|
+
@app.command()
|
|
116
|
+
@_guarded
|
|
117
|
+
def inventory(
|
|
118
|
+
server: Server = None, port: Port = 1433, database: Database = None, user: User = None, driver: Driver = "pymssql",
|
|
119
|
+
trusted: Trusted = False, encrypt: Encrypt = True, trust_server_certificate: TrustCert = False,
|
|
120
|
+
out: Out = Path("db-agent-sqlserver-output"), format: Format = "md,json,xlsx,html",
|
|
121
|
+
databases: Annotated[Optional[str], typer.Option("--databases", help="Comma list of databases to read. Default: all user databases.")] = None,
|
|
122
|
+
include_system: Annotated[bool, typer.Option("--include-system", help="Also read master, model, msdb, tempdb.")] = False,
|
|
123
|
+
target: Annotated[int, typer.Option("--target", help="Upgrade target: 2019 or 2022.")] = 2022,
|
|
124
|
+
code: Annotated[bool, typer.Option("--code/--no-code", help="Save stored procedure and function code to modules.json.")] = True,
|
|
125
|
+
) -> None:
|
|
126
|
+
"""Inventory a SQL Server: version, databases, objects and upgrade risks. Metadata only."""
|
|
127
|
+
fmts = _formats(format)
|
|
128
|
+
if target not in (2019, 2022):
|
|
129
|
+
raise typer.BadParameter("--target must be 2019 or 2022")
|
|
130
|
+
cfg = _config(server, port, database, user, driver, trusted, encrypt, trust_server_certificate)
|
|
131
|
+
inv = inventory_mod.collect(cfg, [d.strip() for d in databases.split(",")] if databases else None, include_system, QueryGuard(), _say)
|
|
132
|
+
risks = [r.as_dict() for r in risks_mod.assess(inv, 15 if target == 2019 else 16)]
|
|
133
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
134
|
+
written = outputs.write_inventory(out, inv, risks, fmts, code)
|
|
135
|
+
if "html" in fmts:
|
|
136
|
+
written += report_mod.write_report(out)
|
|
137
|
+
outputs.order_workbook(out)
|
|
138
|
+
_done(written, out)
|
|
139
|
+
by = {s: sum(1 for r in risks if r["severity"] == s) for s in ("high", "medium", "low")}
|
|
140
|
+
typer.echo(f"{len(inv['objects'])} database(s) read. Risks: {by['high']} high, {by['medium']} medium, {by['low']} low.")
|
|
141
|
+
if inv["warnings"]:
|
|
142
|
+
typer.echo(f"{len(inv['warnings'])} item(s) could not be collected (see inventory.md).")
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@app.command()
|
|
146
|
+
@_guarded
|
|
147
|
+
def ssrs(
|
|
148
|
+
server: Server = None, port: Port = 1433, database: Database = None, user: User = None, driver: Driver = "pymssql",
|
|
149
|
+
trusted: Trusted = False, encrypt: Encrypt = True, trust_server_certificate: TrustCert = False,
|
|
150
|
+
out: Out = Path("db-agent-sqlserver-output"), format: Format = "md,json,xlsx,html",
|
|
151
|
+
reportserver_db: Annotated[Optional[str], typer.Option("--reportserver-db", help="Name of the ReportServer database on --server.")] = None,
|
|
152
|
+
rdl_folder: Annotated[Optional[Path], typer.Option("--rdl-folder", help="Folder with .rdl (and .rsd, .rds) files.")] = None,
|
|
153
|
+
usage_csv: Annotated[Optional[Path], typer.Option("--usage-csv", help="Execution log exported to CSV (ExecutionLog3 columns).")] = None,
|
|
154
|
+
subscriptions_csv: Annotated[Optional[Path], typer.Option("--subscriptions-csv", help="Folder mode: subscriptions (ItemPath, DeliveryExtension, DataDriven).")] = None,
|
|
155
|
+
config: Annotated[Optional[Path], typer.Option("--config", help="Classification rules YAML (see src/db_agent/engines/sqlserver/data/classification.yaml).")] = None,
|
|
156
|
+
as_of: Annotated[Optional[str], typer.Option("--as-of", help="Count 90 and 365 days back from this date (YYYY-MM-DD). Default today.")] = None,
|
|
157
|
+
data_database: Annotated[Optional[str], typer.Option("--data-database", help="Database the reports query. Used to list the tables behind stored procedures.")] = None,
|
|
158
|
+
) -> None:
|
|
159
|
+
"""Inventory SSRS reports and classify each as move, review or retire."""
|
|
160
|
+
fmts = _formats(format)
|
|
161
|
+
if bool(reportserver_db) == bool(rdl_folder):
|
|
162
|
+
raise typer.BadParameter("give exactly one of --reportserver-db or --rdl-folder")
|
|
163
|
+
cfg_rules = classify.load_config(config)
|
|
164
|
+
when = datetime.strptime(as_of, "%Y-%m-%d") if as_of else None
|
|
165
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
166
|
+
usage = usage_mod.load_csv(usage_csv, when) if usage_csv else None
|
|
167
|
+
if rdl_folder:
|
|
168
|
+
if not rdl_folder.is_dir():
|
|
169
|
+
raise FileNotFoundError(f"folder not found: {rdl_folder}")
|
|
170
|
+
res = ssrs_mod.from_folder(rdl_folder, usage, cfg_rules, subscriptions_csv)
|
|
171
|
+
else:
|
|
172
|
+
scfg = _config(server, port, reportserver_db, user, driver, trusted, encrypt, trust_server_certificate)
|
|
173
|
+
with SafeConnection(scfg, QueryGuard()) as conn:
|
|
174
|
+
if usage is None:
|
|
175
|
+
try:
|
|
176
|
+
usage = usage_mod.load_executionlog(conn)
|
|
177
|
+
except DatabaseError as exc:
|
|
178
|
+
_say(f"Usage not read from ExecutionLog3 ({str(exc)[:120]}). Reports will not be classified retire.")
|
|
179
|
+
res = ssrs_mod.from_reportserver(conn, usage, cfg_rules)
|
|
180
|
+
if data_database and server:
|
|
181
|
+
_resolve_procs(res, _config(server, port, data_database, user, driver, trusted, encrypt, trust_server_certificate))
|
|
182
|
+
written = outputs.write_ssrs(out, res, fmts)
|
|
183
|
+
if "html" in fmts:
|
|
184
|
+
written += report_mod.write_report(out)
|
|
185
|
+
outputs.order_workbook(out)
|
|
186
|
+
_done(written, out)
|
|
187
|
+
c = res["classification_counts"]
|
|
188
|
+
typer.echo(f"{res['report_count']} report(s): move {c.get('move', 0)}, review {c.get('review', 0)}, retire {c.get('retire', 0)}.")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _resolve_procs(res: dict[str, Any], cfg: ConnectionConfig) -> None:
|
|
192
|
+
q = "SELECT DISTINCT d.referenced_schema_name AS s, d.referenced_entity_name AS n FROM sys.sql_expression_dependencies d WHERE d.referencing_id = OBJECT_ID(%s)"
|
|
193
|
+
with SafeConnection(cfg, QueryGuard()) as conn:
|
|
194
|
+
for r in res["reports"]:
|
|
195
|
+
for proc in r["procedures"]:
|
|
196
|
+
try:
|
|
197
|
+
rows = conn.query(q, (proc,))
|
|
198
|
+
except DatabaseError:
|
|
199
|
+
continue
|
|
200
|
+
found = [f"{x['s'] + '.' if x['s'] else ''}{x['n']}" for x in rows if x["n"]]
|
|
201
|
+
r["tables"] = sorted(set(r["tables"]) | set(found), key=str.lower)
|
|
202
|
+
from collections import Counter
|
|
203
|
+
|
|
204
|
+
res["table_usage"] = Counter(t for r in res["reports"] for t in r["tables"]).most_common()
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
@app.command()
|
|
208
|
+
@_guarded
|
|
209
|
+
def validate(
|
|
210
|
+
source: Annotated[str, typer.Option("--source", help="server=HOST,database=DB,user=U[,port=1433,driver=pymssql,trusted=no,encrypt=yes,trust_cert=no]. Password: DBAGENT_SOURCE_PASSWORD.")],
|
|
211
|
+
target: Annotated[str, typer.Option("--target", help="Same form. Password: DBAGENT_TARGET_PASSWORD.")],
|
|
212
|
+
reports: Annotated[Path, typer.Option("--reports", help="reports.csv written by the ssrs command (reports.json must be next to it).")],
|
|
213
|
+
params: Annotated[Optional[Path], typer.Option("--params", help="YAML with parameter values: defaults: {Name: value}; reports: {/path: {Name: value}}.")] = None,
|
|
214
|
+
allow_exec: Annotated[bool, typer.Option("--allow-exec", help="Also run datasets that call stored procedures.")] = False,
|
|
215
|
+
timeout: Annotated[int, typer.Option("--timeout", help="Seconds per query.")] = 60,
|
|
216
|
+
max_rows: Annotated[int, typer.Option("--max-rows", help="Skip a dataset that returns more rows than this.")] = 1_000_000,
|
|
217
|
+
out: Out = Path("db-agent-sqlserver-output"), format: Format = "md,json,xlsx,html",
|
|
218
|
+
) -> None:
|
|
219
|
+
"""Run each 'move' report's queries on the old and new server and compare counts and totals."""
|
|
220
|
+
fmts = _formats(format)
|
|
221
|
+
src, tgt = parse_conn_spec(source, "DBAGENT_SOURCE", "--source"), parse_conn_spec(target, "DBAGENT_TARGET", "--target")
|
|
222
|
+
src.query_timeout = tgt.query_timeout = timeout
|
|
223
|
+
sidecar = reports.with_name("reports.json")
|
|
224
|
+
if not sidecar.exists():
|
|
225
|
+
raise FileNotFoundError(f"{sidecar} not found. Run `ssrs` first; validate needs the dataset queries it saved.")
|
|
226
|
+
import json
|
|
227
|
+
|
|
228
|
+
data = json.loads(sidecar.read_text(encoding="utf-8"))
|
|
229
|
+
overrides = yaml.safe_load(params.read_text()) if params else {}
|
|
230
|
+
rows = validate_mod.validate_reports(data["reports"], src, tgt, overrides or {}, allow_exec, max_rows, _say)
|
|
231
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
232
|
+
written = outputs.write_validation(out, rows, fmts)
|
|
233
|
+
if (out / "inventory.json").exists() or (out / "reports.json").exists():
|
|
234
|
+
pass
|
|
235
|
+
if "html" in fmts and (out / "reports.json").exists():
|
|
236
|
+
written += report_mod.write_report(out)
|
|
237
|
+
outputs.order_workbook(out)
|
|
238
|
+
_done(written, out)
|
|
239
|
+
counts: dict[str, int] = {}
|
|
240
|
+
for r in rows:
|
|
241
|
+
counts[r["status"]] = counts.get(r["status"], 0) + 1
|
|
242
|
+
typer.echo("Validation: " + (", ".join(f"{k} {v}" for k, v in sorted(counts.items())) or "nothing to compare"))
|
|
243
|
+
if counts.get("fail"):
|
|
244
|
+
raise typer.Exit(2)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
@app.command()
|
|
248
|
+
@_guarded
|
|
249
|
+
def report(out: Out = Path("db-agent-sqlserver-output")) -> None:
|
|
250
|
+
"""Combine inventory, ssrs and validate outputs found in --out into summary.html and the workbook."""
|
|
251
|
+
if not out.is_dir() or not any((out / f).exists() for f in ("inventory.json", "reports.json", "validation.json")):
|
|
252
|
+
raise FileNotFoundError(f"nothing to report in {out}. Run inventory, ssrs or validate first.")
|
|
253
|
+
_done(report_mod.write_report(out), out)
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
@app.command()
|
|
257
|
+
def version() -> None:
|
|
258
|
+
"""Print the version and the SQL Server versions it supports."""
|
|
259
|
+
typer.echo(f"db-agent {__version__}")
|
|
260
|
+
typer.echo(f"Supported: {SUPPORTED_SQL_SERVER}")
|
|
261
|
+
typer.echo("Python " + sys.version.split()[0])
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
if __name__ == "__main__":
|
|
265
|
+
app()
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Default rules for classifying SSRS reports as move, review or retire.
|
|
2
|
+
# Copy this file, edit it, and pass it with --config.
|
|
3
|
+
|
|
4
|
+
# A report with no runs in this many days is "retire" ... but only if the usage log
|
|
5
|
+
# reaches back at least this far. SSRS keeps only 60 days of log by default, so a short
|
|
6
|
+
# log makes the answer "review", never "retire".
|
|
7
|
+
retire_after_days: 365
|
|
8
|
+
min_log_window_days: 365
|
|
9
|
+
|
|
10
|
+
# Fewer runs than this in the last 365 days sends a report to review (it may be retire later).
|
|
11
|
+
low_usage_runs_365: 6
|
|
12
|
+
|
|
13
|
+
# More than this share of runs ending in an error sends a report to review.
|
|
14
|
+
error_rate_review: 0.5
|
|
15
|
+
|
|
16
|
+
# More expressions than this (formulas that start with "=") sends a report to review.
|
|
17
|
+
complex_expression_threshold: 150
|
|
18
|
+
|
|
19
|
+
# Features that send a report to review. Set to false to ignore one.
|
|
20
|
+
review_if:
|
|
21
|
+
custom_code: true
|
|
22
|
+
custom_assemblies: true
|
|
23
|
+
subreports: true
|
|
24
|
+
drillthrough: true
|
|
25
|
+
complex_subscription: true
|
|
26
|
+
parse_failure: true
|
|
27
|
+
non_sql_data_source: true
|
|
28
|
+
unresolved_shared_dataset: true
|
|
29
|
+
stored_procedure_dataset: false
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""The only module that talks to a SQL Server driver.
|
|
2
|
+
|
|
3
|
+
Nothing else in the package may import pymssql or pyodbc (a test enforces this). Every
|
|
4
|
+
statement goes through SafeConnection.query() or .stream(), which call the QueryGuard first.
|
|
5
|
+
Passwords are held in memory only, hidden from repr(), and scrubbed from error messages.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any, Iterator
|
|
13
|
+
|
|
14
|
+
from .guard import QueryGuard
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class ConnectionConfig:
|
|
19
|
+
server: str
|
|
20
|
+
port: int = 1433
|
|
21
|
+
database: str = ""
|
|
22
|
+
user: str | None = None
|
|
23
|
+
password: str | None = field(default=None, repr=False)
|
|
24
|
+
driver: str = "pymssql" # pymssql | odbc
|
|
25
|
+
trusted: bool = False # Windows auth, odbc only
|
|
26
|
+
encrypt: bool = True
|
|
27
|
+
trust_server_certificate: bool = False
|
|
28
|
+
login_timeout: int = 15
|
|
29
|
+
query_timeout: int = 60
|
|
30
|
+
|
|
31
|
+
def with_database(self, database: str) -> "ConnectionConfig":
|
|
32
|
+
copy = ConnectionConfig(**{**self.__dict__})
|
|
33
|
+
copy.database = database
|
|
34
|
+
return copy
|
|
35
|
+
|
|
36
|
+
def __repr__(self) -> str: # never show the password
|
|
37
|
+
return (
|
|
38
|
+
f"ConnectionConfig(server={self.server!r}, port={self.port}, database={self.database!r}, "
|
|
39
|
+
f"user={self.user!r}, driver={self.driver!r}, trusted={self.trusted})"
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
ODBC_PREFERENCE = ("ODBC Driver 18 for SQL Server", "ODBC Driver 17 for SQL Server", "ODBC Driver 13.1 for SQL Server",
|
|
44
|
+
"ODBC Driver 13 for SQL Server", "ODBC Driver 11 for SQL Server")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def odbc_driver_name(installed: list[str]) -> str:
|
|
48
|
+
"""Newest installed Microsoft ODBC driver for SQL Server. Client machines often have only 17 or 13.
|
|
49
|
+
Override with DBAGENT_ODBC_DRIVER."""
|
|
50
|
+
import os
|
|
51
|
+
|
|
52
|
+
forced = os.environ.get("DBAGENT_ODBC_DRIVER")
|
|
53
|
+
if forced:
|
|
54
|
+
return forced
|
|
55
|
+
for name in ODBC_PREFERENCE:
|
|
56
|
+
if name in installed:
|
|
57
|
+
return name
|
|
58
|
+
raise DatabaseError("no Microsoft ODBC driver for SQL Server found; install ODBC Driver 18 (or 17)")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class DatabaseError(Exception):
|
|
62
|
+
"""A driver or connection error, with secrets removed from the message."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def scrub(text: str, secrets: list[str | None]) -> str:
|
|
66
|
+
for s in secrets:
|
|
67
|
+
if s:
|
|
68
|
+
text = text.replace(s, "***")
|
|
69
|
+
return text
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class SafeConnection:
|
|
73
|
+
def __init__(self, cfg: ConnectionConfig, guard: QueryGuard | None = None):
|
|
74
|
+
self.cfg = cfg
|
|
75
|
+
self.guard = guard or QueryGuard()
|
|
76
|
+
self._conn: Any = None
|
|
77
|
+
|
|
78
|
+
# -- connection ---------------------------------------------------------------------------
|
|
79
|
+
def _connect(self) -> Any:
|
|
80
|
+
cfg = self.cfg
|
|
81
|
+
try:
|
|
82
|
+
if cfg.driver == "pymssql":
|
|
83
|
+
import pymssql
|
|
84
|
+
|
|
85
|
+
kwargs: dict[str, Any] = dict(
|
|
86
|
+
server=cfg.server,
|
|
87
|
+
port=str(cfg.port),
|
|
88
|
+
user=cfg.user,
|
|
89
|
+
password=cfg.password,
|
|
90
|
+
database=cfg.database or "",
|
|
91
|
+
login_timeout=cfg.login_timeout,
|
|
92
|
+
timeout=cfg.query_timeout,
|
|
93
|
+
appname="db-agent",
|
|
94
|
+
read_only=True, # ApplicationIntent=ReadOnly
|
|
95
|
+
autocommit=False,
|
|
96
|
+
)
|
|
97
|
+
kwargs["encryption"] = "require" if cfg.encrypt else "off"
|
|
98
|
+
return pymssql.connect(**kwargs)
|
|
99
|
+
if cfg.driver == "odbc":
|
|
100
|
+
import pyodbc
|
|
101
|
+
|
|
102
|
+
parts = [
|
|
103
|
+
f"DRIVER={{{odbc_driver_name(pyodbc.drivers())}}}",
|
|
104
|
+
f"SERVER={cfg.server},{cfg.port}",
|
|
105
|
+
"ApplicationIntent=ReadOnly",
|
|
106
|
+
"APP=db-agent",
|
|
107
|
+
f"Encrypt={'yes' if cfg.encrypt else 'no'}",
|
|
108
|
+
f"TrustServerCertificate={'yes' if cfg.trust_server_certificate else 'no'}",
|
|
109
|
+
]
|
|
110
|
+
if cfg.database:
|
|
111
|
+
parts.append(f"DATABASE={cfg.database}")
|
|
112
|
+
if cfg.trusted:
|
|
113
|
+
parts.append("Trusted_Connection=yes")
|
|
114
|
+
else:
|
|
115
|
+
parts += [f"UID={cfg.user}", "PWD={" + str(cfg.password).replace("}", "}}") + "}"]
|
|
116
|
+
return pyodbc.connect(";".join(parts), timeout=cfg.login_timeout, autocommit=False)
|
|
117
|
+
except Exception as exc: # noqa: BLE001
|
|
118
|
+
msg = scrub(f"could not connect to {cfg.server}: {exc}", [cfg.password])
|
|
119
|
+
if "certificate" in msg.lower() and "trust" in msg.lower():
|
|
120
|
+
msg += " Hint: the server uses a certificate this machine does not trust. Add --trust-server-certificate (odbc driver) or --no-encrypt."
|
|
121
|
+
raise DatabaseError(msg) from None
|
|
122
|
+
raise DatabaseError(f"unknown driver {cfg.driver!r}")
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def conn(self) -> Any:
|
|
126
|
+
if self._conn is None:
|
|
127
|
+
self._conn = self._connect()
|
|
128
|
+
return self._conn
|
|
129
|
+
|
|
130
|
+
def close(self) -> None:
|
|
131
|
+
if self._conn is not None:
|
|
132
|
+
try:
|
|
133
|
+
self._conn.rollback()
|
|
134
|
+
self._conn.close()
|
|
135
|
+
finally:
|
|
136
|
+
self._conn = None
|
|
137
|
+
|
|
138
|
+
def __enter__(self) -> "SafeConnection":
|
|
139
|
+
return self
|
|
140
|
+
|
|
141
|
+
def __exit__(self, *exc: object) -> None:
|
|
142
|
+
self.close()
|
|
143
|
+
|
|
144
|
+
# -- queries ------------------------------------------------------------------------------
|
|
145
|
+
def _cursor(self, sql: str, params: Any) -> Any:
|
|
146
|
+
self.guard.check(sql) # raises GuardError before anything reaches the server
|
|
147
|
+
cur = self.conn.cursor()
|
|
148
|
+
try:
|
|
149
|
+
cur.execute(sql, params) if params is not None else cur.execute(sql)
|
|
150
|
+
except Exception as exc: # noqa: BLE001
|
|
151
|
+
cur.close()
|
|
152
|
+
self.conn.rollback()
|
|
153
|
+
raise DatabaseError(scrub(str(exc), [self.cfg.password])) from None
|
|
154
|
+
return cur
|
|
155
|
+
|
|
156
|
+
def query(self, sql: str, params: Any = None) -> list[dict[str, Any]]:
|
|
157
|
+
"""Run a guarded SELECT and return all rows as dicts. For metadata-sized results only."""
|
|
158
|
+
cur = self._cursor(sql, params)
|
|
159
|
+
try:
|
|
160
|
+
cols = [d[0] for d in cur.description or []]
|
|
161
|
+
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
|
162
|
+
finally:
|
|
163
|
+
cur.close()
|
|
164
|
+
self.conn.rollback()
|
|
165
|
+
|
|
166
|
+
def stream(self, sql: str, params: Any = None, chunk: int = 5000) -> Iterator[tuple[list[str], list[tuple]]]:
|
|
167
|
+
"""Run a guarded SELECT and yield (columns, rows) chunks. Used by validate so rows are
|
|
168
|
+
folded into counts and totals as they arrive and are never kept."""
|
|
169
|
+
cur = self._cursor(sql, params)
|
|
170
|
+
try:
|
|
171
|
+
cols = [d[0] for d in cur.description or []]
|
|
172
|
+
while True:
|
|
173
|
+
rows = cur.fetchmany(chunk)
|
|
174
|
+
if not rows:
|
|
175
|
+
break
|
|
176
|
+
yield cols, rows
|
|
177
|
+
finally:
|
|
178
|
+
cur.close()
|
|
179
|
+
self.conn.rollback()
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def config_from_env(prefix: str = "DBAGENT_SQL") -> dict[str, str | None]:
|
|
183
|
+
return {k: os.environ.get(f"{prefix}_{k.upper()}") for k in ("server", "port", "database", "user", "password")}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""The query guard. Every statement sent to any SQL Server passes through check().
|
|
2
|
+
|
|
3
|
+
Policy: one statement only. SELECT, or WITH ... SELECT, or EXEC of a stored procedure that
|
|
4
|
+
the caller explicitly allowed (validate --allow-exec). Anything else is refused, and so is
|
|
5
|
+
anything we cannot parse (fail closed).
|
|
6
|
+
|
|
7
|
+
Two independent checks must both pass:
|
|
8
|
+
1. A text check on a sanitized copy (strings, comments and quoted names blanked out).
|
|
9
|
+
2. A sqlglot parse (tsql dialect) with an AST walk.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
|
|
16
|
+
import sqlglot
|
|
17
|
+
from sqlglot import exp
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class GuardError(Exception):
|
|
21
|
+
"""The statement is not allowed. The message never contains the statement text."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
_DENY_WORDS = (
|
|
25
|
+
"INTO INSERT UPDATE DELETE MERGE DROP ALTER CREATE TRUNCATE GRANT REVOKE DENY "
|
|
26
|
+
"BACKUP RESTORE SHUTDOWN KILL DBCC RECONFIGURE USE SET DECLARE WAITFOR BULK "
|
|
27
|
+
"OPENROWSET OPENQUERY OPENDATASOURCE OPENXML READTEXT WRITETEXT UPDATETEXT "
|
|
28
|
+
"EXEC EXECUTE SP_EXECUTESQL BEGIN COMMIT ROLLBACK GOTO RETURN PRINT RAISERROR THROW"
|
|
29
|
+
).split()
|
|
30
|
+
_DENY_RE = re.compile(r"\b(" + "|".join(_DENY_WORDS) + r")\b", re.IGNORECASE)
|
|
31
|
+
_XP_RE = re.compile(r"\bxp_\w*", re.IGNORECASE)
|
|
32
|
+
_FOUR_PART_RE = re.compile(r"[\w@#$]+(?:\.[\w@#$]*){3,}")
|
|
33
|
+
_PLACEHOLDER_RE = re.compile(r"%\((\w+)\)s")
|
|
34
|
+
_GO_RE = re.compile(r"^\s*GO\s*$", re.IGNORECASE | re.MULTILINE)
|
|
35
|
+
|
|
36
|
+
_EXEC_RE = re.compile(
|
|
37
|
+
r"^\s*EXEC(?:UTE)?\s+(?P<proc>[\w\.\[\]]+)\s*(?P<args>.*?)\s*;?\s*$", re.IGNORECASE | re.DOTALL
|
|
38
|
+
)
|
|
39
|
+
_EXEC_ARGS_RE = re.compile(r"^(@\w+\s*=\s*%\(\w+\)s(\s*,\s*@\w+\s*=\s*%\(\w+\)s)*)?$")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _mask(segment: str) -> str:
|
|
43
|
+
return "".join(ch if ch == "\n" else " " for ch in segment)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def sanitize(sql: str, preserve_length: bool = False, keep_strings: bool = False) -> str:
|
|
47
|
+
"""Blank out string literals, quoted names and comments so keywords inside them are ignored.
|
|
48
|
+
|
|
49
|
+
Default mode: bracketed and double-quoted names become a plain word (_q_) so dotted chains
|
|
50
|
+
still count, strings become '' and comments a space.
|
|
51
|
+
preserve_length=True: every blanked character becomes a space (newlines kept), so offsets
|
|
52
|
+
and line numbers in the result match the original text.
|
|
53
|
+
"""
|
|
54
|
+
out: list[str] = []
|
|
55
|
+
i, n = 0, len(sql)
|
|
56
|
+
while i < n:
|
|
57
|
+
c = sql[i]
|
|
58
|
+
two = sql[i : i + 2]
|
|
59
|
+
start = i
|
|
60
|
+
if two == "/*":
|
|
61
|
+
depth = 1
|
|
62
|
+
i += 2
|
|
63
|
+
while i < n and depth:
|
|
64
|
+
if sql[i : i + 2] == "/*":
|
|
65
|
+
depth += 1
|
|
66
|
+
i += 2
|
|
67
|
+
elif sql[i : i + 2] == "*/":
|
|
68
|
+
depth -= 1
|
|
69
|
+
i += 2
|
|
70
|
+
else:
|
|
71
|
+
i += 1
|
|
72
|
+
out.append(_mask(sql[start:i]) if preserve_length else " ")
|
|
73
|
+
elif two == "--":
|
|
74
|
+
while i < n and sql[i] != "\n":
|
|
75
|
+
i += 1
|
|
76
|
+
out.append(_mask(sql[start:i]) if preserve_length else " ")
|
|
77
|
+
elif c == "'":
|
|
78
|
+
i += 1
|
|
79
|
+
while i < n:
|
|
80
|
+
if sql[i] == "'":
|
|
81
|
+
if sql[i : i + 2] == "''":
|
|
82
|
+
i += 2
|
|
83
|
+
continue
|
|
84
|
+
break
|
|
85
|
+
i += 1
|
|
86
|
+
i += 1
|
|
87
|
+
out.append(sql[start:i] if keep_strings else (_mask(sql[start:i]) if preserve_length else "''"))
|
|
88
|
+
elif c == "[":
|
|
89
|
+
i += 1
|
|
90
|
+
while i < n:
|
|
91
|
+
if sql[i] == "]":
|
|
92
|
+
if sql[i : i + 2] == "]]":
|
|
93
|
+
i += 2
|
|
94
|
+
continue
|
|
95
|
+
break
|
|
96
|
+
i += 1
|
|
97
|
+
i += 1
|
|
98
|
+
out.append(_mask(sql[start:i]) if preserve_length else "_q_")
|
|
99
|
+
elif c == '"':
|
|
100
|
+
i += 1
|
|
101
|
+
while i < n and sql[i] != '"':
|
|
102
|
+
i += 1
|
|
103
|
+
i += 1
|
|
104
|
+
out.append(_mask(sql[start:i]) if preserve_length else "_q_")
|
|
105
|
+
else:
|
|
106
|
+
out.append(c)
|
|
107
|
+
i += 1
|
|
108
|
+
return "".join(out)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _normalize_proc(name: str) -> str:
|
|
112
|
+
return name.replace("[", "").replace("]", "").strip().lower()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
_BAD_NODES = tuple(
|
|
116
|
+
getattr(exp, n)
|
|
117
|
+
for n in (
|
|
118
|
+
"Into", "Insert", "Update", "Delete", "Merge", "Create", "Drop", "Alter", "Command",
|
|
119
|
+
"Set", "Use", "TruncateTable", "Grant", "Revoke", "Transaction", "Commit", "Rollback",
|
|
120
|
+
"Declare", "Execute", "Copy", "LoadData", "Pragma",
|
|
121
|
+
)
|
|
122
|
+
if hasattr(exp, n)
|
|
123
|
+
)
|
|
124
|
+
_ROOT_OK = tuple(getattr(exp, n) for n in ("Select", "Union", "Subquery", "SetOperation") if hasattr(exp, n))
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class QueryGuard:
|
|
128
|
+
def __init__(self, allowed_procs: set[str] | None = None):
|
|
129
|
+
self.allowed_procs = {_normalize_proc(p) for p in (allowed_procs or set())}
|
|
130
|
+
|
|
131
|
+
def allow_proc(self, name: str) -> None:
|
|
132
|
+
self.allowed_procs.add(_normalize_proc(name))
|
|
133
|
+
|
|
134
|
+
def check(self, sql: str) -> None:
|
|
135
|
+
"""Raise GuardError unless sql is allowed. Returns None when allowed."""
|
|
136
|
+
if not isinstance(sql, str) or not sql.strip():
|
|
137
|
+
raise GuardError("empty statement")
|
|
138
|
+
first_word = re.match(r"\s*(\w+)", sql)
|
|
139
|
+
if first_word and first_word.group(1).upper() in ("EXEC", "EXECUTE"):
|
|
140
|
+
self._check_exec(sql)
|
|
141
|
+
return
|
|
142
|
+
# Bound parameters (%(name)s) are values the driver sends separately. Check the statement
|
|
143
|
+
# in its @name form, which the parser understands. '%%' is the driver's escape for '%'.
|
|
144
|
+
sql = sql.replace("%%", "\x00")
|
|
145
|
+
sql = _PLACEHOLDER_RE.sub(r"@\1", sql).replace("%s", "@p").replace("\x00", "%")
|
|
146
|
+
clean = sanitize(sql)
|
|
147
|
+
if _GO_RE.search(clean):
|
|
148
|
+
raise GuardError("batch separator GO is not allowed")
|
|
149
|
+
body = clean.strip().rstrip(";").strip()
|
|
150
|
+
if ";" in body:
|
|
151
|
+
raise GuardError("more than one statement")
|
|
152
|
+
first = re.match(r"\s*(\w+)", body)
|
|
153
|
+
word = first.group(1).upper() if first else ""
|
|
154
|
+
if word not in ("SELECT", "WITH"):
|
|
155
|
+
raise GuardError(f"only SELECT is allowed, got {word or 'unknown'}")
|
|
156
|
+
hit = _DENY_RE.search(body)
|
|
157
|
+
if hit:
|
|
158
|
+
raise GuardError(f"keyword not allowed: {hit.group(1).upper()}")
|
|
159
|
+
if _XP_RE.search(body):
|
|
160
|
+
raise GuardError("extended stored procedures are not allowed")
|
|
161
|
+
if _FOUR_PART_RE.search(body):
|
|
162
|
+
raise GuardError("four-part names (linked servers) are not allowed")
|
|
163
|
+
self._check_ast(sql)
|
|
164
|
+
|
|
165
|
+
def _check_ast(self, sql: str) -> None:
|
|
166
|
+
try:
|
|
167
|
+
trees = sqlglot.parse(sql, read="tsql")
|
|
168
|
+
except Exception as exc: # noqa: BLE001 - any parser failure means refuse
|
|
169
|
+
raise GuardError("could not verify the statement is read-only (parse failed)") from exc
|
|
170
|
+
trees = [t for t in trees if t is not None]
|
|
171
|
+
if len(trees) != 1:
|
|
172
|
+
raise GuardError("more than one statement")
|
|
173
|
+
tree = trees[0]
|
|
174
|
+
if not isinstance(tree, _ROOT_OK):
|
|
175
|
+
raise GuardError("statement is not a SELECT")
|
|
176
|
+
for node in tree.walk():
|
|
177
|
+
if isinstance(node, _BAD_NODES):
|
|
178
|
+
raise GuardError(f"statement part not allowed: {type(node).__name__}")
|
|
179
|
+
|
|
180
|
+
def _check_exec(self, sql: str) -> None:
|
|
181
|
+
m = _EXEC_RE.match(sql)
|
|
182
|
+
if not m:
|
|
183
|
+
raise GuardError("EXEC form not recognised")
|
|
184
|
+
proc = _normalize_proc(m.group("proc"))
|
|
185
|
+
if proc not in self.allowed_procs:
|
|
186
|
+
raise GuardError("stored procedure is not on the allowed list")
|
|
187
|
+
if not _EXEC_ARGS_RE.match(m.group("args").strip()):
|
|
188
|
+
raise GuardError("EXEC arguments must be named parameters (@name=%(name)s)")
|
|
189
|
+
if _XP_RE.search(proc) or proc.startswith("xp_") or ".xp_" in proc:
|
|
190
|
+
raise GuardError("extended stored procedures are not allowed")
|