db-agent-kit 0.2.0__py3-none-any.whl

db_agent/engines/databricks/health.py

@@ -0,0 +1,156 @@
+"""Collect, run the checks, assemble the Report. Each collection step fails on its own: a missing permission
+on one system table never hides the other results; it shows up under "not collected"."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import date, datetime, timezone
+from typing import Callable
+
+from db_agent import __version__
+
+from . import checks, cost as cost_mod, queries, runtimes
+from .model import Finding, Report
+
+
+@dataclass
+class Options:
+    today: date
+    lookback_days: int = 30
+    cost_from: str | None = None
+    cost_to: str | None = None
+    eos_warning_days: int = 180
+    all_workspaces: bool = False
+
+
+def _num(v):
+    return None if v is None else float(v)
+
+
+def run(reader, opt: Options) -> Report:
+    rep = Report(meta={"tool": f"db-agent {__version__}", "generated_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+                       "lookback_days": opt.lookback_days, "runtime_table_source": runtimes.SOURCE,
+                       "prices": "list price (system.billing.list_prices), before discounts"})
+
+    def step(name: str, fn: Callable, default=None):
+        try:
+            return fn()
+        except Exception as e:   # noqa: BLE001 - every failure is reported, none is fatal
+            rep.not_collected.append({"check": name, "reason": f"{type(e).__name__}: {str(e)[:300]}"})
+            return default
+
+    wid = step("workspace id", reader.workspace_id)
+    rep.meta["workspace_id"] = wid
+    rep.meta["host"] = step("host", reader.host)
+    ws_filter = None if opt.all_workspaces else wid
+    if not opt.all_workspaces and wid is None:
+        rep.not_collected.append({"check": "system table queries", "reason": "workspace id unknown; pass --all-workspaces to include every workspace in the account"})
+    sql_ok = reader.has_sql and (opt.all_workspaces or wid is not None)
+    rep.meta["warehouse_id"] = reader.warehouse_id if sql_ok else None
+    if not reader.has_sql:
+        why = "skipped by --no-sql" if getattr(reader, "sql_skipped", False) else "no SQL warehouse is running; pass --warehouse-id (a stopped one will start and bill)"
+        rep.not_collected.append({"check": "system table queries", "reason": why})
+
+    clusters = step("clusters (REST)", reader.clusters, [])
+    jobs = step("jobs (REST)", reader.jobs, [])
+    warehouses = step("SQL warehouses (REST)", reader.warehouses, [])
+    rep.meta["counts"] = {"clusters": len(clusters), "jobs": len(jobs), "warehouses": len(warehouses)}
+
+    # 1. runtimes
+    rep.findings += checks.runtime_findings(clusters, jobs, opt.today, opt.eos_warning_days)
+
+    # SQL-backed data
+    def q(name, sql):
+        return step(name, lambda: reader.sql(sql), None) if sql_ok else None
+
+    cluster_spend = q("spend by cluster", queries.spend_by_cluster(opt.lookback_days, ws_filter))
+    cluster_spend = {r["cluster_id"]: r for r in cluster_spend} if cluster_spend is not None else None
+    wh_spend = q("spend by warehouse", queries.spend_by_warehouse(opt.lookback_days, ws_filter))
+    wh_spend = {r["warehouse_id"]: r for r in wh_spend} if wh_spend is not None else None
+    wh_queries = q("queries by warehouse (system.query.history)", queries.queries_by_warehouse(opt.lookback_days, ws_filter))
+    wh_queries = {r["warehouse_id"]: r for r in wh_queries} if wh_queries is not None else None
+    untagged = q("untagged spend", queries.untagged_spend(opt.lookback_days, ws_filter))
+    mounts = q("DBFS mount events (system.access.audit)", queries.dbfs_mount_events(365, ws_filter))
+
+    hms = None
+    if sql_ok:
+        def hms_counts():
+            try:
+                schemas = [list(r.values())[0] for r in reader.sql(queries.SHOW_SCHEMAS)]
+            except Exception as e:   # noqa: BLE001
+                if "HIVE_METASTORE_DISABLED" in str(e):
+                    rep.findings.append(Finding("legacy.hive_metastore_disabled", "info", "Hive metastore is disabled in this workspace",
+                                                "Legacy access to hive_metastore is turned off, so there are no Hive metastore tables to migrate here.", [], ""))
+                    return {}
+                raise
+            if len(schemas) > 300:
+                rep.not_collected.append({"check": "Hive metastore tables", "reason": f"{len(schemas)} schemas; counted the first 300"})
+                schemas = schemas[:300]
+            out = {}
+            for s in schemas:
+                if s == "information_schema":
+                    continue
+                rows = reader.sql(queries.show_tables(s))
+                out[s] = sum(1 for r in rows if str(r.get("isTemporary", "false")).lower() != "true")
+            return out
+        hms = step("Hive metastore tables (SHOW TABLES)", hms_counts, None)
+
+    # 2. legacy dependencies
+    rep.findings += checks.hms_findings(hms)
+    rep.findings += checks.mount_findings(mounts, 365)
+    rep.findings += checks.init_script_findings(clusters, jobs)
+    rep.findings += checks.isolation_findings(clusters, jobs)
+
+    # 3. cost over time
+    if sql_ok:
+        a, b = (opt.cost_from, opt.cost_to) if opt.cost_from and opt.cost_to else cost_mod.default_range(opt.today)
+        def cost_step():
+            months = cost_mod.month_list(a, b)
+            last = months[-1]
+            end = date(last.year + (last.month == 12), last.month % 12 + 1, 1)
+            return cost_mod.analyze(reader.sql(queries.cost_by_month(months[0], end, ws_filter)), opt.today)
+        res = step("DBU spend by month", cost_step, None)
+        if res is not None:
+            rep.cost = res
+            rep.findings.append(_cost_finding(res, a, b))
+
+    # 4. waste
+    rep.findings += checks.jobs_on_all_purpose(jobs, clusters, cluster_spend, opt.lookback_days)
+    rep.findings += checks.no_autotermination(clusters, cluster_spend, opt.lookback_days)
+    rep.findings += checks.warehouse_findings(warehouses, wh_spend, wh_queries, opt.lookback_days)
+    rep.findings += checks.untagged_findings(untagged, opt.lookback_days)
+
+    order = {"high": 0, "medium": 1, "low": 2, "info": 3}
+    rep.findings.sort(key=lambda f: order[f.severity])
+    return rep
+
+
+def _usd(x: float, signed: bool = False) -> str:
+    s = f"{abs(x):,.2f}" if abs(x) < 100 else f"{abs(x):,.0f}"
+    sign = ("+" if x >= 0 else "-") if signed else ("-" if x < 0 else "")
+    return f"{sign}${s}"
+
+
+def _cost_finding(res: dict, a: str, b: str) -> Finding:
+    ms = res["months"]
+    if not ms:
+        return Finding("cost.mom", "info", f"No DBU usage found for {a} to {b}",
+                       "System tables have no rows for these months in this workspace. They may not have been enabled yet.", [], "")
+    lines = []
+    for m in ms:
+        rate = f"{_usd(m['rate'])}/DBU" if m["rate"] else "n/a"
+        lines.append(f"{m['month']}: {m['dbus']:,.0f} DBU, {_usd(m['cost'])} ({m['days']} days of data{', partial month' if m['partial'] else ''}), "
+                     f"{_usd(m['cost_per_day'])}/day, {rate}, tier {m['dominant_tier']}")
+    last = ms[-1].get("vs_first")
+    sev, extra = "info", ""
+    if last and last["pct"] is not None:
+        extra = (f" Against {ms[0]['month']}, cost per day moved {last['pct']:+.0f}% ({_usd(last['cost_per_day_change'], True)}/day): "
+                 f"{_usd(last['volume_effect_per_day'], True)}/day from usage (volume and SKU mix), "
+                 f"{_usd(last['price_effect_per_day'], True)}/day from list price per DBU.")
+        if abs(last["pct"]) >= 20:
+            sev = "medium"
+    if ms[0]["days"] < 15 and len(ms) > 1:
+        extra += f" Weak baseline: {ms[0]['month']} has only {ms[0]['days']} days of data, so the percentage change overstates the trend."
+    if res["tier_changed"]:
+        extra += " The SKU tier changed during this period: " + " then ".join(f"{m}={t}" for m, t in res["tier_sequence"]) + "."
+    return Finding("cost.mom", sev, f"DBU spend {a} to {b}", "; ".join(lines) + "." + extra, [], "")

db_agent/engines/databricks/model.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+SEVERITIES = ("high", "medium", "low", "info")
+
+
+@dataclass
+class Finding:
+    id: str
+    severity: str
+    title: str
+    detail: str
+    items: list[dict] = field(default_factory=list)
+    fix: str = ""
+
+
+@dataclass
+class Report:
+    meta: dict
+    findings: list[Finding] = field(default_factory=list)
+    cost: dict | None = None
+    not_collected: list[dict] = field(default_factory=list)
+
+    def counts(self) -> dict[str, int]:
+        c = {s: 0 for s in SEVERITIES}
+        for f in self.findings:
+            c[f.severity] += 1
+        return c

db_agent/engines/databricks/queries.py

@@ -0,0 +1,98 @@
+"""All SQL the health check sends. Each statement starts with a marker comment, which also makes the tool's
+queries easy to find in query history. Values that go into SQL are validated here, never passed through raw."""
+
+from __future__ import annotations
+
+import re
+from datetime import date
+
+_ID = re.compile(r"^\d+$")
+
+# DBU list price in force when the usage record ended. List price: before any negotiated discount.
+_PRICED = """
+FROM system.billing.usage u
+LEFT JOIN system.billing.list_prices p
+  ON p.sku_name = u.sku_name AND p.usage_unit = u.usage_unit
+ AND u.usage_end_time >= p.price_start_time
+ AND (p.price_end_time IS NULL OR u.usage_end_time < p.price_end_time)
+"""
+_COST = "u.usage_quantity * p.pricing.effective_list.default"
+
+
+def _ws(workspace_id: str | None) -> str:
+    if workspace_id is None:
+        return ""
+    if not _ID.match(workspace_id):
+        raise ValueError("workspace id must be numeric")
+    return f"AND u.workspace_id = '{workspace_id}'"
+
+
+def cost_by_month(start: date, end_exclusive: date, workspace_id: str | None) -> str:
+    return f"""/* dbagent:cost_by_month */
+SELECT date_trunc('month', u.usage_date) AS month, u.sku_name,
+       sum(u.usage_quantity) AS dbus, sum({_COST}) AS cost, count(DISTINCT u.usage_date) AS days
+{_PRICED}
+WHERE u.usage_date >= DATE'{start.isoformat()}' AND u.usage_date < DATE'{end_exclusive.isoformat()}'
+  AND u.usage_unit = 'DBU' {_ws(workspace_id)}
+GROUP BY 1, 2 ORDER BY 1, 2"""
+
+
+def spend_by_cluster(days: int, workspace_id: str | None) -> str:
+    return f"""/* dbagent:spend_by_cluster */
+SELECT u.usage_metadata.cluster_id AS cluster_id, sum(u.usage_quantity) AS dbus, sum({_COST}) AS cost
+{_PRICED}
+WHERE u.usage_date >= date_sub(current_date(), {int(days)}) AND u.usage_metadata.cluster_id IS NOT NULL
+  AND u.billing_origin_product IN ('ALL_PURPOSE', 'INTERACTIVE') AND u.usage_unit = 'DBU' {_ws(workspace_id)}
+GROUP BY 1"""
+
+
+def spend_by_warehouse(days: int, workspace_id: str | None) -> str:
+    return f"""/* dbagent:spend_by_warehouse */
+SELECT u.usage_metadata.warehouse_id AS warehouse_id, sum(u.usage_quantity) AS dbus, sum({_COST}) AS cost
+{_PRICED}
+WHERE u.usage_date >= date_sub(current_date(), {int(days)}) AND u.usage_metadata.warehouse_id IS NOT NULL
+  AND u.usage_unit = 'DBU' {_ws(workspace_id)}
+GROUP BY 1"""
+
+
+def queries_by_warehouse(days: int, workspace_id: str | None) -> str:
+    w = f"AND workspace_id = '{workspace_id}'" if workspace_id else ""
+    if workspace_id and not _ID.match(workspace_id):
+        raise ValueError("workspace id must be numeric")
+    return f"""/* dbagent:queries_by_warehouse */
+SELECT compute.warehouse_id AS warehouse_id, count(*) AS queries, max(start_time) AS last_query
+FROM system.query.history
+WHERE start_time >= current_timestamp() - INTERVAL {int(days)} DAYS AND compute.type = 'WAREHOUSE' {w}
+GROUP BY 1"""
+
+
+def untagged_spend(days: int, workspace_id: str | None) -> str:
+    return f"""/* dbagent:untagged_spend */
+SELECT u.billing_origin_product AS product,
+       sum({_COST}) AS cost,
+       sum(CASE WHEN u.custom_tags IS NULL OR cardinality(u.custom_tags) = 0 THEN {_COST} END) AS untagged_cost,
+       sum(u.usage_quantity) AS dbus,
+       sum(CASE WHEN u.custom_tags IS NULL OR cardinality(u.custom_tags) = 0 THEN u.usage_quantity END) AS untagged_dbus
+{_PRICED}
+WHERE u.usage_date >= date_sub(current_date(), {int(days)}) AND u.usage_unit = 'DBU' {_ws(workspace_id)}
+GROUP BY 1 ORDER BY 3 DESC"""
+
+
+def dbfs_mount_events(days: int, workspace_id: str | None) -> str:
+    w = f"AND workspace_id = '{workspace_id}'" if workspace_id else ""
+    if workspace_id and not _ID.match(workspace_id):
+        raise ValueError("workspace id must be numeric")
+    return f"""/* dbagent:dbfs_mount_events */
+SELECT event_date, user_identity.email AS user_email, to_json(request_params) AS params
+FROM system.access.audit
+WHERE service_name = 'dbfs' AND action_name = 'mount' AND event_date >= date_sub(current_date(), {int(days)}) {w}
+ORDER BY event_date DESC LIMIT 500"""
+
+
+SHOW_SCHEMAS = "/* dbagent:hms_schemas */ SHOW SCHEMAS IN hive_metastore"
+
+
+def show_tables(schema: str) -> str:
+    if not re.match(r"^[\w\-]+$", schema):
+        raise ValueError(f"odd schema name {schema!r}")
+    return f"/* dbagent:hms_tables */ SHOW TABLES IN hive_metastore.`{schema}`"

db_agent/engines/databricks/render.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from .model import Report
+
+_GROUPS = [("runtime.", "Runtimes"), ("legacy.", "Legacy dependencies"), ("cost.", "Cost over time"), ("waste.", "Spend waste")]
+
+
+def to_json(rep: Report) -> str:
+    return json.dumps({"meta": rep.meta, "counts": rep.counts(), "cost": rep.cost,
+                       "findings": [f.__dict__ for f in rep.findings], "not_collected": rep.not_collected},
+                      indent=2, default=str)
+
+
+def _table(items: list[dict], limit: int = 15) -> list[str]:
+    if not items:
+        return []
+    cols = list(items[0].keys())
+    out = ["", "| " + " | ".join(cols) + " |", "|" + "---|" * len(cols)]
+    for r in items[:limit]:
+        out.append("| " + " | ".join(str(r.get(c, "")).replace("|", "/") for c in cols) + " |")
+    if len(items) > limit:
+        out.append(f"\n{len(items) - limit} more in the JSON file.")
+    return out
+
+
+def to_markdown(rep: Report) -> str:
+    c = rep.counts()
+    m = rep.meta
+    lines = ["# Databricks workspace health", "",
+             f"Workspace {m.get('workspace_id')} at {m.get('host')}. Generated {m.get('generated_utc')} by {m.get('tool')}. "
+             f"Read-only. Spend figures are list price. Lookback {m.get('lookback_days')} days.", "",
+             f"**{c['high']} high, {c['medium']} medium, {c['low']} low, {c['info']} info.**", ""]
+    for prefix, title in _GROUPS:
+        fs = [f for f in rep.findings if f.id.startswith(prefix)]
+        if not fs:
+            continue
+        lines += [f"## {title}", ""]
+        for f in fs:
+            lines += [f"### [{f.severity.upper()}] {f.title}", "", f.detail]
+            if f.fix:
+                lines += ["", f"Fix: {f.fix}"]
+            lines += _table(f.items)
+            lines.append("")
+    if rep.cost and rep.cost["months"]:
+        lines += ["### Monthly figures", "", "| month | DBU | list $ | days | $/day | $/DBU | tier |", "|---|---|---|---|---|---|---|"]
+        for x in rep.cost["months"]:
+            lines.append(f"| {x['month']}{' (partial)' if x['partial'] else ''} | {x['dbus']:,.0f} | {x['cost']:,.0f} | {x['days']} | "
+                         f"{x['cost_per_day']:,.0f} | {(('%.3f' % x['rate']) if x['rate'] else 'n/a')} | {x['dominant_tier']} |")
+        lines.append("")
+    if rep.not_collected:
+        lines += ["## Not collected", "", "These checks did not run. Results above do not cover them.", ""]
+        lines += [f"- {n['check']}: {n['reason']}" for n in rep.not_collected]
+    return "\n".join(lines) + "\n"
+
+
+def write(rep: Report, out: Path, formats: set[str]) -> list[Path]:
+    out.mkdir(parents=True, exist_ok=True)
+    paths = []
+    if "json" in formats:
+        paths.append(out / "databricks-health.json")
+        paths[-1].write_text(to_json(rep))
+    if "md" in formats:
+        paths.append(out / "databricks-health.md")
+        paths[-1].write_text(to_markdown(rep))
+    return paths
+
+
+def console(rep: Report) -> str:
+    c = rep.counts()
+    lines = [f"Databricks health: {c['high']} high, {c['medium']} medium, {c['low']} low, {c['info']} info", ""]
+    for f in rep.findings:
+        lines.append(f"  [{f.severity.upper():<6}] {f.title}")
+    if rep.not_collected:
+        lines += ["", f"Not collected ({len(rep.not_collected)}):"] + [f"  - {n['check']}: {n['reason'][:110]}" for n in rep.not_collected]
+    return "\n".join(lines)

db_agent/engines/databricks/runtimes.py

@@ -0,0 +1,58 @@
+"""Databricks Runtime support dates.
+
+Source: https://docs.databricks.com/aws/en/release-notes/runtime/eos (ended versions) and
+https://docs.databricks.com/aws/en/release-notes/runtime/ (supported versions), read on 2026-11-07.
+Update EOS when Databricks publishes new versions. Versions newer than the newest one listed here are
+reported as "unknown", never as supported.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import date
+
+EOS: dict[str, date] = {
+    "10.5": date(2022, 11, 4), "11.0": date(2022, 12, 16), "11.1": date(2023, 1, 27), "11.2": date(2023, 3, 7),
+    "11.3": date(2025, 10, 19), "12.0": date(2023, 6, 15), "12.1": date(2023, 7, 18), "12.2": date(2026, 3, 1),
+    "13.0": date(2023, 10, 14), "13.1": date(2023, 12, 1), "13.2": date(2024, 1, 6), "13.3": date(2026, 8, 22),
+    "14.0": date(2024, 3, 11), "14.1": date(2025, 2, 12), "14.2": date(2024, 11, 5), "14.3": date(2027, 2, 1),
+    "15.0": date(2024, 5, 31), "15.1": date(2024, 10, 30), "15.2": date(2025, 1, 7), "15.3": date(2025, 1, 7),
+    "15.4": date(2027, 8, 19), "16.0": date(2025, 5, 11), "16.1": date(2025, 6, 20), "16.2": date(2025, 8, 5),
+    "16.3": date(2025, 9, 30), "16.4": date(2028, 5, 9), "17.0": date(2025, 12, 23), "17.1": date(2026, 2, 1),
+    "17.2": date(2026, 3, 16), "17.3": date(2028, 10, 22), "18.0": date(2026, 7, 15), "18.1": date(2026, 9, 11),
+    "18.2": date(2026, 11, 5),
+}
+LTS = {"11.3", "12.2", "13.3", "14.3", "15.4", "16.4", "17.3"}
+NEWEST_KNOWN = (18, 2)
+SOURCE = "https://docs.databricks.com/aws/en/release-notes/runtime/eos"
+
+_VER = re.compile(r"^(\d+)\.(\d+)")
+
+
+@dataclass(frozen=True)
+class RuntimeStatus:
+    spark_version: str
+    label: str            # "13.3 LTS"
+    eos: date | None
+    state: str            # ended | ending_soon | supported | unknown
+    days_left: int | None
+
+
+def classify(spark_version: str | None, today: date, warn_days: int = 180) -> RuntimeStatus:
+    sv = spark_version or ""
+    if sv.startswith("apache-spark-"):
+        return RuntimeStatus(sv, sv, None, "ended", None)   # Spark 2.x / 3.0 era images
+    m = _VER.match(sv)
+    if not m:
+        return RuntimeStatus(sv, sv or "unknown", None, "unknown", None)   # custom images, empty
+    key = f"{int(m.group(1))}.{int(m.group(2))}"
+    label = key + (" LTS" if key in LTS else "")
+    eos = EOS.get(key)
+    if eos is None:
+        if (int(m.group(1)), int(m.group(2))) < (10, 5):
+            return RuntimeStatus(sv, label, None, "ended", None)   # 10.4 LTS and older are end-of-life
+        return RuntimeStatus(sv, label, None, "unknown", None)     # newer than this table: check the docs
+    left = (eos - today).days
+    state = "ended" if left < 0 else "ending_soon" if left <= warn_days else "supported"
+    return RuntimeStatus(sv, label, eos, state, left)

db_agent/engines/databricks/sqlguard.py

@@ -0,0 +1,80 @@
+"""Every statement sent to a SQL warehouse goes through here first.
+
+Allowed: one SELECT/WITH statement, SHOW SCHEMAS IN hive_metastore, SHOW TABLES IN hive_metastore.<schema>.
+Anything else is refused, including statements that mention a write keyword anywhere outside literals.
+"""
+
+from __future__ import annotations
+
+import re
+
+
+class GuardError(Exception):
+    pass
+
+
+_FORBIDDEN = re.compile(
+    r"\b(insert|update|delete|merge|drop|create|alter|truncate|grant|revoke|deny|copy|optimize|vacuum|call|set|use|put|"
+    r"msck|repair|refresh|analyze|cache|uncache|load|restore|undrop|overwrite|into|exec|execute)\b", re.I)
+_SHOW = re.compile(r"^show\s+(schemas\s+in\s+hive_metastore|tables\s+in\s+hive_metastore\.`?[\w\-]+`?)$", re.I)
+
+
+def _strip(sql: str) -> str:
+    """Remove comments, string literals and quoted identifiers so keywords inside them do not count."""
+    out, i, n = [], 0, len(sql)
+    while i < n:
+        c = sql[i]
+        if sql.startswith("--", i):
+            j = sql.find("\n", i)
+            i = n if j < 0 else j
+        elif sql.startswith("/*", i):
+            j = sql.find("*/", i + 2)
+            if j < 0:
+                raise GuardError("unterminated comment")
+            i = j + 2
+        elif c in ("'", '"', "`"):
+            j = i + 1
+            while j < n:
+                if sql[j] == "\\" and c != "`":
+                    j += 2
+                    continue
+                if sql[j] == c:
+                    if j + 1 < n and sql[j + 1] == c:   # doubled quote
+                        j += 2
+                        continue
+                    break
+                j += 1
+            else:
+                raise GuardError("unterminated quote")
+            out.append(" ")
+            if c == "`":
+                out[-1] = "x"   # keep backtick identifiers non-empty, so SHOW ... `schema` still matches
+            i = j + 1
+        else:
+            out.append(c)
+            i += 1
+    return "".join(out)
+
+
+def check(sql: str) -> None:
+    body = _strip(sql).strip().rstrip(";").strip()
+    if not body:
+        raise GuardError("empty statement")
+    if ";" in body:
+        raise GuardError("only one statement is allowed")
+    flat = re.sub(r"\s+", " ", body)
+    if _SHOW.match(sql_without_comments(sql)):
+        return
+    first = flat.split(" ", 1)[0].lower()
+    if first not in ("select", "with"):
+        raise GuardError(f"only SELECT/WITH and SHOW ... IN hive_metastore are allowed, got {first.upper()}")
+    bad = _FORBIDDEN.search(flat)
+    if bad:
+        raise GuardError(f"refused: {bad.group(0).upper()} is not allowed")
+
+
+def sql_without_comments(sql: str) -> str:
+    """Comments removed but literals and identifiers kept, for the exact SHOW forms."""
+    s = re.sub(r"/\*.*?\*/", " ", sql, flags=re.S)
+    s = re.sub(r"--[^\n]*", " ", s)
+    return re.sub(r"\s+", " ", s).strip().rstrip(";").strip()

db_agent/engines/sqlserver/__init__.py

@@ -0,0 +1,11 @@
+"""db-agent: read-only inventory of SQL Server and SSRS reports."""
+
+from db_agent import __version__  # noqa: F401  (one version for the whole CLI)
+
+SUPPORTED_SQL_SERVER = "SQL Server 2016 to 2022 (read), Azure SQL Database and Managed Instance (read)"
+
+# Verified against Microsoft's lifecycle pages on 2026-11-05.
+END_OF_SUPPORT = {
+    13: ("SQL Server 2016", "2026-07-14"),
+    14: ("SQL Server 2017", "2027-10-12"),
+}

db_agent/engines/sqlserver/classify.py

@@ -0,0 +1,102 @@
+"""Classify each report as move, review or retire, with the reasons."""
+
+from __future__ import annotations
+
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .rdl import layout_hint, non_sql_sources
+from .usage import ReportUsage, UsageData
+
+
+def load_config(path: Path | None = None) -> dict[str, Any]:
+    default = yaml.safe_load(resources.files("db_agent.engines.sqlserver").joinpath("data/classification.yaml").read_text())
+    if path:
+        user = yaml.safe_load(Path(path).read_text()) or {}
+        review = {**default.get("review_if", {}), **user.get("review_if", {})}
+        default.update(user)
+        default["review_if"] = review
+    return default
+
+
+def classify(
+    report: Any,
+    usage: UsageData | None,
+    cfg: dict[str, Any],
+    subscriptions: dict[str, Any] | None = None,
+    unresolved_shared: bool = False,
+    sql_parse_failed: list[str] | None = None,
+) -> dict[str, Any]:
+    """Return {classification, reasons, usage_known, layout_hint, ...} for one parsed report."""
+    reasons: list[str] = []
+    rv = cfg["review_if"]
+    u: ReportUsage | None = None
+    usage_known = usage is not None and usage.source in ("csv", "executionlog3")
+    if usage_known:
+        u = usage.get(report.path)
+
+    # 1. Retire, only when the log is long enough to prove it.
+    if usage_known and u is not None and u.runs_365 == 0:
+        if usage.window_days >= cfg["min_log_window_days"]:
+            return _result(report, usage, u, "retire", [f"No runs in the last {cfg['retire_after_days']} days (log covers {usage.window_days} days)"])
+        reasons.append(
+            f"No runs in the log, but the log covers only {usage.window_days} days, so it cannot show the report is unused. "
+            f"Raise ExecutionLogDaysKept or supply a longer log"
+        )
+
+    # 2. Review reasons.
+    if report.parse_error and rv.get("parse_failure", True):
+        reasons.append(f"Report definition could not be parsed ({report.parse_error})")
+    if sql_parse_failed and rv.get("parse_failure", True):
+        reasons.append(f"Dataset SQL could not be fully parsed ({', '.join(sql_parse_failed)}); table list is a guess")
+    if report.has_custom_code and rv.get("custom_code", True):
+        reasons.append(f"Custom code block ({report.custom_code_lines} lines)")
+    if report.custom_assemblies and rv.get("custom_assemblies", True):
+        reasons.append("Uses custom assemblies or classes")
+    if report.subreports and rv.get("subreports", True):
+        reasons.append(f"Subreports: {', '.join(report.subreports)}")
+    if report.drillthrough and rv.get("drillthrough", True):
+        reasons.append(f"Drillthrough to: {', '.join(sorted(set(report.drillthrough)))}")
+    subs = (subscriptions or {}).get(report.path.lower())
+    if subs and rv.get("complex_subscription", True) and (subs.get("data_driven") or subs.get("non_email")):
+        kinds = [k for k, v in (("data-driven", subs.get("data_driven")), ("non-email delivery", subs.get("non_email"))) if v]
+        reasons.append(f"Subscriptions: {subs['count']} ({', '.join(kinds)})")
+    ns = non_sql_sources(report)
+    if ns and rv.get("non_sql_data_source", True):
+        reasons.append(f"Non-SQL data source: {', '.join(ns)}")
+    if unresolved_shared and rv.get("unresolved_shared_dataset", True):
+        reasons.append("Uses a shared dataset or data source that was not supplied")
+    if report.expressions > cfg["complex_expression_threshold"]:
+        reasons.append(f"{report.expressions} expressions (threshold {cfg['complex_expression_threshold']})")
+    if rv.get("stored_procedure_dataset", False) and any(d.command_type == "StoredProcedure" for d in report.datasets):
+        reasons.append("Dataset calls a stored procedure")
+    if usage_known and u is not None and 0 < u.runs_365 < cfg["low_usage_runs_365"]:
+        reasons.append(f"Low usage: {u.runs_365} runs in 365 days")
+    if usage_known and u is not None and u.runs_365 and u.errors / max(u.runs_365, 1) > cfg["error_rate_review"]:
+        reasons.append(f"{u.errors} of {u.runs_365} runs failed")
+
+    label = "review" if reasons else "move"
+    if label == "move":
+        reasons = ["Plain report, no hard-to-move features" + ("" if usage_known else "; usage unknown")]
+    return _result(report, usage, u, label, reasons, subs)
+
+
+def _result(report: Any, usage: UsageData | None, u: ReportUsage | None, label: str, reasons: list[str], subs: dict | None = None) -> dict[str, Any]:
+    usage_known = usage is not None
+    return {
+        "classification": label,
+        "reasons": reasons,
+        "usage_known": usage_known,
+        "log_window_days": usage.window_days if usage else None,
+        "layout_hint": layout_hint(report),
+        "last_run": u.last_run.strftime("%Y-%m-%d") if (u and u.last_run) else None,
+        "runs_90": u.runs_90 if u else None,
+        "runs_365": u.runs_365 if u else None,
+        "distinct_users": u.distinct_users if u else None,
+        "avg_ms": u.avg_ms if u else None,
+        "errors": u.errors if u else None,
+        "subscriptions": (subs or {}).get("count", 0),
+    }