sqlquality 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlquality/__init__.py +3 -0
- sqlquality/adapters/__init__.py +20 -0
- sqlquality/adapters/base.py +21 -0
- sqlquality/adapters/postgres.py +60 -0
- sqlquality/adapters/redshift.py +46 -0
- sqlquality/antipatterns.py +174 -0
- sqlquality/changeset.py +93 -0
- sqlquality/cli.py +540 -0
- sqlquality/complexity.py +41 -0
- sqlquality/config.py +86 -0
- sqlquality/dbtproject.py +126 -0
- sqlquality/delta.py +76 -0
- sqlquality/dialects.py +23 -0
- sqlquality/gate.py +37 -0
- sqlquality/keys.py +106 -0
- sqlquality/linter.py +70 -0
- sqlquality/llm.py +101 -0
- sqlquality/models.py +56 -0
- sqlquality/py.typed +0 -0
- sqlquality/report.py +124 -0
- sqlquality/sqlast.py +128 -0
- sqlquality-0.2.0.dist-info/METADATA +519 -0
- sqlquality-0.2.0.dist-info/RECORD +26 -0
- sqlquality-0.2.0.dist-info/WHEEL +4 -0
- sqlquality-0.2.0.dist-info/entry_points.txt +2 -0
- sqlquality-0.2.0.dist-info/licenses/LICENSE +21 -0
sqlquality/cli.py
ADDED
|
@@ -0,0 +1,540 @@
|
|
|
1
|
+
"""Command-line interface for sqlquality."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
from dataclasses import asdict, replace
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import NoReturn
|
|
10
|
+
|
|
11
|
+
import typer
|
|
12
|
+
from rich.console import Console
|
|
13
|
+
from rich.table import Table
|
|
14
|
+
|
|
15
|
+
from sqlquality import __version__
|
|
16
|
+
from sqlquality.adapters import get_adapter
|
|
17
|
+
from sqlquality.changeset import ChangeSetError, compute_changeset, run_state_modified
|
|
18
|
+
from sqlquality.complexity import ComplexityEngine
|
|
19
|
+
from sqlquality.config import ConfigError, load_config
|
|
20
|
+
from sqlquality.dbtproject import DbtProject, DbtProjectError
|
|
21
|
+
from sqlquality.delta import compute_deltas
|
|
22
|
+
from sqlquality.dialects import validate_dialect
|
|
23
|
+
from sqlquality.gate import evaluate_gate
|
|
24
|
+
from sqlquality.linter import fix_sql, lint_sql
|
|
25
|
+
from sqlquality.llm import Suggestion, enrich_findings, resolve_provider
|
|
26
|
+
from sqlquality.models import Severity
|
|
27
|
+
from sqlquality.report import gate_payload, render_html, render_markdown, verdict_label
|
|
28
|
+
from sqlquality.sqlast import SqlParseError, analyze_sql, parse, strip_jinja
|
|
29
|
+
|
|
30
|
+
console = Console()
|
|
31
|
+
|
|
32
|
+
app = typer.Typer(
|
|
33
|
+
no_args_is_help=True,
|
|
34
|
+
add_completion=False,
|
|
35
|
+
help="Measure dbt model performance and complexity.",
|
|
36
|
+
epilog=(
|
|
37
|
+
"Exit codes: 0 = pass / no findings; 1 = findings or gate failure; "
|
|
38
|
+
"2 = usage, config, or input error."
|
|
39
|
+
),
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
#: Substrings whose presence marks a source as containing dbt/Jinja templating.
|
|
43
|
+
_JINJA_MARKERS = ("{{", "{%")
|
|
44
|
+
#: Notice emitted (stderr only) when analysis falls back to Jinja placeholders.
|
|
45
|
+
_JINJA_NOTICE = (
|
|
46
|
+
"analyzed with Jinja placeholders — results are approximate; "
|
|
47
|
+
"prefer compiled SQL from target/compiled/"
|
|
48
|
+
)
|
|
49
|
+
#: Appended to a parse-error message when the source contained Jinja markers.
|
|
50
|
+
_COMPILED_HINT = (
|
|
51
|
+
" — the source contains Jinja; supply compiled SQL from target/compiled/ for accurate results"
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _version_callback(value: bool) -> None:
|
|
56
|
+
if value:
|
|
57
|
+
typer.echo(__version__)
|
|
58
|
+
raise typer.Exit()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _has_jinja(sql: str) -> bool:
|
|
62
|
+
"""True if the source contains any dbt/Jinja templating markers."""
|
|
63
|
+
return any(marker in sql for marker in _JINJA_MARKERS)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _labels(path: Path) -> tuple[str, str]:
|
|
67
|
+
"""Return (display_name, machine_path) for a source; '<stdin>' when path is '-'."""
|
|
68
|
+
if str(path) == "-":
|
|
69
|
+
return "<stdin>", "<stdin>"
|
|
70
|
+
return path.name, str(path)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def read_sql_file(path: Path) -> str:
|
|
74
|
+
"""Read SQL text from a file, or from stdin when ``path`` is ``-``.
|
|
75
|
+
|
|
76
|
+
Prints a friendly message and exits 2 on a missing file, a non-UTF-8 source, or
|
|
77
|
+
any other read error, so callers get a consistent input-error experience. Stdin
|
|
78
|
+
is decoded from raw bytes so a non-UTF-8 pipe fails the same way a file does
|
|
79
|
+
(never a traceback / exit 1 that CI would misread as findings).
|
|
80
|
+
"""
|
|
81
|
+
is_stdin = str(path) == "-"
|
|
82
|
+
source = "<stdin>" if is_stdin else str(path)
|
|
83
|
+
try:
|
|
84
|
+
if is_stdin:
|
|
85
|
+
return sys.stdin.buffer.read().decode("utf-8")
|
|
86
|
+
return path.read_text(encoding="utf-8")
|
|
87
|
+
except FileNotFoundError:
|
|
88
|
+
typer.echo(f"No such file: {path}", err=True)
|
|
89
|
+
raise typer.Exit(code=2)
|
|
90
|
+
except UnicodeDecodeError:
|
|
91
|
+
typer.echo(f"{source} is not valid UTF-8 — supply UTF-8 encoded SQL.", err=True)
|
|
92
|
+
raise typer.Exit(code=2)
|
|
93
|
+
except OSError as exc:
|
|
94
|
+
typer.echo(f"Could not read {source}: {exc}", err=True)
|
|
95
|
+
raise typer.Exit(code=2)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _validate_dialect_or_exit(name: str) -> str:
|
|
99
|
+
"""Normalize a dialect name or print the friendly error and exit 2."""
|
|
100
|
+
try:
|
|
101
|
+
return validate_dialect(name)
|
|
102
|
+
except ValueError as exc:
|
|
103
|
+
typer.echo(str(exc), err=True)
|
|
104
|
+
raise typer.Exit(code=2)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _fail_parse(exc: SqlParseError, *, had_jinja: bool) -> NoReturn:
|
|
108
|
+
"""Print a parse-error message (with a compiled-SQL hint for Jinja) and exit 2."""
|
|
109
|
+
message = str(exc)
|
|
110
|
+
if had_jinja:
|
|
111
|
+
message += _COMPILED_HINT
|
|
112
|
+
typer.echo(message, err=True)
|
|
113
|
+
raise typer.Exit(code=2)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@app.callback()
|
|
117
|
+
def _root(
|
|
118
|
+
version: bool = typer.Option(
|
|
119
|
+
False,
|
|
120
|
+
"--version",
|
|
121
|
+
callback=_version_callback,
|
|
122
|
+
is_eager=True,
|
|
123
|
+
help="Show the version and exit.",
|
|
124
|
+
),
|
|
125
|
+
) -> None:
|
|
126
|
+
"""sqlquality — measure dbt model performance and complexity."""
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@app.command()
|
|
130
|
+
def complexity(
|
|
131
|
+
path: Path = typer.Argument(
|
|
132
|
+
...,
|
|
133
|
+
dir_okay=False,
|
|
134
|
+
help="Path to a .sql file (or '-' for stdin).",
|
|
135
|
+
),
|
|
136
|
+
dialect: str = typer.Option(
|
|
137
|
+
"postgres", "--dialect", "-d", help="SQL dialect (e.g. postgres, redshift)."
|
|
138
|
+
),
|
|
139
|
+
json_out: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."),
|
|
140
|
+
) -> None:
|
|
141
|
+
"""Score the structural complexity of a single SQL file."""
|
|
142
|
+
dialect = _validate_dialect_or_exit(dialect)
|
|
143
|
+
sql = read_sql_file(path)
|
|
144
|
+
display_name, machine_path = _labels(path)
|
|
145
|
+
|
|
146
|
+
jinja_notice = False
|
|
147
|
+
try:
|
|
148
|
+
metrics = analyze_sql(sql, dialect)
|
|
149
|
+
except SqlParseError as exc:
|
|
150
|
+
if not _has_jinja(sql):
|
|
151
|
+
_fail_parse(exc, had_jinja=False)
|
|
152
|
+
try:
|
|
153
|
+
metrics = analyze_sql(strip_jinja(sql), dialect)
|
|
154
|
+
except SqlParseError as retry_exc:
|
|
155
|
+
_fail_parse(retry_exc, had_jinja=True)
|
|
156
|
+
jinja_notice = True
|
|
157
|
+
|
|
158
|
+
if jinja_notice:
|
|
159
|
+
typer.echo(_JINJA_NOTICE, err=True)
|
|
160
|
+
|
|
161
|
+
result = ComplexityEngine().score(metrics)
|
|
162
|
+
|
|
163
|
+
if json_out:
|
|
164
|
+
payload = {
|
|
165
|
+
"path": machine_path,
|
|
166
|
+
"dialect": dialect,
|
|
167
|
+
"composite": result.composite,
|
|
168
|
+
"components": result.components,
|
|
169
|
+
"metrics": asdict(metrics),
|
|
170
|
+
}
|
|
171
|
+
typer.echo(json.dumps(payload, indent=2, sort_keys=True))
|
|
172
|
+
return
|
|
173
|
+
|
|
174
|
+
table = Table(title=f"Complexity — {display_name} (composite {result.composite})")
|
|
175
|
+
table.add_column("metric")
|
|
176
|
+
table.add_column("value", justify="right")
|
|
177
|
+
table.add_column("contribution", justify="right")
|
|
178
|
+
for name, contribution in result.components.items():
|
|
179
|
+
raw_value = getattr(metrics, name, None)
|
|
180
|
+
table.add_row(
|
|
181
|
+
name,
|
|
182
|
+
"" if raw_value is None else str(raw_value),
|
|
183
|
+
str(contribution),
|
|
184
|
+
)
|
|
185
|
+
console.print(table)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _resolve_check_dialect(candidate: DbtProject) -> str:
|
|
189
|
+
"""Resolve check's dialect from the manifest's adapter_type, else postgres.
|
|
190
|
+
|
|
191
|
+
Emits a stderr notice describing the source. Only called when no explicit
|
|
192
|
+
``--dialect`` was given.
|
|
193
|
+
"""
|
|
194
|
+
adapter_type = candidate.adapter_type()
|
|
195
|
+
if isinstance(adapter_type, str) and adapter_type:
|
|
196
|
+
try:
|
|
197
|
+
resolved = validate_dialect(adapter_type)
|
|
198
|
+
except ValueError:
|
|
199
|
+
resolved = None
|
|
200
|
+
if resolved is not None:
|
|
201
|
+
typer.echo(f"dialect: {resolved} (from manifest adapter_type)", err=True)
|
|
202
|
+
return resolved
|
|
203
|
+
typer.echo(
|
|
204
|
+
"dialect: postgres (default — manifest adapter_type absent or unrecognized)",
|
|
205
|
+
err=True,
|
|
206
|
+
)
|
|
207
|
+
return "postgres"
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@app.command()
|
|
211
|
+
def check(
|
|
212
|
+
project_dir: Path = typer.Option(
|
|
213
|
+
..., "--project-dir", help="dbt project dir containing target/manifest.json."
|
|
214
|
+
),
|
|
215
|
+
state: Path = typer.Option(
|
|
216
|
+
..., "--state", help="Baseline artifacts dir (contains manifest.json)."
|
|
217
|
+
),
|
|
218
|
+
config: Path | None = typer.Option(
|
|
219
|
+
None, "--config", help="Path to sqlquality.yml (default: <project-dir>/sqlquality.yml)."
|
|
220
|
+
),
|
|
221
|
+
dialect: str | None = typer.Option(
|
|
222
|
+
None,
|
|
223
|
+
"--dialect",
|
|
224
|
+
"-d",
|
|
225
|
+
help="SQL dialect (default: manifest adapter_type, else postgres).",
|
|
226
|
+
),
|
|
227
|
+
json_out: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."),
|
|
228
|
+
html: Path | None = typer.Option(None, "--html", help="Write a self-contained HTML report."),
|
|
229
|
+
markdown: Path | None = typer.Option(
|
|
230
|
+
None, "--markdown", help="Write a markdown report (e.g. for a PR comment)."
|
|
231
|
+
),
|
|
232
|
+
dbt: str = typer.Option("dbt", "--dbt", help="dbt executable to invoke."),
|
|
233
|
+
) -> None:
|
|
234
|
+
"""Gate a dbt change on the complexity delta of its changed models."""
|
|
235
|
+
# An explicit --config that isn't a readable file (missing, or a directory)
|
|
236
|
+
# is a user error; the implicit <project-dir>/sqlquality.yml default stays
|
|
237
|
+
# lenient (absent -> defaults).
|
|
238
|
+
if config is not None and not config.is_file():
|
|
239
|
+
typer.echo(f"--config path is not a file: {config}", err=True)
|
|
240
|
+
raise typer.Exit(code=2)
|
|
241
|
+
# An explicit --dialect is validated up front; the manifest-derived default is
|
|
242
|
+
# resolved after the candidate manifest loads.
|
|
243
|
+
if dialect is not None:
|
|
244
|
+
dialect = _validate_dialect_or_exit(dialect)
|
|
245
|
+
cfg_path = config if config is not None else project_dir / "sqlquality.yml"
|
|
246
|
+
try:
|
|
247
|
+
cfg = load_config(cfg_path)
|
|
248
|
+
except ConfigError as exc:
|
|
249
|
+
typer.echo(str(exc), err=True)
|
|
250
|
+
raise typer.Exit(code=2)
|
|
251
|
+
|
|
252
|
+
manifest_path = project_dir / "target" / "manifest.json"
|
|
253
|
+
try:
|
|
254
|
+
candidate = DbtProject.from_path(manifest_path)
|
|
255
|
+
except DbtProjectError as exc:
|
|
256
|
+
typer.echo(str(exc), err=True)
|
|
257
|
+
raise typer.Exit(code=2)
|
|
258
|
+
|
|
259
|
+
resolved_dialect = dialect if dialect is not None else _resolve_check_dialect(candidate)
|
|
260
|
+
|
|
261
|
+
schema_version = candidate.schema_version()
|
|
262
|
+
if "/v12" not in schema_version:
|
|
263
|
+
found = schema_version or "(absent)"
|
|
264
|
+
typer.echo(
|
|
265
|
+
f"warning: candidate manifest dbt_schema_version is {found}, "
|
|
266
|
+
"expected a v12 schema — results may be unreliable",
|
|
267
|
+
err=True,
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
try:
|
|
271
|
+
ls_stdout = run_state_modified(project_dir, state, dbt)
|
|
272
|
+
except ChangeSetError as exc:
|
|
273
|
+
typer.echo(str(exc), err=True)
|
|
274
|
+
raise typer.Exit(code=2)
|
|
275
|
+
|
|
276
|
+
changeset = compute_changeset(candidate, ls_stdout)
|
|
277
|
+
|
|
278
|
+
baseline_path = state / "manifest.json"
|
|
279
|
+
try:
|
|
280
|
+
baseline = DbtProject.from_path(baseline_path) if baseline_path.exists() else None
|
|
281
|
+
except DbtProjectError as exc:
|
|
282
|
+
typer.echo(str(exc), err=True)
|
|
283
|
+
raise typer.Exit(code=2)
|
|
284
|
+
|
|
285
|
+
deltas, skipped = compute_deltas(baseline, candidate, changeset.changed, resolved_dialect)
|
|
286
|
+
report = evaluate_gate(deltas, cfg)
|
|
287
|
+
|
|
288
|
+
if html is not None:
|
|
289
|
+
Path(html).write_text(render_html(report, skipped))
|
|
290
|
+
|
|
291
|
+
if markdown is not None:
|
|
292
|
+
Path(markdown).write_text(render_markdown(report, skipped))
|
|
293
|
+
|
|
294
|
+
if json_out:
|
|
295
|
+
typer.echo(
|
|
296
|
+
json.dumps(gate_payload(report, changeset.neighbors, skipped), indent=2, sort_keys=True)
|
|
297
|
+
)
|
|
298
|
+
else:
|
|
299
|
+
verdict = verdict_label(report, emoji=True)
|
|
300
|
+
table = Table(
|
|
301
|
+
title=f"sqlquality: {verdict} (changed {len(deltas)}, neighbors {len(changeset.neighbors)})"
|
|
302
|
+
)
|
|
303
|
+
table.add_column("model")
|
|
304
|
+
table.add_column("baseline", justify="right")
|
|
305
|
+
table.add_column("candidate", justify="right")
|
|
306
|
+
table.add_column("delta", justify="right")
|
|
307
|
+
table.add_column("", justify="center")
|
|
308
|
+
for d in report.deltas:
|
|
309
|
+
flag = "⚠️" if d.unique_id in report.regressions else ("new" if d.is_new else "")
|
|
310
|
+
table.add_row(d.unique_id, str(d.baseline), str(d.candidate), f"{d.delta:+}", flag)
|
|
311
|
+
console.print(table)
|
|
312
|
+
for uid, reason in skipped:
|
|
313
|
+
console.print(f"[yellow]skipped[/] {uid}: {reason}")
|
|
314
|
+
|
|
315
|
+
raise typer.Exit(code=0 if report.passed else 1)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
@app.command()
|
|
319
|
+
def lint(
|
|
320
|
+
paths: list[Path] = typer.Argument(
|
|
321
|
+
..., dir_okay=False, help="One or more .sql files (or '-' for stdin)."
|
|
322
|
+
),
|
|
323
|
+
dialect: str = typer.Option("postgres", "--dialect", "-d", help="SQL dialect."),
|
|
324
|
+
fix: bool = typer.Option(
|
|
325
|
+
False,
|
|
326
|
+
"--fix",
|
|
327
|
+
help="Rewrite the file with auto-fixes. Exit code reflects pre-fix findings "
|
|
328
|
+
"(a fully-fixed file still exits 1).",
|
|
329
|
+
),
|
|
330
|
+
exclude_rules: str | None = typer.Option(
|
|
331
|
+
None, "--exclude-rules", help="Comma-separated rule codes to skip."
|
|
332
|
+
),
|
|
333
|
+
sqlfluff_config: Path | None = typer.Option(
|
|
334
|
+
None,
|
|
335
|
+
"--sqlfluff-config",
|
|
336
|
+
exists=True,
|
|
337
|
+
dir_okay=False,
|
|
338
|
+
readable=True,
|
|
339
|
+
help="Path to a SQLFluff config file (e.g. .sqlfluff) to apply.",
|
|
340
|
+
),
|
|
341
|
+
warn_only: bool = typer.Option(
|
|
342
|
+
False, "--warn-only", help="Print/emit findings but always exit 0."
|
|
343
|
+
),
|
|
344
|
+
json_out: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."),
|
|
345
|
+
) -> None:
|
|
346
|
+
"""Lint SQL files for best-practice violations (SQLFluff); --fix rewrites them."""
|
|
347
|
+
dialect = _validate_dialect_or_exit(dialect)
|
|
348
|
+
if fix and any(str(p) == "-" for p in paths):
|
|
349
|
+
typer.echo("--fix cannot rewrite stdin ('-'); pass a file path instead.", err=True)
|
|
350
|
+
raise typer.Exit(code=2)
|
|
351
|
+
excl = [r.strip() for r in exclude_rules.split(",")] if exclude_rules else None
|
|
352
|
+
config_path = str(sqlfluff_config) if sqlfluff_config is not None else None
|
|
353
|
+
# Pre-flight every path before touching any file: a bad path (missing, non-UTF-8)
|
|
354
|
+
# must exit 2 with no side effects, not rewrite earlier files then abort.
|
|
355
|
+
sources = [(path, read_sql_file(path)) for path in paths]
|
|
356
|
+
file_reports: list[dict] = []
|
|
357
|
+
gating = False
|
|
358
|
+
for path, sql in sources:
|
|
359
|
+
_, machine_path = _labels(path)
|
|
360
|
+
findings = lint_sql(sql, dialect, excl, config_path)
|
|
361
|
+
changed = False
|
|
362
|
+
if fix:
|
|
363
|
+
fixed_sql = fix_sql(sql, dialect, excl, config_path)
|
|
364
|
+
if fixed_sql != sql:
|
|
365
|
+
path.write_text(fixed_sql)
|
|
366
|
+
changed = True
|
|
367
|
+
# INFO (unresolved-Jinja) findings are advisory and never gate the commit.
|
|
368
|
+
gating = gating or any(f.severity in (Severity.WARNING, Severity.ERROR) for f in findings)
|
|
369
|
+
file_reports.append(
|
|
370
|
+
{
|
|
371
|
+
"path": machine_path,
|
|
372
|
+
"fixed": changed,
|
|
373
|
+
"findings": [
|
|
374
|
+
{
|
|
375
|
+
"code": f.code,
|
|
376
|
+
"message": f.message,
|
|
377
|
+
"line": f.line,
|
|
378
|
+
"severity": f.severity.value,
|
|
379
|
+
"fixable": f.fixable,
|
|
380
|
+
}
|
|
381
|
+
for f in findings
|
|
382
|
+
],
|
|
383
|
+
}
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
if json_out:
|
|
387
|
+
typer.echo(json.dumps({"files": file_reports}, indent=2, sort_keys=True))
|
|
388
|
+
else:
|
|
389
|
+
for report in file_reports:
|
|
390
|
+
table = Table(title=f"Lint — {report['path']} ({len(report['findings'])} findings)")
|
|
391
|
+
table.add_column("line", justify="right")
|
|
392
|
+
table.add_column("code")
|
|
393
|
+
table.add_column("severity")
|
|
394
|
+
table.add_column("fix?", justify="center")
|
|
395
|
+
table.add_column("message")
|
|
396
|
+
for item in report["findings"]:
|
|
397
|
+
table.add_row(
|
|
398
|
+
str(item["line"]),
|
|
399
|
+
item["code"],
|
|
400
|
+
item["severity"],
|
|
401
|
+
"✓" if item["fixable"] else "",
|
|
402
|
+
item["message"],
|
|
403
|
+
)
|
|
404
|
+
console.print(table)
|
|
405
|
+
if report["fixed"]:
|
|
406
|
+
console.print(f"[green]Rewrote {report['path']} with auto-fixes.[/]")
|
|
407
|
+
|
|
408
|
+
raise typer.Exit(code=1 if gating and not warn_only else 0)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
@app.command()
|
|
412
|
+
def perf(
|
|
413
|
+
path: Path = typer.Argument(
|
|
414
|
+
..., exists=True, dir_okay=False, readable=True, help="Path to a .sql file."
|
|
415
|
+
),
|
|
416
|
+
dialect: str = typer.Option("postgres", "--dialect", "-d", help="SQL dialect/engine."),
|
|
417
|
+
explain: Path | None = typer.Option(
|
|
418
|
+
None,
|
|
419
|
+
"--explain",
|
|
420
|
+
exists=True,
|
|
421
|
+
dir_okay=False,
|
|
422
|
+
readable=True,
|
|
423
|
+
help="A captured EXPLAIN file (FORMAT JSON for Postgres; plan text for Redshift).",
|
|
424
|
+
),
|
|
425
|
+
json_out: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."),
|
|
426
|
+
suggest: bool = typer.Option(
|
|
427
|
+
False, "--suggest", help="Enrich findings with LLM suggestions (needs SQLQUALITY_LLM set)."
|
|
428
|
+
),
|
|
429
|
+
) -> None:
|
|
430
|
+
"""Analyze a SQL file for performance anti-patterns (+ optional EXPLAIN plan)."""
|
|
431
|
+
if str(path) == "-":
|
|
432
|
+
typer.echo("stdin ('-') is not supported for perf; pass a file path.", err=True)
|
|
433
|
+
raise typer.Exit(code=2)
|
|
434
|
+
dialect = _validate_dialect_or_exit(dialect)
|
|
435
|
+
try:
|
|
436
|
+
adapter = get_adapter(dialect)
|
|
437
|
+
except ValueError as exc:
|
|
438
|
+
# A valid-but-unsupported dialect (only postgres/redshift have perf adapters).
|
|
439
|
+
typer.echo(str(exc), err=True)
|
|
440
|
+
raise typer.Exit(code=2)
|
|
441
|
+
|
|
442
|
+
sql = read_sql_file(path)
|
|
443
|
+
display_name, machine_path = _labels(path)
|
|
444
|
+
|
|
445
|
+
# static_findings swallows parse errors into an SQ000 finding, so a raw dbt
|
|
446
|
+
# model would otherwise yield only SQ000. When the source is Jinja, parse-check
|
|
447
|
+
# first and retry against stripped placeholders so we get real anti-pattern
|
|
448
|
+
# findings; if stripping still fails, fall back to SQ000 (annotated below).
|
|
449
|
+
# Plain SQL skips the pre-parse — static_findings parses it once itself.
|
|
450
|
+
analyze_target = sql
|
|
451
|
+
jinja_notice = False
|
|
452
|
+
had_jinja = _has_jinja(sql)
|
|
453
|
+
if had_jinja:
|
|
454
|
+
try:
|
|
455
|
+
parse(sql, dialect)
|
|
456
|
+
except SqlParseError:
|
|
457
|
+
stripped = strip_jinja(sql)
|
|
458
|
+
try:
|
|
459
|
+
parse(stripped, dialect)
|
|
460
|
+
except SqlParseError:
|
|
461
|
+
pass
|
|
462
|
+
else:
|
|
463
|
+
analyze_target = stripped
|
|
464
|
+
jinja_notice = True
|
|
465
|
+
|
|
466
|
+
findings = adapter.static_findings(analyze_target)
|
|
467
|
+
if had_jinja and not jinja_notice:
|
|
468
|
+
# Stripping did not yield parseable SQL: annotate the SQ000 parse error with
|
|
469
|
+
# the compiled-SQL hint so the user knows why and what to do.
|
|
470
|
+
findings = [
|
|
471
|
+
replace(f, message=f.message + _COMPILED_HINT) if f.code == "SQ000" else f
|
|
472
|
+
for f in findings
|
|
473
|
+
]
|
|
474
|
+
if jinja_notice:
|
|
475
|
+
typer.echo(_JINJA_NOTICE, err=True)
|
|
476
|
+
|
|
477
|
+
if explain is not None:
|
|
478
|
+
try:
|
|
479
|
+
findings = findings + adapter.plan_findings(explain.read_text())
|
|
480
|
+
except ValueError as exc:
|
|
481
|
+
typer.echo(str(exc), err=True)
|
|
482
|
+
raise typer.Exit(code=2)
|
|
483
|
+
|
|
484
|
+
suggestions: list[Suggestion] = []
|
|
485
|
+
if suggest:
|
|
486
|
+
try:
|
|
487
|
+
provider = resolve_provider()
|
|
488
|
+
if provider is None:
|
|
489
|
+
typer.echo(
|
|
490
|
+
"LLM suggestions require SQLQUALITY_LLM=anthropic (or 1/true) "
|
|
491
|
+
"(and `pip install 'sqlquality[llm]'` + credentials).",
|
|
492
|
+
err=True,
|
|
493
|
+
)
|
|
494
|
+
else:
|
|
495
|
+
suggestions = enrich_findings(findings, sql, provider)
|
|
496
|
+
if len(suggestions) < len(findings):
|
|
497
|
+
# enrich_findings skips per-finding call failures silently, so
|
|
498
|
+
# surface a single note when some (or all) calls dropped out.
|
|
499
|
+
missing = len(findings) - len(suggestions)
|
|
500
|
+
typer.echo(f"LLM suggestions unavailable for {missing} finding(s).", err=True)
|
|
501
|
+
except Exception as exc: # advisory-only: never affect the exit code or report
|
|
502
|
+
# Covers provider construction (missing package/credentials); findings
|
|
503
|
+
# still print and the exit code is unchanged.
|
|
504
|
+
typer.echo(f"LLM suggestions unavailable: {exc}", err=True)
|
|
505
|
+
|
|
506
|
+
if json_out:
|
|
507
|
+
payload = {
|
|
508
|
+
"path": machine_path,
|
|
509
|
+
"dialect": dialect,
|
|
510
|
+
"findings": [
|
|
511
|
+
{
|
|
512
|
+
"code": f.code,
|
|
513
|
+
"message": f.message,
|
|
514
|
+
"line": f.line,
|
|
515
|
+
"severity": f.severity.value,
|
|
516
|
+
"fixable": f.fixable,
|
|
517
|
+
}
|
|
518
|
+
for f in findings
|
|
519
|
+
],
|
|
520
|
+
"suggestions": [{"code": s.code, "text": s.text} for s in suggestions],
|
|
521
|
+
}
|
|
522
|
+
typer.echo(json.dumps(payload, indent=2, sort_keys=True))
|
|
523
|
+
else:
|
|
524
|
+
table = Table(title=f"Perf — {display_name} ({dialect}, {len(findings)} findings)")
|
|
525
|
+
table.add_column("code")
|
|
526
|
+
table.add_column("severity")
|
|
527
|
+
table.add_column("message")
|
|
528
|
+
for f in findings:
|
|
529
|
+
table.add_row(f.code, f.severity.value, f.message)
|
|
530
|
+
console.print(table)
|
|
531
|
+
for s in suggestions:
|
|
532
|
+
console.print(f"[cyan]{s.code}[/]: {s.text}")
|
|
533
|
+
|
|
534
|
+
has_error = any(f.severity is Severity.ERROR for f in findings)
|
|
535
|
+
raise typer.Exit(code=1 if has_error else 0)
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def main() -> None:
|
|
539
|
+
"""Console-script entry point."""
|
|
540
|
+
app()
|
sqlquality/complexity.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Turn structural metrics (+ optional DAG facts) into a complexity score.
|
|
2
|
+
|
|
3
|
+
The composite is open-ended (not capped): ~100 is very complex, but a
|
|
4
|
+
sufficiently large model can score higher. Capping would blind the delta gate
|
|
5
|
+
for models already at the cap.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from sqlquality.models import ComplexityMetrics, ComplexityScore, DagFacts
|
|
11
|
+
|
|
12
|
+
METRIC_WEIGHTS: dict[str, float] = {
|
|
13
|
+
"join_count": 6.0,
|
|
14
|
+
"cte_count": 2.0,
|
|
15
|
+
"subquery_count": 5.0,
|
|
16
|
+
"window_count": 4.0,
|
|
17
|
+
"case_count": 2.0,
|
|
18
|
+
"union_count": 3.0,
|
|
19
|
+
"distinct_count": 2.0,
|
|
20
|
+
"max_select_depth": 5.0,
|
|
21
|
+
"projected_columns": 0.2,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
DAG_WEIGHTS: dict[str, float] = {
|
|
25
|
+
"fan_out": 1.5,
|
|
26
|
+
"lineage_depth": 2.0,
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ComplexityEngine:
|
|
31
|
+
"""Compute a weighted composite complexity score."""
|
|
32
|
+
|
|
33
|
+
def score(self, metrics: ComplexityMetrics, dag: DagFacts | None = None) -> ComplexityScore:
|
|
34
|
+
components: dict[str, float] = {}
|
|
35
|
+
for name, weight in METRIC_WEIGHTS.items():
|
|
36
|
+
components[name] = round(weight * getattr(metrics, name), 2)
|
|
37
|
+
if dag is not None:
|
|
38
|
+
for name, weight in DAG_WEIGHTS.items():
|
|
39
|
+
components[f"dag.{name}"] = round(weight * getattr(dag, name), 2)
|
|
40
|
+
composite = round(sum(components.values()), 1)
|
|
41
|
+
return ComplexityScore(composite=composite, components=components, metrics=metrics, dag=dag)
|
sqlquality/config.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Load sqlquality.yml into typed config (with defaults)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import yaml
|
|
10
|
+
|
|
11
|
+
_VALID_MODES = ("warn", "fail")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ConfigError(ValueError):
|
|
15
|
+
"""Raised when sqlquality.yml is malformed or holds an invalid value."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class GateConfig:
|
|
20
|
+
mode: str = "warn" # "warn" | "fail"
|
|
21
|
+
max_complexity_increase: float = 10.0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass(frozen=True)
|
|
25
|
+
class Config:
|
|
26
|
+
gate: GateConfig = field(default_factory=GateConfig)
|
|
27
|
+
waivers: tuple[str, ...] = ()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def load_config(path: Path | None) -> Config:
|
|
31
|
+
"""Load config from YAML, or return defaults if absent."""
|
|
32
|
+
if path is None or not Path(path).exists():
|
|
33
|
+
return Config()
|
|
34
|
+
try:
|
|
35
|
+
raw_text = Path(path).read_text()
|
|
36
|
+
except OSError as exc: # e.g. --config points at a directory
|
|
37
|
+
raise ConfigError(f"Could not read config {path}: {exc}") from exc
|
|
38
|
+
try:
|
|
39
|
+
data = yaml.safe_load(raw_text)
|
|
40
|
+
except yaml.YAMLError as exc:
|
|
41
|
+
raise ConfigError(f"Malformed YAML in {path}: {exc}") from exc
|
|
42
|
+
if data is None:
|
|
43
|
+
data = {}
|
|
44
|
+
if not isinstance(data, dict):
|
|
45
|
+
raise ConfigError(f"top-level of {path} must be a mapping (got {type(data).__name__})")
|
|
46
|
+
|
|
47
|
+
defaults = GateConfig()
|
|
48
|
+
gate_data = data.get("gate")
|
|
49
|
+
if gate_data is None: # null or absent -> defaults (e.g. a commented-out block)
|
|
50
|
+
gate_data = {}
|
|
51
|
+
if not isinstance(gate_data, dict):
|
|
52
|
+
raise ConfigError(
|
|
53
|
+
f"`gate` must be a mapping (got {type(gate_data).__name__}: {gate_data!r})"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
mode = gate_data.get("mode", defaults.mode)
|
|
57
|
+
if mode not in _VALID_MODES:
|
|
58
|
+
raise ConfigError(f"`gate.mode` must be 'warn' or 'fail' (got {mode!r})")
|
|
59
|
+
|
|
60
|
+
raw_threshold = gate_data.get("max_complexity_increase", defaults.max_complexity_increase)
|
|
61
|
+
# Reject bools up front: bool is an int subclass, so float(True) == 1.0 would
|
|
62
|
+
# silently accept `max_complexity_increase: true` as a threshold of 1.0.
|
|
63
|
+
if isinstance(raw_threshold, bool):
|
|
64
|
+
raise ConfigError(
|
|
65
|
+
f"`gate.max_complexity_increase` must be a number (got bool {raw_threshold!r})"
|
|
66
|
+
)
|
|
67
|
+
try:
|
|
68
|
+
max_complexity_increase = float(raw_threshold)
|
|
69
|
+
except (TypeError, ValueError) as exc:
|
|
70
|
+
raise ConfigError(
|
|
71
|
+
f"`gate.max_complexity_increase` must be a number (got {raw_threshold!r})"
|
|
72
|
+
) from exc
|
|
73
|
+
# NaN/inf would make every `delta > threshold` comparison False, silently
|
|
74
|
+
# turning a fail-mode gate into a no-op.
|
|
75
|
+
if not math.isfinite(max_complexity_increase):
|
|
76
|
+
raise ConfigError(
|
|
77
|
+
f"`gate.max_complexity_increase` must be a finite number (got {raw_threshold!r})"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
gate = GateConfig(mode=mode, max_complexity_increase=max_complexity_increase)
|
|
81
|
+
|
|
82
|
+
raw_waivers = data.get("waivers") or ()
|
|
83
|
+
if isinstance(raw_waivers, str):
|
|
84
|
+
raw_waivers = [raw_waivers]
|
|
85
|
+
waivers = tuple(raw_waivers)
|
|
86
|
+
return Config(gate=gate, waivers=waivers)
|