sqlassert 0.1.1__py3-none-any.whl → 0.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sqlassert/__init__.py +34 -9
- sqlassert/__main__.py +138 -0
- sqlassert/check.py +444 -0
- sqlassert/diagnostics.py +57 -0
- sqlassert/dialects.py +211 -0
- sqlassert/discovery/__init__.py +30 -0
- sqlassert/discovery/bigquery.py +71 -0
- sqlassert/discovery/databricks.py +18 -0
- sqlassert/discovery/duckdb.py +163 -0
- sqlassert/discovery/fetcher.py +402 -0
- sqlassert/discovery/information_schema.py +326 -0
- sqlassert/discovery/mysql.py +70 -0
- sqlassert/discovery/postgres.py +143 -0
- sqlassert/discovery/redshift.py +36 -0
- sqlassert/discovery/snowflake.py +63 -0
- sqlassert/discovery/trino.py +37 -0
- sqlassert/improvements.py +528 -0
- sqlassert/ir/__init__.py +56 -0
- sqlassert/ir/convert.py +2377 -0
- sqlassert/ir/model.py +676 -0
- sqlassert/main.py +57 -0
- sqlassert/properties.py +121 -0
- sqlassert/provenance.py +73 -0
- sqlassert/python_engine.py +299 -0
- sqlassert/reporting.py +102 -0
- sqlassert/rules/STYLEGUIDE.md +19 -0
- sqlassert/rules/__init__.py +11 -0
- sqlassert/rules/column_set.py +34 -0
- sqlassert/rules/domain.py +376 -0
- sqlassert/rules/equality.py +233 -0
- sqlassert/rules/exactly_one.py +75 -0
- sqlassert/rules/foreign_key.py +316 -0
- sqlassert/rules/join_cardinality.py +94 -0
- sqlassert/rules/nullability.py +418 -0
- sqlassert/rules/uniqueness.py +601 -0
- sqlassert/smells.py +306 -0
- sqlassert/sql_parse.py +974 -0
- sqlassert/suggest.py +339 -0
- sqlassert-0.3.1.dist-info/METADATA +725 -0
- sqlassert-0.3.1.dist-info/RECORD +44 -0
- {sqlassert-0.1.1.dist-info → sqlassert-0.3.1.dist-info}/WHEEL +1 -1
- sqlassert-0.3.1.dist-info/entry_points.txt +2 -0
- sqlassert-0.3.1.dist-info/licenses/LICENSE +21 -0
- sqlassert/unique.py +0 -605
- sqlassert-0.1.1.dist-info/METADATA +0 -173
- sqlassert-0.1.1.dist-info/RECORD +0 -6
- {sqlassert-0.1.1.dist-info → sqlassert-0.3.1.dist-info}/top_level.txt +0 -0
sqlassert/__init__.py
CHANGED
|
@@ -1,13 +1,38 @@
|
|
|
1
|
-
from sqlassert.
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
from sqlassert.main import analyze, analyze_database
|
|
2
|
+
from sqlassert.diagnostics import Diagnostic
|
|
3
|
+
from sqlassert.properties import (
|
|
4
|
+
EnumeratedDomain, ExactlyOneJoin, ExhaustiveCase, FunctionalDependency, ManyToOneJoin, NonNullColumn, NonNullColumns, NonNullElements, NonNullUniqueSet, OneToManyJoin,
|
|
5
|
+
OneToOneJoin, Property, UniqueJoin, UniqueSet,
|
|
6
6
|
)
|
|
7
|
+
from sqlassert.provenance import AttachmentSite, Origin
|
|
8
|
+
from sqlassert.reporting import AssertionReport, Outcome, RelationFacts, Report
|
|
9
|
+
from sqlassert.suggest import Suggestion, SuggestionReport, suggest
|
|
7
10
|
|
|
8
11
|
__all__ = [
|
|
9
|
-
"
|
|
10
|
-
"
|
|
11
|
-
"
|
|
12
|
-
"
|
|
12
|
+
"AssertionReport",
|
|
13
|
+
"AttachmentSite",
|
|
14
|
+
"Diagnostic",
|
|
15
|
+
"EnumeratedDomain",
|
|
16
|
+
"ExactlyOneJoin",
|
|
17
|
+
"ExhaustiveCase",
|
|
18
|
+
"FunctionalDependency",
|
|
19
|
+
"ManyToOneJoin",
|
|
20
|
+
"NonNullColumn",
|
|
21
|
+
"NonNullColumns",
|
|
22
|
+
"NonNullElements",
|
|
23
|
+
"NonNullUniqueSet",
|
|
24
|
+
"OneToManyJoin",
|
|
25
|
+
"OneToOneJoin",
|
|
26
|
+
"Origin",
|
|
27
|
+
"Outcome",
|
|
28
|
+
"Property",
|
|
29
|
+
"RelationFacts",
|
|
30
|
+
"Report",
|
|
31
|
+
"Suggestion",
|
|
32
|
+
"SuggestionReport",
|
|
33
|
+
"UniqueSet",
|
|
34
|
+
"UniqueJoin",
|
|
35
|
+
"analyze",
|
|
36
|
+
"analyze_database",
|
|
37
|
+
"suggest",
|
|
13
38
|
]
|
sqlassert/__main__.py
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Command line: `sqlassert check PATH...` checks the assertions in SQL Programs for CI (`check.py`);
|
|
2
|
+
`sqlassert suggest FILE` prints the improvements and markers sqlassert can prove.
|
|
3
|
+
dbt projects are checked by the separate `sqlassert-dbt` package (`integrations/dbt/`)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import argparse
|
|
8
|
+
import sys
|
|
9
|
+
from collections.abc import Iterable
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from sqlassert import check
|
|
13
|
+
from sqlassert.main import DEFAULT_DIALECT
|
|
14
|
+
from sqlassert.suggest import DEFAULT_MAX_SIZE, suggest
|
|
15
|
+
|
|
16
|
+
# How each suggested Property reads in the listing.
|
|
17
|
+
_KINDS = {
|
|
18
|
+
"ExactlyOneJoin": "Exactly One Join",
|
|
19
|
+
"OneToOneJoin": "One To One Join",
|
|
20
|
+
"ManyToOneJoin": "Many To One Join",
|
|
21
|
+
"OneToManyJoin": "One To Many Join",
|
|
22
|
+
"UniqueJoin": "Unique Join",
|
|
23
|
+
"UniqueSet": "Unique Set",
|
|
24
|
+
"NonNullUniqueSet": "Non-Null Unique Set",
|
|
25
|
+
"NonNullColumns": "Not Null",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main(argv: list[str] | None = None) -> int:
|
|
30
|
+
parser = argparse.ArgumentParser(prog="sqlassert")
|
|
31
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
32
|
+
suggest_command = commands.add_parser(
|
|
33
|
+
"suggest", help="list the improvements and markers sqlassert can prove for a SQL Program"
|
|
34
|
+
)
|
|
35
|
+
suggest_command.add_argument("file", help="SQL Program to read, or - for stdin")
|
|
36
|
+
suggest_command.add_argument("--dialect", default=DEFAULT_DIALECT)
|
|
37
|
+
suggest_command.add_argument(
|
|
38
|
+
"--max-size", type=int, default=DEFAULT_MAX_SIZE, help="widest Unique Set searched exhaustively"
|
|
39
|
+
)
|
|
40
|
+
suggest_command.add_argument("--explain", action="store_true", help="print each suggestion's evidence")
|
|
41
|
+
suggest_command.add_argument(
|
|
42
|
+
"--annotate", action="store_true",
|
|
43
|
+
help="paste every suggested marker into FILE (for -, print the annotated SQL instead of the list)",
|
|
44
|
+
)
|
|
45
|
+
check_command = commands.add_parser(
|
|
46
|
+
"check",
|
|
47
|
+
help="analyze SQL Programs and fail when an assertion is UNKNOWN or a Diagnostic is reported",
|
|
48
|
+
description="Analyze each SQL Program from its SQL alone. Exit status: "
|
|
49
|
+
f"{check.EXIT_OK} every assertion PROVED (warnings allowed), {check.EXIT_FINDINGS} an assertion UNKNOWN "
|
|
50
|
+
f"or a Diagnostic reported, {check.EXIT_USAGE} bad arguments, configuration, or unreadable SQL. "
|
|
51
|
+
"Defaults come from [tool.sqlassert] in the nearest pyproject.toml; flags replace them, except --ignore, which adds to them.",
|
|
52
|
+
)
|
|
53
|
+
check_command.add_argument(
|
|
54
|
+
"paths", nargs="*", metavar="PATH",
|
|
55
|
+
help="SQL files, directories (every *.sql under them), or globs; - for stdin (default: [tool.sqlassert] paths)",
|
|
56
|
+
)
|
|
57
|
+
check_command.add_argument("--dialect", help=f"SQL dialect (default: [tool.sqlassert] dialect, else {DEFAULT_DIALECT})")
|
|
58
|
+
ignoring = check_command.add_mutually_exclusive_group()
|
|
59
|
+
ignoring.add_argument(
|
|
60
|
+
"--ignore", action="append", metavar="CODE",
|
|
61
|
+
help="also drop Findings with this code, e.g. undeclared-table, besides [tool.sqlassert] ignore; "
|
|
62
|
+
"repeatable or comma-separated",
|
|
63
|
+
)
|
|
64
|
+
ignoring.add_argument(
|
|
65
|
+
"--only-ignore", action="append", metavar="CODE",
|
|
66
|
+
help="drop Findings with these codes instead of [tool.sqlassert] ignore; repeatable or comma-separated "
|
|
67
|
+
"(--only-ignore '' ignores nothing)",
|
|
68
|
+
)
|
|
69
|
+
check_command.add_argument("--format", choices=sorted(check.FORMATS), default="text")
|
|
70
|
+
check_command.add_argument("--explain", action="store_true", help="print each UNKNOWN assertion's evidence")
|
|
71
|
+
arguments = parser.parse_args(argv)
|
|
72
|
+
if arguments.command == "check":
|
|
73
|
+
return _check(arguments)
|
|
74
|
+
|
|
75
|
+
try:
|
|
76
|
+
sql = check.read_sql(Path(arguments.file))
|
|
77
|
+
except check.UsageError as error:
|
|
78
|
+
print(f"sqlassert suggest: {error}", file=sys.stderr)
|
|
79
|
+
return check.EXIT_USAGE
|
|
80
|
+
report = suggest(sql, dialect=arguments.dialect, max_size=arguments.max_size)
|
|
81
|
+
for diagnostic in report.diagnostics:
|
|
82
|
+
print(f"{diagnostic.code}: {diagnostic.message}", file=sys.stderr)
|
|
83
|
+
for warning in report.warnings:
|
|
84
|
+
print(f"warning: {warning.code}: {warning.message}", file=sys.stderr)
|
|
85
|
+
if arguments.annotate and arguments.file == "-":
|
|
86
|
+
sys.stdout.write(report.annotated())
|
|
87
|
+
return 1 if report.diagnostics else 0
|
|
88
|
+
if arguments.annotate and report.suggestions:
|
|
89
|
+
Path(arguments.file).write_text(report.annotated())
|
|
90
|
+
# Improvements (High) first, then markers (Mid), then smells (Low).
|
|
91
|
+
items = (*report.improvements, *report.suggestions, *report.smells)
|
|
92
|
+
rows = [
|
|
93
|
+
(i.level.value, f"line {i.site.line}", i.site.label, i.rewrite.value, i.message) for i in report.improvements
|
|
94
|
+
] + [
|
|
95
|
+
(s.level.value, f"line {s.site.line}", s.site.label, _KINDS[type(s.property).__name__], s.marker)
|
|
96
|
+
for s in report.suggestions
|
|
97
|
+
] + [
|
|
98
|
+
(s.level.value, f"line {s.site.line}", s.site.label, s.concern.value, s.message) for s in report.smells
|
|
99
|
+
]
|
|
100
|
+
widths = [max((len(row[i]) for row in rows), default=0) for i in range(4)]
|
|
101
|
+
for row, item in zip(rows, items):
|
|
102
|
+
print(" ".join(cell.ljust(width) for cell, width in zip(row[:4], widths)) + " " + row[4])
|
|
103
|
+
if arguments.explain:
|
|
104
|
+
print(check.indented(item.evidence.explain()))
|
|
105
|
+
return 1 if report.diagnostics else 0
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _check(arguments: argparse.Namespace) -> int:
|
|
109
|
+
try:
|
|
110
|
+
config = check.load_config(Path.cwd())
|
|
111
|
+
dialect = arguments.dialect or config.dialect
|
|
112
|
+
ignore = (
|
|
113
|
+
_codes(arguments.only_ignore) if arguments.only_ignore is not None
|
|
114
|
+
else config.ignore | _codes(arguments.ignore or ())
|
|
115
|
+
)
|
|
116
|
+
check.validate(dialect, ignore)
|
|
117
|
+
if arguments.paths:
|
|
118
|
+
paths = check.expand(arguments.paths, Path.cwd())
|
|
119
|
+
elif config.paths:
|
|
120
|
+
paths = check.expand(config.paths, config.root)
|
|
121
|
+
else:
|
|
122
|
+
raise check.UsageError("no SQL to check: name files, directories or globs, or set paths in [tool.sqlassert]")
|
|
123
|
+
programs = [(check.display(path), check.read_sql(path)) for path in paths]
|
|
124
|
+
except check.UsageError as error:
|
|
125
|
+
print(f"sqlassert check: {error}", file=sys.stderr)
|
|
126
|
+
return check.EXIT_USAGE
|
|
127
|
+
checked = [check.check(path, sql, dialect=dialect, ignore=ignore) for path, sql in programs]
|
|
128
|
+
sys.stdout.write(check.FORMATS[arguments.format](checked, explain=arguments.explain))
|
|
129
|
+
return check.exit_code(checked)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _codes(values: Iterable[str]) -> frozenset[str]:
|
|
133
|
+
"""The codes of repeated, comma-separated flag values."""
|
|
134
|
+
return frozenset(code.strip() for value in values for code in value.split(",") if code.strip())
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
if __name__ == "__main__":
|
|
138
|
+
sys.exit(main())
|
sqlassert/check.py
ADDED
|
@@ -0,0 +1,444 @@
|
|
|
1
|
+
"""`sqlassert check`: analyze SQL Programs for CI, and render what they find.
|
|
2
|
+
|
|
3
|
+
Each file is one SQL Program, analyzed from its SQL alone (`analyze`). Every
|
|
4
|
+
`UNKNOWN` assertion and every Diagnostic becomes a Finding; a Connection
|
|
5
|
+
Warning is a warning-level Finding and never fails the check, as it never fails
|
|
6
|
+
a `Report`. The exit codes, Findings and formatters are this module's
|
|
7
|
+
interface for any command that checks SQL (the CLI, the `sqlassert-dbt` plugin);
|
|
8
|
+
`__main__.py` only parses arguments.
|
|
9
|
+
|
|
10
|
+
Configuration comes from `[tool.sqlassert]` in the nearest `pyproject.toml`
|
|
11
|
+
that has one, searched from the working directory upwards; command-line flags
|
|
12
|
+
replace its values.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import glob
|
|
18
|
+
import json
|
|
19
|
+
import os
|
|
20
|
+
import sys
|
|
21
|
+
import tomllib
|
|
22
|
+
from collections.abc import Callable, Iterable, Sequence
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from enum import Enum
|
|
25
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from urllib.parse import quote
|
|
28
|
+
|
|
29
|
+
from sqlglot.dialects.dialect import Dialect
|
|
30
|
+
|
|
31
|
+
from sqlassert import diagnostics as diag
|
|
32
|
+
from sqlassert.diagnostics import Diagnostic
|
|
33
|
+
from sqlassert.main import DEFAULT_DIALECT, analyze
|
|
34
|
+
from sqlassert.provenance import Origin, Span
|
|
35
|
+
from sqlassert.reporting import ASSERTED_KINDS, Outcome, Report
|
|
36
|
+
|
|
37
|
+
# Exit codes, shared by every command that checks SQL.
|
|
38
|
+
EXIT_OK = 0 # every assertion PROVED and no Diagnostic (warnings allowed)
|
|
39
|
+
EXIT_FINDINGS = 1 # an assertion UNKNOWN or a Diagnostic reported
|
|
40
|
+
EXIT_USAGE = 2 # bad arguments or configuration, or SQL that could not be read
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def unproved(kind: str) -> str:
|
|
44
|
+
"""The code of the Finding an `UNKNOWN` assertion of this marker kind
|
|
45
|
+
produces: it names what was asserted and not proved, e.g. `unproved-unique`."""
|
|
46
|
+
return f"unproved-{kind}"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Every code a Finding can carry, so a mistyped ignored code is reported rather than ignored.
|
|
50
|
+
UNPROVED_CODES = frozenset(unproved(kind) for kind in ASSERTED_KINDS.values())
|
|
51
|
+
CODES = frozenset(
|
|
52
|
+
UNPROVED_CODES | {value for name, value in vars(diag).items() if name.isupper() and isinstance(value, str)}
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
STDIN = "-"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Severity(Enum):
|
|
59
|
+
"""An `error` fails the check; a `warning` (a Connection Warning) never does."""
|
|
60
|
+
|
|
61
|
+
ERROR = "error"
|
|
62
|
+
WARNING = "warning"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class Finding:
|
|
67
|
+
"""One thing a check reports: an `UNKNOWN` assertion or a Diagnostic.
|
|
68
|
+
|
|
69
|
+
`path` is the file as shown to the user. `evidence` is an `UNKNOWN`
|
|
70
|
+
assertion's explained Evidence, printed only when asked for.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
path: str
|
|
74
|
+
severity: Severity
|
|
75
|
+
code: str
|
|
76
|
+
message: str
|
|
77
|
+
origin: Origin | None = None
|
|
78
|
+
evidence: str | None = None
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(frozen=True)
|
|
82
|
+
class Checked:
|
|
83
|
+
"""One SQL Program's Report and the Findings it produced, ignored codes removed."""
|
|
84
|
+
|
|
85
|
+
path: str
|
|
86
|
+
report: Report
|
|
87
|
+
findings: tuple[Finding, ...]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class UsageError(Exception):
|
|
91
|
+
"""Arguments, configuration, or files that leave nothing to check: exit `EXIT_USAGE`."""
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
# Checking ------------------------------------------------------------------
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def findings(path: str, sql: str, report: Report, *, ignore: Iterable[str] = ()) -> tuple[Finding, ...]:
|
|
98
|
+
"""The Findings of one Report, in source order within each kind, codes in `ignore` left out."""
|
|
99
|
+
ignored = frozenset(ignore)
|
|
100
|
+
unknown = [
|
|
101
|
+
Finding(
|
|
102
|
+
path, Severity.ERROR, unproved(assertion.kind), f"{_written(sql, assertion.origin)} is not proved (UNKNOWN)",
|
|
103
|
+
assertion.origin, assertion.evidence.explain() if assertion.evidence is not None else None,
|
|
104
|
+
)
|
|
105
|
+
for assertion in report.assertions
|
|
106
|
+
if assertion.outcome is Outcome.UNKNOWN
|
|
107
|
+
]
|
|
108
|
+
reported = [_diagnostic(path, diagnostic, Severity.ERROR) for diagnostic in report.diagnostics]
|
|
109
|
+
warned = [_diagnostic(path, warning, Severity.WARNING) for warning in report.warnings]
|
|
110
|
+
return tuple(
|
|
111
|
+
finding for finding in sorted((*unknown, *reported, *warned), key=_source_order)
|
|
112
|
+
if finding.code not in ignored
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def check(path: str, sql: str, *, dialect: str = DEFAULT_DIALECT, ignore: Iterable[str] = ()) -> Checked:
|
|
117
|
+
"""Analyze one SQL Program from its SQL alone."""
|
|
118
|
+
report = analyze(sql, dialect=dialect)
|
|
119
|
+
return Checked(path, report, findings(path, sql, report, ignore=ignore))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def exit_code(checked: Iterable[Checked]) -> int:
|
|
123
|
+
"""`EXIT_FINDINGS` when any error-level Finding remains, else `EXIT_OK`."""
|
|
124
|
+
failed = any(finding.severity is Severity.ERROR for result in checked for finding in result.findings)
|
|
125
|
+
return EXIT_FINDINGS if failed else EXIT_OK
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _diagnostic(path: str, diagnostic: Diagnostic, severity: Severity) -> Finding:
|
|
129
|
+
return Finding(path, severity, diagnostic.code, diagnostic.message, diagnostic.origin)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _source_order(finding: Finding) -> tuple[int, int]:
|
|
133
|
+
"""Findings without a line first, then by line and column."""
|
|
134
|
+
origin = finding.origin
|
|
135
|
+
if origin is None or origin.line is None:
|
|
136
|
+
return 0, 0
|
|
137
|
+
return origin.line, origin.span.column if origin.span is not None else 0
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _written(sql: str, origin: Origin) -> str:
|
|
141
|
+
"""The marker as written at `origin`, whitespace collapsed; its detail when it has no span."""
|
|
142
|
+
if origin.span is None:
|
|
143
|
+
return origin.detail
|
|
144
|
+
return " ".join(_text(sql, origin.span).split())
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _text(sql: str, span: Span) -> str:
|
|
148
|
+
return sql[_offset(sql, span.line, span.column):_offset(sql, span.end_line, span.end_column)]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _offset(sql: str, line: int, column: int) -> int:
|
|
152
|
+
"""The inverse of `Span.of`: lines end at `\\n` only."""
|
|
153
|
+
start = 0
|
|
154
|
+
for _ in range(line - 1):
|
|
155
|
+
start = sql.index("\n", start) + 1
|
|
156
|
+
return start + column - 1
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
# Configuration ---------------------------------------------------------------
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
@dataclass(frozen=True)
|
|
163
|
+
class Config:
|
|
164
|
+
"""`[tool.sqlassert]`: `dialect`, `paths` (files, directories, globs;
|
|
165
|
+
relative to `root`), and `ignore` (codes whose Findings are dropped)."""
|
|
166
|
+
|
|
167
|
+
root: Path
|
|
168
|
+
dialect: str = DEFAULT_DIALECT
|
|
169
|
+
paths: tuple[str, ...] = ()
|
|
170
|
+
ignore: frozenset[str] = field(default_factory=frozenset)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
_CONFIG_KEYS = {"dialect": str, "paths": list, "ignore": list}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def load_config(start: Path) -> Config:
|
|
177
|
+
"""The `[tool.sqlassert]` table of the nearest `pyproject.toml` that has
|
|
178
|
+
one, from `start` upwards; defaults rooted at `start` when none does."""
|
|
179
|
+
for directory in (start, *start.parents):
|
|
180
|
+
pyproject = directory / "pyproject.toml"
|
|
181
|
+
if not pyproject.is_file():
|
|
182
|
+
continue
|
|
183
|
+
try:
|
|
184
|
+
table = tomllib.loads(pyproject.read_text()).get("tool", {}).get("sqlassert")
|
|
185
|
+
except (OSError, tomllib.TOMLDecodeError) as error:
|
|
186
|
+
raise UsageError(f"{pyproject}: {error}") from error
|
|
187
|
+
if table is not None:
|
|
188
|
+
return _config(pyproject, table)
|
|
189
|
+
return Config(start)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _config(pyproject: Path, table: object) -> Config:
|
|
193
|
+
if not isinstance(table, dict):
|
|
194
|
+
raise UsageError(f"{pyproject}: [tool.sqlassert] must be a table")
|
|
195
|
+
for key, value in table.items():
|
|
196
|
+
expected = _CONFIG_KEYS.get(key)
|
|
197
|
+
if expected is None:
|
|
198
|
+
raise UsageError(f"{pyproject}: unknown key {key!r} in [tool.sqlassert]; the keys are {', '.join(_CONFIG_KEYS)}")
|
|
199
|
+
if not isinstance(value, expected) or (expected is list and not all(isinstance(item, str) for item in value)):
|
|
200
|
+
shape = "a string" if expected is str else "a list of strings"
|
|
201
|
+
raise UsageError(f"{pyproject}: [tool.sqlassert] {key} must be {shape}")
|
|
202
|
+
return Config(
|
|
203
|
+
pyproject.parent,
|
|
204
|
+
dialect=table.get("dialect", DEFAULT_DIALECT),
|
|
205
|
+
paths=tuple(table.get("paths", ())),
|
|
206
|
+
ignore=frozenset(table.get("ignore", ())),
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def validate(dialect: str, ignore: Iterable[str]) -> None:
|
|
211
|
+
"""Reject a dialect SQLGlot does not know, and an ignored code no Finding carries."""
|
|
212
|
+
try:
|
|
213
|
+
Dialect.get_or_raise(dialect)
|
|
214
|
+
except ValueError as error:
|
|
215
|
+
raise UsageError(str(error)) from error
|
|
216
|
+
unknown = sorted(set(ignore) - CODES)
|
|
217
|
+
if unknown:
|
|
218
|
+
raise UsageError(f"unknown code(s) to ignore: {', '.join(unknown)}; the codes are {', '.join(sorted(CODES))}")
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def expand(patterns: Sequence[str], root: Path) -> list[Path]:
|
|
222
|
+
"""The SQL files `patterns` name, relative to `root`, in order and without repeats.
|
|
223
|
+
|
|
224
|
+
A file is itself; a directory is every `*.sql` under it; anything with a
|
|
225
|
+
glob character is a recursive glob (`**` crosses directories). A pattern
|
|
226
|
+
that names no file is a usage error: a check that silently checks nothing
|
|
227
|
+
would pass.
|
|
228
|
+
"""
|
|
229
|
+
found: dict[Path, None] = {}
|
|
230
|
+
for pattern in patterns:
|
|
231
|
+
if pattern == STDIN:
|
|
232
|
+
found[Path(STDIN)] = None
|
|
233
|
+
continue
|
|
234
|
+
path = root / pattern
|
|
235
|
+
if glob.has_magic(pattern):
|
|
236
|
+
matches = sorted(Path(match) for match in glob.glob(str(path), recursive=True) if Path(match).is_file())
|
|
237
|
+
elif path.is_dir():
|
|
238
|
+
matches = sorted(path.rglob("*.sql"))
|
|
239
|
+
elif path.is_file():
|
|
240
|
+
matches = [path]
|
|
241
|
+
else:
|
|
242
|
+
raise UsageError(f"{pattern}: no such file or directory")
|
|
243
|
+
if not matches:
|
|
244
|
+
raise UsageError(f"{pattern} matches no SQL files")
|
|
245
|
+
found.update(dict.fromkeys(matches))
|
|
246
|
+
return list(found)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def display(path: Path) -> str:
|
|
250
|
+
"""How a checked file is named in output: relative to the working directory when inside it."""
|
|
251
|
+
if path == Path(STDIN):
|
|
252
|
+
return "<stdin>"
|
|
253
|
+
try:
|
|
254
|
+
return path.resolve().relative_to(Path.cwd().resolve()).as_posix()
|
|
255
|
+
except ValueError:
|
|
256
|
+
return str(path)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def read_sql(path: Path) -> str:
|
|
260
|
+
"""A SQL Program's text; `-` reads stdin."""
|
|
261
|
+
if path == Path(STDIN):
|
|
262
|
+
return sys.stdin.read()
|
|
263
|
+
try:
|
|
264
|
+
return path.read_text()
|
|
265
|
+
except (OSError, UnicodeDecodeError) as error:
|
|
266
|
+
raise UsageError(f"{path}: {error}") from error
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
# Output --------------------------------------------------------------------
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def indented(text: str, prefix: str = " ") -> str:
|
|
273
|
+
"""Explained Evidence, set under the line it explains."""
|
|
274
|
+
return "\n".join(prefix + line for line in text.splitlines())
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def render_text(checked: Sequence[Checked], *, explain: bool = False) -> str:
|
|
278
|
+
"""`path:line:column: severity code: message`, then a summary line."""
|
|
279
|
+
lines = []
|
|
280
|
+
for result in checked:
|
|
281
|
+
for finding in result.findings:
|
|
282
|
+
lines.append(f"{_place(finding)}: {finding.severity.value} {finding.code}: {finding.message}")
|
|
283
|
+
if explain and finding.evidence is not None:
|
|
284
|
+
lines.append(indented(finding.evidence))
|
|
285
|
+
lines.append(_summary_line(checked))
|
|
286
|
+
return "\n".join(lines) + "\n"
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def render_json(checked: Sequence[Checked], *, explain: bool = False) -> str:
|
|
290
|
+
"""Every assertion's outcome and every Finding, as one JSON document."""
|
|
291
|
+
def finding(item: Finding) -> dict:
|
|
292
|
+
shown = {
|
|
293
|
+
"path": item.path, "severity": item.severity.value, "code": item.code, "message": item.message,
|
|
294
|
+
"location": _location(item.origin),
|
|
295
|
+
}
|
|
296
|
+
if explain and item.evidence is not None:
|
|
297
|
+
shown["evidence"] = item.evidence
|
|
298
|
+
return shown
|
|
299
|
+
|
|
300
|
+
document = {
|
|
301
|
+
"assertions": [
|
|
302
|
+
{"path": result.path, "id": assertion.assertion_id, "outcome": assertion.outcome.value, "location": _location(assertion.origin)}
|
|
303
|
+
for result in checked for assertion in result.report.assertions
|
|
304
|
+
],
|
|
305
|
+
"findings": [finding(item) for result in checked for item in result.findings],
|
|
306
|
+
"summary": _summary(checked),
|
|
307
|
+
}
|
|
308
|
+
return json.dumps(document, indent=2) + "\n"
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def render_sarif(checked: Sequence[Checked], *, explain: bool = False) -> str:
|
|
312
|
+
"""A SARIF 2.1.0 log, as GitHub code scanning reads it. Columns count characters."""
|
|
313
|
+
found = [finding for result in checked for finding in result.findings]
|
|
314
|
+
codes = sorted({finding.code for finding in found})
|
|
315
|
+
driver: dict = {"name": "sqlassert", "informationUri": "https://github.com/polvoazul/sqlassert", "rules": [
|
|
316
|
+
{"id": code, "shortDescription": {"text": _RULE_DESCRIPTIONS.get(code, code.replace("-", " "))}}
|
|
317
|
+
for code in codes
|
|
318
|
+
]}
|
|
319
|
+
try:
|
|
320
|
+
driver["version"] = version("sqlassert")
|
|
321
|
+
except PackageNotFoundError:
|
|
322
|
+
pass
|
|
323
|
+
|
|
324
|
+
def result(finding: Finding) -> dict:
|
|
325
|
+
message = finding.message
|
|
326
|
+
if explain and finding.evidence is not None:
|
|
327
|
+
message += "\n\n" + finding.evidence
|
|
328
|
+
location: dict = {"artifactLocation": {"uri": _uri(finding.path)}}
|
|
329
|
+
origin = finding.origin
|
|
330
|
+
if origin is not None and origin.line is not None:
|
|
331
|
+
region: dict = {"startLine": origin.line}
|
|
332
|
+
if origin.span is not None:
|
|
333
|
+
region.update(
|
|
334
|
+
startColumn=origin.span.column, endLine=origin.span.end_line, endColumn=origin.span.end_column
|
|
335
|
+
)
|
|
336
|
+
location["region"] = region
|
|
337
|
+
return {
|
|
338
|
+
"ruleId": finding.code,
|
|
339
|
+
"ruleIndex": codes.index(finding.code),
|
|
340
|
+
"level": finding.severity.value,
|
|
341
|
+
"message": {"text": message},
|
|
342
|
+
"locations": [{"physicalLocation": location}],
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
log = {
|
|
346
|
+
"$schema": "https://json.schemastore.org/sarif-2.1.0.json",
|
|
347
|
+
"version": "2.1.0",
|
|
348
|
+
"runs": [{"tool": {"driver": driver}, "columnKind": "unicodeCodePoints", "results": [result(f) for f in found]}],
|
|
349
|
+
}
|
|
350
|
+
return json.dumps(log, indent=2) + "\n"
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def render_github(checked: Sequence[Checked], *, explain: bool = False) -> str:
|
|
354
|
+
"""GitHub Actions workflow commands: one `::error` or `::warning` annotation per Finding."""
|
|
355
|
+
lines = []
|
|
356
|
+
for result in checked:
|
|
357
|
+
for finding in result.findings:
|
|
358
|
+
properties = {"file": finding.path}
|
|
359
|
+
origin = finding.origin
|
|
360
|
+
if origin is not None and origin.line is not None:
|
|
361
|
+
properties["line"] = str(origin.line)
|
|
362
|
+
span = origin.span
|
|
363
|
+
if span is not None:
|
|
364
|
+
properties["endLine"] = str(span.end_line)
|
|
365
|
+
# GitHub honours columns only within one line.
|
|
366
|
+
if span.end_line == span.line:
|
|
367
|
+
properties.update(col=str(span.column), endColumn=str(span.end_column))
|
|
368
|
+
properties["title"] = f"sqlassert {finding.code}"
|
|
369
|
+
message = finding.message
|
|
370
|
+
if explain and finding.evidence is not None:
|
|
371
|
+
message += "\n" + finding.evidence
|
|
372
|
+
shown = ",".join(f"{key}={_escape_property(value)}" for key, value in properties.items())
|
|
373
|
+
lines.append(f"::{finding.severity.value} {shown}::{_escape_data(message)}")
|
|
374
|
+
return "".join(line + "\n" for line in lines)
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
FORMATS: dict[str, Callable[..., str]] = {
|
|
378
|
+
"text": render_text,
|
|
379
|
+
"json": render_json,
|
|
380
|
+
"sarif": render_sarif,
|
|
381
|
+
"github": render_github,
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
_RULE_DESCRIPTIONS = {
|
|
385
|
+
unproved(kind): f"An asserted {kind.replace('-', ' ')} Property is not proved (UNKNOWN)" for kind in ASSERTED_KINDS.values()
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def _place(finding: Finding) -> str:
|
|
390
|
+
origin = finding.origin
|
|
391
|
+
if origin is None or origin.line is None:
|
|
392
|
+
return finding.path
|
|
393
|
+
if origin.span is None:
|
|
394
|
+
return f"{finding.path}:{origin.line}"
|
|
395
|
+
return f"{finding.path}:{origin.line}:{origin.span.column}"
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def _location(origin: Origin | None) -> dict | None:
|
|
399
|
+
if origin is None or origin.line is None:
|
|
400
|
+
return None
|
|
401
|
+
if origin.span is None:
|
|
402
|
+
return {"line": origin.line}
|
|
403
|
+
span = origin.span
|
|
404
|
+
return {"line": span.line, "column": span.column, "end_line": span.end_line, "end_column": span.end_column}
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def _summary(checked: Sequence[Checked]) -> dict:
|
|
408
|
+
assertions = [assertion for result in checked for assertion in result.report.assertions]
|
|
409
|
+
found = [finding for result in checked for finding in result.findings]
|
|
410
|
+
return {
|
|
411
|
+
"files": len(checked),
|
|
412
|
+
"assertions": len(assertions),
|
|
413
|
+
"proved": sum(assertion.outcome is Outcome.PROVED for assertion in assertions),
|
|
414
|
+
"unknown": sum(assertion.outcome is Outcome.UNKNOWN for assertion in assertions),
|
|
415
|
+
"errors": sum(finding.severity is Severity.ERROR for finding in found),
|
|
416
|
+
"warnings": sum(finding.severity is Severity.WARNING for finding in found),
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _summary_line(checked: Sequence[Checked]) -> str:
|
|
421
|
+
counts = _summary(checked)
|
|
422
|
+
|
|
423
|
+
def plural(count: int, noun: str) -> str:
|
|
424
|
+
return f"{count} {noun}{'' if count == 1 else 's'}"
|
|
425
|
+
|
|
426
|
+
return (
|
|
427
|
+
f"{plural(counts['files'], 'file')} checked: {plural(counts['assertions'], 'assertion')} "
|
|
428
|
+
f"({counts['proved']} PROVED, {counts['unknown']} UNKNOWN), "
|
|
429
|
+
f"{plural(counts['errors'], 'error')}, {plural(counts['warnings'], 'warning')}"
|
|
430
|
+
)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _uri(path: str) -> str:
|
|
434
|
+
if os.path.isabs(path):
|
|
435
|
+
return Path(path).as_uri()
|
|
436
|
+
return quote(path)
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _escape_data(text: str) -> str:
|
|
440
|
+
return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def _escape_property(text: str) -> str:
|
|
444
|
+
return _escape_data(text).replace(":", "%3A").replace(",", "%2C")
|
sqlassert/diagnostics.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Explicit statements about what analysis could not do.
|
|
2
|
+
|
|
3
|
+
A diagnostic reports a program the engine does not support. It is never a
|
|
4
|
+
Disproof and never an inference: it says that analysis stopped short, and why.
|
|
5
|
+
|
|
6
|
+
A warning is a Diagnostic too, reported apart (`Report.warnings`): the program
|
|
7
|
+
is supported, but a fact a connection would supply is missing, so analysis
|
|
8
|
+
treats it as unknown (ADR 0005). So are a constraint the dialect does not
|
|
9
|
+
enforce, and, about the SQL itself (`SQL_WARNINGS`), an operator whose
|
|
10
|
+
meaning a setting decides, a comparison that depends on something the
|
|
11
|
+
values do not show, like a zoned timestamp's zone (`dialects.py`), and a
|
|
12
|
+
comparison of composites whose elements may be NULL.
|
|
13
|
+
Warnings never make a Report fail.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
|
|
20
|
+
from sqlassert.provenance import Origin
|
|
21
|
+
|
|
22
|
+
CATALOG_METADATA_UNAVAILABLE = "catalog-metadata-unavailable"
|
|
23
|
+
AMBIGUOUS_COLUMN = "ambiguous-column"
|
|
24
|
+
AMBIGUOUS_OPERATOR = "ambiguous-operator"
|
|
25
|
+
CARDINALITY_ON_FILTERING_JOIN = "cardinality-on-filtering-join"
|
|
26
|
+
CATALOG_RELATION_NOT_FOUND = "catalog-relation-not-found"
|
|
27
|
+
COMPOSITE_COMPARISON = "composite-comparison"
|
|
28
|
+
CONSTRAINT_WITHOUT_TABLE = "constraint-without-table"
|
|
29
|
+
DUPLICATE_DECLARATION = "duplicate-declaration"
|
|
30
|
+
INCOMPLETE_CATALOG_METADATA = "incomplete-catalog-metadata"
|
|
31
|
+
INVALID_FOREIGN_KEY_CONSTRAINT = "invalid-foreign-key-constraint"
|
|
32
|
+
MULTIPLE_ROOT_SELECTS = "multiple-root-selects"
|
|
33
|
+
RECURSIVE_VIEW_DEFINITION = "recursive-view-definition"
|
|
34
|
+
SQL_PARSE_FAILED = "sql-parse-failed"
|
|
35
|
+
UNANALYZED_ASSERTION = "unanalyzed-assertion"
|
|
36
|
+
UNATTACHED_MARKER = "unattached-marker"
|
|
37
|
+
UNDECLARED_TABLE = "undeclared-table"
|
|
38
|
+
UNENFORCED_CONSTRAINT = "unenforced-constraint"
|
|
39
|
+
UNKNOWN_ASSERTED_COLUMN = "unknown-asserted-column"
|
|
40
|
+
UNRESOLVED_ASSERTED_COLUMN = "unresolved-asserted-column"
|
|
41
|
+
UNKNOWN_REFERENCED_COLUMN = "unknown-referenced-column"
|
|
42
|
+
UNRECOGNIZED_MARKER = "unrecognized-marker"
|
|
43
|
+
UNSUPPORTED_CREATE_STATEMENT = "unsupported-create-statement"
|
|
44
|
+
UNSUPPORTED_NATURAL_JOIN = "unsupported-natural-join"
|
|
45
|
+
UNSUPPORTED_STATEMENT = "unsupported-statement"
|
|
46
|
+
ZONED_TIMESTAMP_CAST = "zoned-timestamp-cast"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@dataclass(frozen=True)
|
|
50
|
+
class Diagnostic:
|
|
51
|
+
code: str
|
|
52
|
+
message: str
|
|
53
|
+
origin: Origin | None = None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
# Warnings about the SQL itself, which no connection answers: `analyze_database` keeps them.
|
|
57
|
+
SQL_WARNINGS = frozenset({AMBIGUOUS_OPERATOR, COMPOSITE_COMPARISON, ZONED_TIMESTAMP_CAST})
|