sqlakit 0.22.3__tar.gz → 0.22.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sqlakit-0.22.3 → sqlakit-0.22.5}/PKG-INFO +1 -1
- {sqlakit-0.22.3 → sqlakit-0.22.5}/pyproject.toml +1 -1
- {sqlakit-0.22.3 → sqlakit-0.22.5}/pyproject.toml.orig +1 -1
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_cli.py +159 -6
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_linters.py +118 -1
- sqlakit-0.22.5/sqlakit/_render.py +379 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/LICENSE +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/README.md +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/__init__.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_base.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_db.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_discovery.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_model.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_project.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_pycharm.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_query.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_recording.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_registry.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_routing.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_sql.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_sql_formatter.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/_static.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/asyncio/__init__.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/asyncio/_db.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/asyncio/_registry.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/asyncio/orm.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/asyncio/sql.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/exceptions.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/orm.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/py.typed +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/pytest_plugin.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/sql.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/testing.py +0 -0
- {sqlakit-0.22.3 → sqlakit-0.22.5}/sqlakit/types.py +0 -0
|
@@ -5,18 +5,20 @@ from __future__ import annotations
|
|
|
5
5
|
import argparse
|
|
6
6
|
import json
|
|
7
7
|
import os
|
|
8
|
+
import shutil
|
|
8
9
|
import sys
|
|
9
10
|
from pathlib import Path
|
|
10
11
|
|
|
11
12
|
from . import _sql_formatter
|
|
12
|
-
from ._linters import LINTERS, settings, stale
|
|
13
|
+
from ._linters import LINTERS, dialect_of, settings, stale
|
|
13
14
|
from ._project import Problem, Project, load_project
|
|
14
15
|
from ._pycharm import DIALECTS, ddl, dialects
|
|
16
|
+
from ._render import import_macros, rendered, unparsed
|
|
15
17
|
from ._sql import NAMESPACE, registered, signature_of
|
|
16
18
|
from .exceptions import MacroDefinitionError, ProjectConfigError, UnknownImportPathError
|
|
17
19
|
|
|
18
20
|
|
|
19
|
-
def main(argv: list[str] | None = None) -> int:
|
|
21
|
+
def main(argv: list[str] | None = None) -> int: # noqa: PLR0911 - one return a command
|
|
20
22
|
"""Run a command, and return what the shell should make of it."""
|
|
21
23
|
parser = argparse.ArgumentParser(prog="sqlakit", description="SQLAKit.")
|
|
22
24
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
@@ -52,6 +54,31 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
52
54
|
help="a directory in the project, the current one by default",
|
|
53
55
|
)
|
|
54
56
|
check.add_argument("--format", choices=("text", "json"), default="text")
|
|
57
|
+
check.add_argument(
|
|
58
|
+
"--lint",
|
|
59
|
+
choices=tuple(LINTERS),
|
|
60
|
+
help="also render every template, running the project's macros, and "
|
|
61
|
+
"fail where the linter cannot parse the SQL",
|
|
62
|
+
)
|
|
63
|
+
check.add_argument(
|
|
64
|
+
"--dialect", help="the dialect to render for, in place of the project's"
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
render = commands.add_parser(
|
|
68
|
+
"render",
|
|
69
|
+
help="write every template as the SQL it renders, for a linter to read",
|
|
70
|
+
)
|
|
71
|
+
render.add_argument(
|
|
72
|
+
"--out", required=True, help="the directory to write the rendered SQL into"
|
|
73
|
+
)
|
|
74
|
+
render.add_argument(
|
|
75
|
+
"--project",
|
|
76
|
+
default=".",
|
|
77
|
+
help="a directory in the project, the current one by default",
|
|
78
|
+
)
|
|
79
|
+
render.add_argument(
|
|
80
|
+
"--dialect", help="the dialect to render for, in place of the project's"
|
|
81
|
+
)
|
|
55
82
|
|
|
56
83
|
export = commands.add_parser(
|
|
57
84
|
"export", help="write what another tool needs to read the templates"
|
|
@@ -73,7 +100,16 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
73
100
|
|
|
74
101
|
arguments = parser.parse_args(argv)
|
|
75
102
|
if arguments.command == "check":
|
|
76
|
-
return _check(
|
|
103
|
+
return _check(
|
|
104
|
+
Path(arguments.project),
|
|
105
|
+
json_output=arguments.format == "json",
|
|
106
|
+
lint=arguments.lint,
|
|
107
|
+
dialect=arguments.dialect,
|
|
108
|
+
)
|
|
109
|
+
if arguments.command == "render":
|
|
110
|
+
return _render(
|
|
111
|
+
Path(arguments.project), Path(arguments.out), dialect=arguments.dialect
|
|
112
|
+
)
|
|
77
113
|
if arguments.command == "export":
|
|
78
114
|
if arguments.tool == "sql-formatter":
|
|
79
115
|
return _export_sql_formatter(
|
|
@@ -148,11 +184,14 @@ def _load(directory: Path, *, json_output: bool = False) -> Project | None:
|
|
|
148
184
|
return None
|
|
149
185
|
|
|
150
186
|
|
|
151
|
-
def _check(
|
|
187
|
+
def _check(
|
|
188
|
+
directory: Path, *, json_output: bool, lint: str | None, dialect: str | None
|
|
189
|
+
) -> int:
|
|
152
190
|
"""Print what is wrong with the project's templates, and fail if anything is.
|
|
153
191
|
|
|
154
|
-
|
|
155
|
-
|
|
192
|
+
With a linter, every template is rendered too, and what it cannot parse is
|
|
193
|
+
a problem. The exit is 1 for problems in the templates, and 2 for a project
|
|
194
|
+
that cannot be read, holds no template to check, or cannot be linted.
|
|
156
195
|
"""
|
|
157
196
|
project = _load(directory, json_output=json_output)
|
|
158
197
|
if project is None:
|
|
@@ -163,6 +202,12 @@ def _check(directory: Path, *, json_output: bool) -> int:
|
|
|
163
202
|
_say(json.dumps({"error": message}) if json_output else message)
|
|
164
203
|
return 2
|
|
165
204
|
problems = list(project.problems())
|
|
205
|
+
if lint is not None:
|
|
206
|
+
try:
|
|
207
|
+
problems.extend(_lint(project, lint, dialect))
|
|
208
|
+
except _CannotRenderError as error:
|
|
209
|
+
_say(json.dumps({"error": str(error)}) if json_output else str(error))
|
|
210
|
+
return 2
|
|
166
211
|
if json_output:
|
|
167
212
|
_say(json.dumps([_as_json(problem) for problem in problems], indent=2))
|
|
168
213
|
else:
|
|
@@ -180,6 +225,114 @@ def _check(directory: Path, *, json_output: bool) -> int:
|
|
|
180
225
|
return 1 if problems else 0
|
|
181
226
|
|
|
182
227
|
|
|
228
|
+
_WRITTEN = {"given": "every parameter given", "not_given": "no parameter given"}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class _CannotRenderError(Exception):
|
|
232
|
+
"""Why the templates cannot be rendered, or the rendering linted."""
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _lint(project: Project, name: str, given: str | None) -> list[Problem]:
|
|
236
|
+
"""Return where the linter cannot parse what the templates render.
|
|
237
|
+
|
|
238
|
+
Raises:
|
|
239
|
+
_CannotRenderError: if the linter cannot read the templates as the
|
|
240
|
+
project is set up.
|
|
241
|
+
|
|
242
|
+
"""
|
|
243
|
+
linter = LINTERS[name]
|
|
244
|
+
if shutil.which(name) is None:
|
|
245
|
+
problem = f"`{name}` is not installed, so the rendered SQL cannot be linted"
|
|
246
|
+
raise _CannotRenderError(problem)
|
|
247
|
+
pyproject = project.root / "pyproject.toml"
|
|
248
|
+
text = pyproject.read_text(encoding="utf-8") if pyproject.exists() else ""
|
|
249
|
+
if stale(linter, project, text):
|
|
250
|
+
problem = (
|
|
251
|
+
f"the `{name}` settings do not read the templates: run "
|
|
252
|
+
f"`sqlakit export {name}` first"
|
|
253
|
+
)
|
|
254
|
+
raise _CannotRenderError(problem)
|
|
255
|
+
configured = dialect_of(linter, text)
|
|
256
|
+
chosen = _dialect_to_render(project, given, configured)
|
|
257
|
+
if configured is not None and configured != chosen:
|
|
258
|
+
problem = (
|
|
259
|
+
f"the templates render on {chosen}, and `{name}` reads {configured}: "
|
|
260
|
+
f"pass --dialect {configured}, or set the dialect in `[tool.{name}.core]`"
|
|
261
|
+
)
|
|
262
|
+
raise _CannotRenderError(problem)
|
|
263
|
+
try:
|
|
264
|
+
import_macros(project)
|
|
265
|
+
except Exception as error:
|
|
266
|
+
problem = (
|
|
267
|
+
f"the project's macros cannot be imported: {type(error).__name__}: {error}"
|
|
268
|
+
)
|
|
269
|
+
raise _CannotRenderError(problem) from error
|
|
270
|
+
return unparsed(project, linter, chosen)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _dialect_to_render(
|
|
274
|
+
project: Project, given: str | None, configured: str | None = None
|
|
275
|
+
) -> str:
|
|
276
|
+
"""Return the dialect to render the templates on.
|
|
277
|
+
|
|
278
|
+
`--dialect` wins, then the one the project's database URL names, then the
|
|
279
|
+
one a linter's settings read in. A macro writes by `ctx.dialect`, so a
|
|
280
|
+
guess would render SQL the database never gets.
|
|
281
|
+
|
|
282
|
+
Raises:
|
|
283
|
+
_CannotRenderError: if nothing names one.
|
|
284
|
+
|
|
285
|
+
"""
|
|
286
|
+
chosen = given or project.dialect or configured
|
|
287
|
+
if chosen is None:
|
|
288
|
+
problem = (
|
|
289
|
+
"the code names no database URL to take the dialect from: pass --dialect"
|
|
290
|
+
)
|
|
291
|
+
raise _CannotRenderError(problem)
|
|
292
|
+
return chosen.lower()
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _render(directory: Path, out: Path, *, dialect: str | None) -> int:
|
|
296
|
+
"""Write each template twice, rendered with its parameters given and not.
|
|
297
|
+
|
|
298
|
+
The project's Python macros are imported and run, so what they write is
|
|
299
|
+
in the SQL. Exit 1 when a template cannot be rendered.
|
|
300
|
+
"""
|
|
301
|
+
project = _load(directory)
|
|
302
|
+
if project is None:
|
|
303
|
+
return 2
|
|
304
|
+
try:
|
|
305
|
+
import_macros(project)
|
|
306
|
+
except Exception as error: # noqa: BLE001 - the project's code raises anything
|
|
307
|
+
_say(
|
|
308
|
+
f"the project's macros cannot be imported: {type(error).__name__}: {error}"
|
|
309
|
+
)
|
|
310
|
+
return 2
|
|
311
|
+
try:
|
|
312
|
+
chosen = _dialect_to_render(project, dialect)
|
|
313
|
+
except _CannotRenderError as error:
|
|
314
|
+
_say(str(error))
|
|
315
|
+
return 2
|
|
316
|
+
problems = 0
|
|
317
|
+
count = 0
|
|
318
|
+
for one in rendered(project, chosen):
|
|
319
|
+
if one.sql is None:
|
|
320
|
+
problems += 1
|
|
321
|
+
_say(f"{one.name} ({one.variant}): {one.problem}")
|
|
322
|
+
continue
|
|
323
|
+
target = out / f"{one.name.removesuffix('.sql')}.{one.variant}.sql"
|
|
324
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
325
|
+
header = [f"-- {one.name} on {chosen}, {_WRITTEN[one.variant]}"]
|
|
326
|
+
if one.kept:
|
|
327
|
+
header.append(
|
|
328
|
+
f"-- {', '.join(one.kept)} could not be made, and stays a call"
|
|
329
|
+
)
|
|
330
|
+
target.write_text("\n".join(header) + "\n" + one.sql, encoding="utf-8")
|
|
331
|
+
count += 1
|
|
332
|
+
_say(f"wrote {count} files to {_relative(out)}")
|
|
333
|
+
return 1 if problems else 0
|
|
334
|
+
|
|
335
|
+
|
|
183
336
|
def _export_linter(
|
|
184
337
|
name: str, directory: Path, *, dialect: str | None, check: bool
|
|
185
338
|
) -> int:
|
|
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
|
|
|
27
27
|
|
|
28
28
|
from ._project import Project
|
|
29
29
|
|
|
30
|
-
__all__ = ["LINTERS", "Linter", "settings", "stale"]
|
|
30
|
+
__all__ = ["LINTERS", "Linter", "dialect_of", "parse_errors", "settings", "stale"]
|
|
31
31
|
|
|
32
32
|
_NEXT_TABLE = re.compile(r"^\[", re.MULTILINE)
|
|
33
33
|
|
|
@@ -47,6 +47,9 @@ class Linter:
|
|
|
47
47
|
kept: Callable[[str, list[str], str | None], set[int]]
|
|
48
48
|
"""Given the executable, the names and the dialect, the index of each name it
|
|
49
49
|
cannot read as a column."""
|
|
50
|
+
unparsed: Callable[[str, Path, Path], dict[Path, list[tuple[int, int]]]]
|
|
51
|
+
"""Given the executable, a directory of SQL and the project's root, where in
|
|
52
|
+
each file it could not parse: the line and the column, from one."""
|
|
50
53
|
|
|
51
54
|
@property
|
|
52
55
|
def placeholder(self) -> str:
|
|
@@ -130,6 +133,118 @@ def settings(
|
|
|
130
133
|
return written_out
|
|
131
134
|
|
|
132
135
|
|
|
136
|
+
def dialect_of(linter: Linter, pyproject: str) -> str | None:
|
|
137
|
+
"""Return the dialect the linter's settings read in, by SQLAlchemy's name."""
|
|
138
|
+
tables = tomllib.loads(pyproject).get("tool", {}).get(linter.name, {})
|
|
139
|
+
written = tables.get("core", {}).get("dialect")
|
|
140
|
+
if not isinstance(written, str):
|
|
141
|
+
return None
|
|
142
|
+
named = {theirs: ours for ours, theirs in linter.dialects.items()}
|
|
143
|
+
return named.get(written, written)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def parse_errors(
|
|
147
|
+
linter: Linter, folder: Path, root: Path
|
|
148
|
+
) -> dict[Path, list[tuple[int, int]]]:
|
|
149
|
+
"""Return where the linter could not parse each file of a directory.
|
|
150
|
+
|
|
151
|
+
It runs in the project's root, so it reads the project's settings, and an
|
|
152
|
+
empty result says the linter is not installed.
|
|
153
|
+
"""
|
|
154
|
+
binary = shutil.which(linter.name)
|
|
155
|
+
if binary is None:
|
|
156
|
+
return {}
|
|
157
|
+
return linter.unparsed(binary, folder, root)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
_CHEAP_RULE = "LT12"
|
|
161
|
+
"""The one rule a run for parse errors keeps: it reads only a file's end, and a
|
|
162
|
+
linter parses the whole file whatever rules it runs."""
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _sqruff_unparsed(
|
|
166
|
+
binary: str, folder: Path, root: Path
|
|
167
|
+
) -> dict[Path, list[tuple[int, int]]]:
|
|
168
|
+
pyproject = root / "pyproject.toml"
|
|
169
|
+
text = pyproject.read_text(encoding="utf-8") if pyproject.exists() else ""
|
|
170
|
+
tables = tomllib.loads(text).get("tool", {}).get("sqruff", {})
|
|
171
|
+
core = tables.get("core", {})
|
|
172
|
+
placeholder = tables.get("templater", {}).get("placeholder", {})
|
|
173
|
+
# The project's reading of the templates, and none of its rules: sqruff
|
|
174
|
+
# takes no rules on its command line, so a file of settings of its own.
|
|
175
|
+
settings = [
|
|
176
|
+
"[sqruff]",
|
|
177
|
+
*([f"dialect = {core['dialect']}"] if "dialect" in core else []),
|
|
178
|
+
f"templater = {core.get('templater', 'placeholder')}",
|
|
179
|
+
f"rules = {_CHEAP_RULE}",
|
|
180
|
+
"",
|
|
181
|
+
"[sqruff:templater:placeholder]",
|
|
182
|
+
*(f"{name} = {value}" for name, value in placeholder.items()),
|
|
183
|
+
]
|
|
184
|
+
with tempfile.TemporaryDirectory() as scratch:
|
|
185
|
+
config = Path(scratch, ".sqruff")
|
|
186
|
+
config.write_text("\n".join(settings) + "\n", encoding="utf-8")
|
|
187
|
+
found = subprocess.run( # noqa: S603 - sqruff, found on the PATH
|
|
188
|
+
[
|
|
189
|
+
binary,
|
|
190
|
+
"lint",
|
|
191
|
+
"--config",
|
|
192
|
+
str(config),
|
|
193
|
+
"--parsing-errors",
|
|
194
|
+
"--format",
|
|
195
|
+
"json",
|
|
196
|
+
str(folder),
|
|
197
|
+
],
|
|
198
|
+
capture_output=True,
|
|
199
|
+
text=True,
|
|
200
|
+
cwd=root,
|
|
201
|
+
check=False,
|
|
202
|
+
)
|
|
203
|
+
return {
|
|
204
|
+
Path(path).resolve(): [
|
|
205
|
+
(problem["range"]["start"]["line"], problem["range"]["start"]["character"])
|
|
206
|
+
for problem in problems
|
|
207
|
+
# A problem of parsing has no rule, and so no code.
|
|
208
|
+
if problem.get("code") is None
|
|
209
|
+
]
|
|
210
|
+
for path, problems in _report(found.stdout, found.stderr, shape=dict).items()
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _sqlfluff_unparsed(
|
|
215
|
+
binary: str, folder: Path, root: Path
|
|
216
|
+
) -> dict[Path, list[tuple[int, int]]]:
|
|
217
|
+
found = subprocess.run( # noqa: S603 - sqlfluff, found on the PATH
|
|
218
|
+
# A file outside the project reads no setting of it unless told where.
|
|
219
|
+
[
|
|
220
|
+
binary,
|
|
221
|
+
"lint",
|
|
222
|
+
"--config",
|
|
223
|
+
str(root / "pyproject.toml"),
|
|
224
|
+
# Only parse errors count: one cheap rule, run on every core.
|
|
225
|
+
"--rules",
|
|
226
|
+
_CHEAP_RULE,
|
|
227
|
+
"--processes",
|
|
228
|
+
"-1",
|
|
229
|
+
"--format",
|
|
230
|
+
"json",
|
|
231
|
+
str(folder),
|
|
232
|
+
],
|
|
233
|
+
capture_output=True,
|
|
234
|
+
text=True,
|
|
235
|
+
cwd=root,
|
|
236
|
+
check=False,
|
|
237
|
+
)
|
|
238
|
+
return {
|
|
239
|
+
Path(report["filepath"]).resolve(): [
|
|
240
|
+
(problem["start_line_no"], problem["start_line_pos"])
|
|
241
|
+
for problem in report.get("violations", ())
|
|
242
|
+
if problem.get("code") == "PRS"
|
|
243
|
+
]
|
|
244
|
+
for report in _report(found.stdout, found.stderr, shape=list)
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
|
|
133
248
|
def _dialect(
|
|
134
249
|
linter: Linter, tables: dict, given: str | None, project: Project
|
|
135
250
|
) -> str | None:
|
|
@@ -242,6 +357,7 @@ SQRUFF = Linter(
|
|
|
242
357
|
# sqruff reads settings without a dialect as ANSI.
|
|
243
358
|
default_dialect=None,
|
|
244
359
|
kept=_sqruff_kept,
|
|
360
|
+
unparsed=_sqruff_unparsed,
|
|
245
361
|
)
|
|
246
362
|
|
|
247
363
|
SQLFLUFF = Linter(
|
|
@@ -252,6 +368,7 @@ SQLFLUFF = Linter(
|
|
|
252
368
|
# sqlfluff reads no template without a dialect.
|
|
253
369
|
default_dialect="ansi",
|
|
254
370
|
kept=_sqlfluff_kept,
|
|
371
|
+
unparsed=_sqlfluff_unparsed,
|
|
255
372
|
)
|
|
256
373
|
|
|
257
374
|
LINTERS = {linter.name: linter for linter in (SQRUFF, SQLFLUFF)}
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
"""Every template of a project, rendered, for a linter to read.
|
|
2
|
+
|
|
3
|
+
A macro written in Python is SQL only once it runs, so a linter reading the
|
|
4
|
+
templates reads its call and never what it writes. `rendered` runs the macros:
|
|
5
|
+
each template is written twice, once with every parameter given a made-up
|
|
6
|
+
value and once with none, as a macro decides its SQL by what it is given.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import re
|
|
12
|
+
import sys
|
|
13
|
+
import tempfile
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import TYPE_CHECKING, Any
|
|
17
|
+
|
|
18
|
+
import sqlalchemy as sa
|
|
19
|
+
import sqlalchemy.engine.default
|
|
20
|
+
import sqlalchemy.exc
|
|
21
|
+
|
|
22
|
+
from ._linters import parse_errors
|
|
23
|
+
from ._project import Problem
|
|
24
|
+
from ._sql import _rendered, calls_kept, registered
|
|
25
|
+
from ._static import StaticMacro
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from collections.abc import Iterator, Sequence
|
|
29
|
+
|
|
30
|
+
from ._linters import Linter
|
|
31
|
+
from ._project import Project
|
|
32
|
+
|
|
33
|
+
__all__ = ["Rendered", "import_macros", "rendered", "unparsed"]
|
|
34
|
+
|
|
35
|
+
VARIANTS = ("given", "not_given")
|
|
36
|
+
"""The two ways each template is written: every parameter given, and none."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Example:
|
|
40
|
+
"""A made-up value: any path read off it is another, and it writes `x`.
|
|
41
|
+
|
|
42
|
+
`:criteria.teams` and `:filters["kind"]` both read one, so a template
|
|
43
|
+
renders whatever shape its values have.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
def __getattr__(self, _: str) -> Example:
|
|
47
|
+
return self
|
|
48
|
+
|
|
49
|
+
def __getitem__(self, _: object) -> Example:
|
|
50
|
+
return self
|
|
51
|
+
|
|
52
|
+
def __iter__(self) -> Iterator[Example]:
|
|
53
|
+
return iter((self,))
|
|
54
|
+
|
|
55
|
+
def __len__(self) -> int:
|
|
56
|
+
return 1
|
|
57
|
+
|
|
58
|
+
def __str__(self) -> str:
|
|
59
|
+
return "x"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass(frozen=True, slots=True)
|
|
63
|
+
class Piece:
|
|
64
|
+
"""The stretch of rendered SQL a macro call wrote, and where the call is."""
|
|
65
|
+
|
|
66
|
+
start: int
|
|
67
|
+
end: int
|
|
68
|
+
call: str
|
|
69
|
+
"""The call's name under its namespace: `tpl.owned_by`."""
|
|
70
|
+
span: tuple[int, int]
|
|
71
|
+
"""Where the call is written in the template."""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True, slots=True)
|
|
75
|
+
class Rendered:
|
|
76
|
+
"""A template written one way, or why it could not be."""
|
|
77
|
+
|
|
78
|
+
name: str
|
|
79
|
+
variant: str
|
|
80
|
+
sql: str | None
|
|
81
|
+
problem: str | None = None
|
|
82
|
+
kept: tuple[str, ...] = ()
|
|
83
|
+
"""The macro calls that could not be made, and stay calls in the SQL."""
|
|
84
|
+
pieces: tuple[Piece, ...] = ()
|
|
85
|
+
"""What each call of the template itself wrote, in order."""
|
|
86
|
+
read: bool = True
|
|
87
|
+
"""Whether the template could be read: `sqlakit check` says why not."""
|
|
88
|
+
|
|
89
|
+
def written_by(self, offset: int) -> Piece | None:
|
|
90
|
+
"""Return the call that wrote the SQL at an offset, if a call did."""
|
|
91
|
+
return next(
|
|
92
|
+
(piece for piece in self.pieces if piece.start <= offset < piece.end), None
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def import_macros(project: Project) -> None:
|
|
97
|
+
"""Import the modules of the project's Python macros, in place of their reading.
|
|
98
|
+
|
|
99
|
+
`load_project` reads the macros from their source, so a call to one is
|
|
100
|
+
checked and never made. The modules are imported the way the code names
|
|
101
|
+
them, from the directory above their package.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
Exception: whatever importing the project's code raises.
|
|
105
|
+
|
|
106
|
+
"""
|
|
107
|
+
macros = project.templates.macros
|
|
108
|
+
modules = {
|
|
109
|
+
macro.path for macro in macros.values() if isinstance(macro, StaticMacro)
|
|
110
|
+
}
|
|
111
|
+
for path in sorted(modules):
|
|
112
|
+
folder, name = _module(path)
|
|
113
|
+
if str(folder) not in sys.path:
|
|
114
|
+
sys.path.insert(0, str(folder))
|
|
115
|
+
found = registered([name])
|
|
116
|
+
macros.update(
|
|
117
|
+
(key, macro)
|
|
118
|
+
for key, macro in found.items()
|
|
119
|
+
if isinstance(macros.get(key), StaticMacro)
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def rendered(project: Project, dialect: str) -> Iterator[Rendered]:
|
|
124
|
+
"""Yield each template of the project, written each way."""
|
|
125
|
+
preparer = _preparer(dialect)
|
|
126
|
+
namespace = project.templates.namespace
|
|
127
|
+
for name in project.templates.names():
|
|
128
|
+
path = project.path_of(name)
|
|
129
|
+
if path is None:
|
|
130
|
+
continue
|
|
131
|
+
source = path.read_text(encoding="utf-8")
|
|
132
|
+
try:
|
|
133
|
+
template = project.load(name, source)
|
|
134
|
+
except Exception as error: # noqa: BLE001 - `sqlakit check` says more
|
|
135
|
+
for variant in VARIANTS:
|
|
136
|
+
yield Rendered(name, variant, None, str(error), read=False)
|
|
137
|
+
continue
|
|
138
|
+
calls = list(_calls(template.parts))
|
|
139
|
+
refused = _refused(project, name, source, calls, preparer)
|
|
140
|
+
for variant in VARIANTS:
|
|
141
|
+
given = variant == "given"
|
|
142
|
+
values: dict[str, Any] = {
|
|
143
|
+
param: Example() if given and param not in refused else None
|
|
144
|
+
for param in template.parameters()
|
|
145
|
+
}
|
|
146
|
+
try:
|
|
147
|
+
with calls_kept():
|
|
148
|
+
sql, _ = _rendered(
|
|
149
|
+
template, {"dialect": dialect, **values}, preparer
|
|
150
|
+
)
|
|
151
|
+
except Exception as error: # noqa: BLE001 - the project's macros raise anything
|
|
152
|
+
yield Rendered(name, variant, None, f"{type(error).__name__}: {error}")
|
|
153
|
+
continue
|
|
154
|
+
kept = tuple(
|
|
155
|
+
sorted(
|
|
156
|
+
{
|
|
157
|
+
f"{namespace}.{call.name}"
|
|
158
|
+
for call in calls
|
|
159
|
+
if re.search(
|
|
160
|
+
rf"(?<![\w.]){re.escape(namespace)}\.{re.escape(call.name)}\s*\(",
|
|
161
|
+
sql,
|
|
162
|
+
re.IGNORECASE,
|
|
163
|
+
)
|
|
164
|
+
}
|
|
165
|
+
)
|
|
166
|
+
)
|
|
167
|
+
sql = sql.strip() + "\n"
|
|
168
|
+
pieces = _pieces(
|
|
169
|
+
project,
|
|
170
|
+
name,
|
|
171
|
+
source,
|
|
172
|
+
parts=template.parts,
|
|
173
|
+
values=values,
|
|
174
|
+
preparer=preparer,
|
|
175
|
+
sql=sql,
|
|
176
|
+
)
|
|
177
|
+
yield Rendered(name, variant, sql, kept=kept, pieces=pieces)
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _pieces( # noqa: PLR0913 - the template, and what it was rendered with
|
|
181
|
+
project: Project,
|
|
182
|
+
name: str,
|
|
183
|
+
source: str,
|
|
184
|
+
*,
|
|
185
|
+
parts: Sequence[Any],
|
|
186
|
+
values: dict[str, Any],
|
|
187
|
+
preparer: Any, # noqa: ANN401
|
|
188
|
+
sql: str,
|
|
189
|
+
) -> tuple[Piece, ...]:
|
|
190
|
+
"""Return what each call of the template wrote, found in the whole SQL.
|
|
191
|
+
|
|
192
|
+
Each call is rendered alone with the same values, and looked for after the
|
|
193
|
+
one before it.
|
|
194
|
+
"""
|
|
195
|
+
namespace = project.templates.namespace
|
|
196
|
+
dialect = getattr(preparer.dialect, "name", "")
|
|
197
|
+
found = []
|
|
198
|
+
after = 0
|
|
199
|
+
for call in parts:
|
|
200
|
+
if isinstance(call, str):
|
|
201
|
+
continue
|
|
202
|
+
try:
|
|
203
|
+
alone = project.load(name, source[call.span[0] : call.span[1]])
|
|
204
|
+
with calls_kept():
|
|
205
|
+
text, _ = _rendered(
|
|
206
|
+
alone,
|
|
207
|
+
{
|
|
208
|
+
"dialect": dialect,
|
|
209
|
+
**{param: values.get(param) for param in alone.parameters()},
|
|
210
|
+
},
|
|
211
|
+
preparer,
|
|
212
|
+
)
|
|
213
|
+
except Exception: # noqa: BLE001, S112 - a call that fails alone has no piece
|
|
214
|
+
continue
|
|
215
|
+
text = text.strip()
|
|
216
|
+
start = sql.find(text, after) if text else -1
|
|
217
|
+
if start < 0:
|
|
218
|
+
continue
|
|
219
|
+
after = start + len(text)
|
|
220
|
+
found.append(Piece(start, after, f"{namespace}.{call.name}", call.span))
|
|
221
|
+
return tuple(found)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
_WAYS = {"given": "with every parameter given", "not_given": "with none given"}
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def unparsed(project: Project, linter: Linter, dialect: str) -> list[Problem]:
|
|
228
|
+
"""Return where the linter cannot parse what the templates render.
|
|
229
|
+
|
|
230
|
+
Each template is rendered each way, and the SQL the linter cannot parse is
|
|
231
|
+
put on the call that wrote it. SQL the template writes itself is put on
|
|
232
|
+
its line, found by its text. A template that cannot be rendered is a
|
|
233
|
+
problem too.
|
|
234
|
+
"""
|
|
235
|
+
written = list(rendered(project, dialect))
|
|
236
|
+
with tempfile.TemporaryDirectory() as folder:
|
|
237
|
+
# The two ways often write the same SQL, which is read once.
|
|
238
|
+
files: dict[Path, list[Rendered]] = {}
|
|
239
|
+
by_text: dict[tuple[str, str], Path] = {}
|
|
240
|
+
for one in written:
|
|
241
|
+
if one.sql is None:
|
|
242
|
+
continue
|
|
243
|
+
file = by_text.get((one.name, one.sql))
|
|
244
|
+
if file is None:
|
|
245
|
+
file = Path(folder, f"{len(by_text)}.sql").resolve()
|
|
246
|
+
file.write_text(one.sql, encoding="utf-8")
|
|
247
|
+
by_text[one.name, one.sql] = file
|
|
248
|
+
files.setdefault(file, []).append(one)
|
|
249
|
+
errors = parse_errors(linter, Path(folder).resolve(), project.root)
|
|
250
|
+
found: dict[tuple[Path, int, int, str], list[str]] = {}
|
|
251
|
+
for one in written:
|
|
252
|
+
path = project.path_of(one.name)
|
|
253
|
+
if path is None or one.sql is not None or not one.read:
|
|
254
|
+
continue
|
|
255
|
+
key = (path, 0, 0, f"cannot be rendered {{}}: {one.problem}")
|
|
256
|
+
found.setdefault(key, []).append(_WAYS[one.variant])
|
|
257
|
+
for file, places in errors.items():
|
|
258
|
+
for one in files.get(file, ()):
|
|
259
|
+
_place(project, linter, one, places, found)
|
|
260
|
+
return [
|
|
261
|
+
Problem(path, start, end, message.format(" and ".join(ways)))
|
|
262
|
+
for (path, start, end, message), ways in found.items()
|
|
263
|
+
]
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _place(
|
|
267
|
+
project: Project,
|
|
268
|
+
linter: Linter,
|
|
269
|
+
one: Rendered,
|
|
270
|
+
places: list[tuple[int, int]],
|
|
271
|
+
found: dict[tuple[Path, int, int, str], list[str]],
|
|
272
|
+
) -> None:
|
|
273
|
+
"""Put each place the linter could not parse on the call that wrote it."""
|
|
274
|
+
path = project.path_of(one.name)
|
|
275
|
+
if path is None or one.sql is None:
|
|
276
|
+
return
|
|
277
|
+
source = path.read_text(encoding="utf-8")
|
|
278
|
+
starts = [0, *(index + 1 for index, char in enumerate(one.sql) if char == "\n")]
|
|
279
|
+
for line, column in places:
|
|
280
|
+
offset = starts[min(line, len(starts)) - 1] + column - 1
|
|
281
|
+
written = one.sql[starts[line - 1] :].split("\n", 1)[0]
|
|
282
|
+
text = written.strip()
|
|
283
|
+
# The line the linter stopped on, and a caret under where it stopped.
|
|
284
|
+
indent = len(written) - len(written.lstrip())
|
|
285
|
+
shown = f"\n {text}\n {' ' * max(column - 1 - indent, 0)}^"
|
|
286
|
+
piece = one.written_by(offset)
|
|
287
|
+
if piece is not None:
|
|
288
|
+
message = f"{piece.call} writes SQL {linter.name} cannot parse, {{}}{shown}"
|
|
289
|
+
key = (path, *piece.span, message)
|
|
290
|
+
else:
|
|
291
|
+
at = source.find(text) if text else -1
|
|
292
|
+
start = max(at, 0)
|
|
293
|
+
message = f"{linter.name} cannot parse the rendered SQL, {{}}{shown}"
|
|
294
|
+
key = (path, start, start + len(text), message)
|
|
295
|
+
ways = found.setdefault(key, [])
|
|
296
|
+
if _WAYS[one.variant] not in ways:
|
|
297
|
+
ways.append(_WAYS[one.variant])
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _refused(
|
|
301
|
+
project: Project,
|
|
302
|
+
name: str,
|
|
303
|
+
source: str,
|
|
304
|
+
calls: Sequence[Any],
|
|
305
|
+
preparer: Any, # noqa: ANN401
|
|
306
|
+
) -> set[str]:
|
|
307
|
+
"""Return the parameters a macro refuses made up, and takes when not given.
|
|
308
|
+
|
|
309
|
+
`tpl.order_by` takes a sort that names one of its columns, and nothing
|
|
310
|
+
made up does, so `:sort` counts as not given when the rest are.
|
|
311
|
+
"""
|
|
312
|
+
found: set[str] = set()
|
|
313
|
+
namespace = project.templates.namespace
|
|
314
|
+
dialect = getattr(preparer.dialect, "name", "")
|
|
315
|
+
for call in calls:
|
|
316
|
+
text = source[call.span[0] : call.span[1]]
|
|
317
|
+
try:
|
|
318
|
+
alone = project.load(name, text)
|
|
319
|
+
except Exception: # noqa: BLE001, S112 - the whole template says what is wrong
|
|
320
|
+
continue
|
|
321
|
+
names = alone.parameters()
|
|
322
|
+
if not names:
|
|
323
|
+
continue
|
|
324
|
+
written = []
|
|
325
|
+
for value in (Example(), None):
|
|
326
|
+
try:
|
|
327
|
+
with calls_kept():
|
|
328
|
+
sql, _ = _rendered(
|
|
329
|
+
alone,
|
|
330
|
+
{"dialect": dialect, **dict.fromkeys(names, value)},
|
|
331
|
+
preparer,
|
|
332
|
+
)
|
|
333
|
+
except Exception: # noqa: BLE001 - a macro may refuse either
|
|
334
|
+
sql = None
|
|
335
|
+
written.append(sql)
|
|
336
|
+
made_up, missing = (
|
|
337
|
+
sql is None
|
|
338
|
+
or bool(
|
|
339
|
+
re.match(
|
|
340
|
+
rf"{re.escape(namespace)}\.{re.escape(call.name)}\s*\(",
|
|
341
|
+
sql.strip(),
|
|
342
|
+
re.IGNORECASE,
|
|
343
|
+
)
|
|
344
|
+
)
|
|
345
|
+
for sql in written
|
|
346
|
+
)
|
|
347
|
+
if made_up and not missing:
|
|
348
|
+
found.update(names)
|
|
349
|
+
return found
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _calls(parts: Sequence[Any]) -> Iterator[Any]:
|
|
353
|
+
"""Yield every macro call of a read template, the ones in arguments too."""
|
|
354
|
+
for part in parts:
|
|
355
|
+
if isinstance(part, str):
|
|
356
|
+
continue
|
|
357
|
+
yield part
|
|
358
|
+
for argument in part.args:
|
|
359
|
+
yield from _calls(argument.parts)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _module(path: Path) -> tuple[Path, str]:
|
|
363
|
+
"""Return the directory a module is imported from, and its dotted name."""
|
|
364
|
+
parts = [path.stem]
|
|
365
|
+
folder = path.parent
|
|
366
|
+
while (folder / "__init__.py").is_file():
|
|
367
|
+
parts.append(folder.name)
|
|
368
|
+
folder = folder.parent
|
|
369
|
+
return folder, ".".join(reversed(parts))
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _preparer(dialect: str) -> Any: # noqa: ANN401 - SQLAlchemy's IdentifierPreparer
|
|
373
|
+
"""Return how a dialect quotes names, loaded without a driver or a server."""
|
|
374
|
+
try:
|
|
375
|
+
return sa.engine.make_url(f"{dialect}://").get_dialect()().identifier_preparer
|
|
376
|
+
except sa.exc.NoSuchModuleError:
|
|
377
|
+
default = sa.engine.default.DefaultDialect()
|
|
378
|
+
default.name = dialect
|
|
379
|
+
return default.identifier_preparer
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|