localtools-sql-formatter 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- localtools_sql_formatter-1.0.0/.gitignore +8 -0
- localtools_sql_formatter-1.0.0/LICENSE +21 -0
- localtools_sql_formatter-1.0.0/PKG-INFO +84 -0
- localtools_sql_formatter-1.0.0/README.md +71 -0
- localtools_sql_formatter-1.0.0/localtools_sql_formatter/__init__.py +0 -0
- localtools_sql_formatter-1.0.0/localtools_sql_formatter/__main__.py +311 -0
- localtools_sql_formatter-1.0.0/localtools_sql_formatter/engine.py +284 -0
- localtools_sql_formatter-1.0.0/localtools_sql_formatter/localtools.py +223 -0
- localtools_sql_formatter-1.0.0/pyproject.toml +25 -0
- localtools_sql_formatter-1.0.0/ui/index.html +294 -0
- localtools_sql_formatter-1.0.0/ui/kit/fonts/OFL.txt +93 -0
- localtools_sql_formatter-1.0.0/ui/kit/fonts/jetbrains-mono-latin-bold.woff2 +0 -0
- localtools_sql_formatter-1.0.0/ui/kit/fonts/jetbrains-mono-latin-regular.woff2 +0 -0
- localtools_sql_formatter-1.0.0/ui/kit/localtools.css +189 -0
- localtools_sql_formatter-1.0.0/ui/kit/localtools.js +191 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Chaitanya Prabuddha
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: localtools-sql-formatter
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Lint, fix and transpile SQL in any major dialect with sqlfluff and sqlglot (localtools, offline)
|
|
5
|
+
Project-URL: Source, https://github.com/chayprabs/localtools
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Requires-Dist: multipart>=1.2
|
|
10
|
+
Requires-Dist: sqlfluff[rs]<5,>=4.4
|
|
11
|
+
Requires-Dist: sqlglot>=30
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# SQL Formatter (localtools)
|
|
15
|
+
|
|
16
|
+
Lint, auto-fix and format SQL in 28 dialects with **sqlfluff**, and transpile between dialects with **sqlglot**.
|
|
17
|
+
Runs on your machine: a one-page web UI on `http://127.0.0.1:47117`, plus a CLI for scripts and CI.
|
|
18
|
+
|
|
19
|
+
## Install
|
|
20
|
+
|
|
21
|
+
| OS | command |
|
|
22
|
+
|---|---|
|
|
23
|
+
| Windows (PowerShell) | `irm https://raw.githubusercontent.com/chayprabs/localtools/main/sql-formatter/install.ps1 \| iex` |
|
|
24
|
+
| macOS / Linux | `curl -fsSL https://raw.githubusercontent.com/chayprabs/localtools/main/sql-formatter/install.sh \| bash` |
|
|
25
|
+
| PyPI | `pipx install localtools-sql-formatter` |
|
|
26
|
+
|
|
27
|
+
The script installs a self-contained build (bundled Python) into `~/.localtools/sql-formatter` and adds a launcher.
|
|
28
|
+
|
|
29
|
+
## Use
|
|
30
|
+
|
|
31
|
+
`localtools-sql-formatter` opens the UI. Paste SQL or drop `.sql` files (several at once), pick the dialect, then:
|
|
32
|
+
|
|
33
|
+
- **Lint**: every violation with line:column, rule code (linked to the sqlfluff docs) and whether it can be auto-fixed. Download the report as JSON.
|
|
34
|
+
- **Fix**: the fixed SQL with a diff, Copy and Download, plus what still needs a human.
|
|
35
|
+
- **Transpile**: rewrite between dialects (e.g. T-SQL `TOP`/`ISNULL` to Postgres `LIMIT`/`COALESCE`), with notes on unsupported constructs.
|
|
36
|
+
|
|
37
|
+
Options: rule selection/exclusion (`LT01,CP01`, `core`, `layout`), templater (Jinja, or placeholders `:name`, `?`, `$1`, `%(name)s`, `@name`), and an uploaded `.sqlfluff` / `setup.cfg` / `tox.ini` / `pyproject.toml`.
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
localtools-sql-formatter lint models/*.sql --dialect postgres --format json # or human | sarif; exit 1 on violations
|
|
41
|
+
localtools-sql-formatter fix query.sql --dialect snowflake # fixed SQL to stdout
|
|
42
|
+
localtools-sql-formatter fix *.sql --in-place --config .sqlfluff # rewrite files; exit 1 if anything is left
|
|
43
|
+
localtools-sql-formatter fix - --diff < query.sql # stdin, unified diff
|
|
44
|
+
localtools-sql-formatter transpile q.sql --read tsql --write duckdb
|
|
45
|
+
localtools-sql-formatter dialects | rules
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
API: `POST /api/lint`, `/api/fix`, `/api/transpile` (multipart: `sql` and/or `file`..., `dialect`, `rules`, `exclude_rules`,
|
|
49
|
+
`templater`, `config` file; transpile: `read`, `write`), `GET /api/meta`. `/api/lint` with `format=sarif` returns SARIF 2.1.0.
|
|
50
|
+
|
|
51
|
+
## Engine and why
|
|
52
|
+
|
|
53
|
+
- **sqlfluff 4** (with its Rust parser, sqlfluffrs) is the most complete open-source SQL linter: dialect-aware parsing,
|
|
54
|
+
~70 rules, real auto-fix, Jinja/placeholder templating. Results are identical to the `sqlfluff` CLI with the same config
|
|
55
|
+
(tested on TPC-H, sqlfluff's dialect fixtures, dbt's jaffle_shop and GitLab's schema).
|
|
56
|
+
- `fix` repeats sqlfluff's fix pass until nothing changes, so running it twice gives the same SQL; when sqlfluff
|
|
57
|
+
oscillates between two layouts (it happens), the tool settles on one of them deterministically.
|
|
58
|
+
- **sqlglot** transpiles (sqlfluff cannot): it parses into an AST and regenerates SQL for 31 dialects. Output equals the
|
|
59
|
+
`python -m sqlglot --identify false` CLI. It reformats in its own style, which is why lint/fix stay with sqlfluff.
|
|
60
|
+
|
|
61
|
+
## Limits and safety
|
|
62
|
+
|
|
63
|
+
- The web UI/API takes up to 512 KB of SQL and about 2,000 statements per run, and each run has a 120 s deadline
|
|
64
|
+
(sqlfluff is thorough but slow: a messy 20 KB file can take 10+ s). Every run is a separate worker process that is
|
|
65
|
+
killed at the deadline, when the browser disconnects, or when the server exits; at most 1–4 run at once (else HTTP 503).
|
|
66
|
+
On Linux each worker is capped at 2 GB of memory. The CLI has no size limit or deadline.
|
|
67
|
+
- Config files are parsed, never executed: `library_path`, macro/loader paths (any `*_path` / `*_dir` key), the python and
|
|
68
|
+
dbt templaters and `processes` are ignored and listed in the result. sqlfluff's 20 KB "skip large file" default is off.
|
|
69
|
+
- Files that do not parse are not fixed (like the sqlfluff CLI); the parse error shows line and column.
|
|
70
|
+
|
|
71
|
+
## Cut
|
|
72
|
+
|
|
73
|
+
- dbt templater (needs a dbt project, adapters and a database connection) and the python templater (runs code): use
|
|
74
|
+
`jinja`, which provides dbt's `ref`/`source`/`config` builtins.
|
|
75
|
+
- Automatic discovery of `.sqlfluff` files next to the SQL: pass `--config` explicitly.
|
|
76
|
+
- winget/Homebrew/Scoop packages: Python-runtime tools ship through the install scripts and PyPI.
|
|
77
|
+
|
|
78
|
+
## Update / uninstall
|
|
79
|
+
|
|
80
|
+
`localtools-sql-formatter update` · `localtools-sql-formatter uninstall` (pipx: `pipx upgrade|uninstall localtools-sql-formatter`).
|
|
81
|
+
|
|
82
|
+
## License
|
|
83
|
+
|
|
84
|
+
MIT. Bundled components: see THIRD_PARTY_NOTICES.md.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# SQL Formatter (localtools)
|
|
2
|
+
|
|
3
|
+
Lint, auto-fix and format SQL in 28 dialects with **sqlfluff**, and transpile between dialects with **sqlglot**.
|
|
4
|
+
Runs on your machine: a one-page web UI on `http://127.0.0.1:47117`, plus a CLI for scripts and CI.
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
|
|
8
|
+
| OS | command |
|
|
9
|
+
|---|---|
|
|
10
|
+
| Windows (PowerShell) | `irm https://raw.githubusercontent.com/chayprabs/localtools/main/sql-formatter/install.ps1 \| iex` |
|
|
11
|
+
| macOS / Linux | `curl -fsSL https://raw.githubusercontent.com/chayprabs/localtools/main/sql-formatter/install.sh \| bash` |
|
|
12
|
+
| PyPI | `pipx install localtools-sql-formatter` |
|
|
13
|
+
|
|
14
|
+
The script installs a self-contained build (bundled Python) into `~/.localtools/sql-formatter` and adds a launcher.
|
|
15
|
+
|
|
16
|
+
## Use
|
|
17
|
+
|
|
18
|
+
`localtools-sql-formatter` opens the UI. Paste SQL or drop `.sql` files (several at once), pick the dialect, then:
|
|
19
|
+
|
|
20
|
+
- **Lint**: every violation with line:column, rule code (linked to the sqlfluff docs) and whether it can be auto-fixed. Download the report as JSON.
|
|
21
|
+
- **Fix**: the fixed SQL with a diff, Copy and Download, plus what still needs a human.
|
|
22
|
+
- **Transpile**: rewrite between dialects (e.g. T-SQL `TOP`/`ISNULL` to Postgres `LIMIT`/`COALESCE`), with notes on unsupported constructs.
|
|
23
|
+
|
|
24
|
+
Options: rule selection/exclusion (`LT01,CP01`, `core`, `layout`), templater (Jinja, or placeholders `:name`, `?`, `$1`, `%(name)s`, `@name`), and an uploaded `.sqlfluff` / `setup.cfg` / `tox.ini` / `pyproject.toml`.
|
|
25
|
+
|
|
26
|
+
```sh
|
|
27
|
+
localtools-sql-formatter lint models/*.sql --dialect postgres --format json # or human | sarif; exit 1 on violations
|
|
28
|
+
localtools-sql-formatter fix query.sql --dialect snowflake # fixed SQL to stdout
|
|
29
|
+
localtools-sql-formatter fix *.sql --in-place --config .sqlfluff # rewrite files; exit 1 if anything is left
|
|
30
|
+
localtools-sql-formatter fix - --diff < query.sql # stdin, unified diff
|
|
31
|
+
localtools-sql-formatter transpile q.sql --read tsql --write duckdb
|
|
32
|
+
localtools-sql-formatter dialects | rules
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
API: `POST /api/lint`, `/api/fix`, `/api/transpile` (multipart: `sql` and/or `file`..., `dialect`, `rules`, `exclude_rules`,
|
|
36
|
+
`templater`, `config` file; transpile: `read`, `write`), `GET /api/meta`. `/api/lint` with `format=sarif` returns SARIF 2.1.0.
|
|
37
|
+
|
|
38
|
+
## Engine and why
|
|
39
|
+
|
|
40
|
+
- **sqlfluff 4** (with its Rust parser, sqlfluffrs) is the most complete open-source SQL linter: dialect-aware parsing,
|
|
41
|
+
~70 rules, real auto-fix, Jinja/placeholder templating. Results are identical to the `sqlfluff` CLI with the same config
|
|
42
|
+
(tested on TPC-H, sqlfluff's dialect fixtures, dbt's jaffle_shop and GitLab's schema).
|
|
43
|
+
- `fix` repeats sqlfluff's fix pass until nothing changes, so running it twice gives the same SQL; when sqlfluff
|
|
44
|
+
oscillates between two layouts (it happens), the tool settles on one of them deterministically.
|
|
45
|
+
- **sqlglot** transpiles (sqlfluff cannot): it parses into an AST and regenerates SQL for 31 dialects. Output equals the
|
|
46
|
+
`python -m sqlglot --identify false` CLI. It reformats in its own style, which is why lint/fix stay with sqlfluff.
|
|
47
|
+
|
|
48
|
+
## Limits and safety
|
|
49
|
+
|
|
50
|
+
- The web UI/API takes up to 512 KB of SQL and about 2,000 statements per run, and each run has a 120 s deadline
|
|
51
|
+
(sqlfluff is thorough but slow: a messy 20 KB file can take 10+ s). Every run is a separate worker process that is
|
|
52
|
+
killed at the deadline, when the browser disconnects, or when the server exits; at most 1–4 run at once (else HTTP 503).
|
|
53
|
+
On Linux each worker is capped at 2 GB of memory. The CLI has no size limit or deadline.
|
|
54
|
+
- Config files are parsed, never executed: `library_path`, macro/loader paths (any `*_path` / `*_dir` key), the python and
|
|
55
|
+
dbt templaters and `processes` are ignored and listed in the result. sqlfluff's 20 KB "skip large file" default is off.
|
|
56
|
+
- Files that do not parse are not fixed (like the sqlfluff CLI); the parse error shows line and column.
|
|
57
|
+
|
|
58
|
+
## Cut
|
|
59
|
+
|
|
60
|
+
- dbt templater (needs a dbt project, adapters and a database connection) and the python templater (runs code): use
|
|
61
|
+
`jinja`, which provides dbt's `ref`/`source`/`config` builtins.
|
|
62
|
+
- Automatic discovery of `.sqlfluff` files next to the SQL: pass `--config` explicitly.
|
|
63
|
+
- winget/Homebrew/Scoop packages: Python-runtime tools ship through the install scripts and PyPI.
|
|
64
|
+
|
|
65
|
+
## Update / uninstall
|
|
66
|
+
|
|
67
|
+
`localtools-sql-formatter update` · `localtools-sql-formatter uninstall` (pipx: `pipx upgrade|uninstall localtools-sql-formatter`).
|
|
68
|
+
|
|
69
|
+
## License
|
|
70
|
+
|
|
71
|
+
MIT. Bundled components: see THIRD_PARTY_NOTICES.md.
|
|
File without changes
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import select
|
|
5
|
+
import socket
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import multipart
|
|
14
|
+
|
|
15
|
+
from . import engine
|
|
16
|
+
from .localtools import HTTPError, Tool, main as run
|
|
17
|
+
|
|
18
|
+
USAGE = """localtools-sql-formatter: lint, fix and transpile SQL in any major dialect, offline (sqlfluff + sqlglot).
|
|
19
|
+
|
|
20
|
+
Usage:
|
|
21
|
+
localtools-sql-formatter [serve] [--port N] [--no-open] start the web UI (default port 47117)
|
|
22
|
+
localtools-sql-formatter lint FILE... [options] report rule violations; exit 1 if any
|
|
23
|
+
--format human|json|sarif
|
|
24
|
+
localtools-sql-formatter fix FILE... [options] print the fixed SQL (one file), or:
|
|
25
|
+
--in-place rewrite the files --diff print a unified diff
|
|
26
|
+
exit 1 if violations that cannot be fixed remain (or the file does not parse)
|
|
27
|
+
localtools-sql-formatter transpile FILE... --read D --write D [--no-pretty] convert between dialects (sqlglot)
|
|
28
|
+
localtools-sql-formatter dialects | rules list dialects / lint rules
|
|
29
|
+
localtools-sql-formatter update | uninstall re-run the installer / remove the tool
|
|
30
|
+
localtools-sql-formatter --version | --help
|
|
31
|
+
|
|
32
|
+
Options for lint and fix (FILE "-" reads stdin):
|
|
33
|
+
--dialect D ansi (default), postgres, mysql, bigquery, snowflake, tsql, sqlite, duckdb, ...
|
|
34
|
+
--rules R only these rules, e.g. LT01,CP01 or core --exclude-rules R skip these
|
|
35
|
+
--templater T raw | jinja | placeholder:colon | placeholder:question_mark | placeholder:dollar | ...
|
|
36
|
+
--config FILE .sqlfluff, setup.cfg, tox.ini or pyproject.toml ([tool.sqlfluff]); paths, python/dbt
|
|
37
|
+
templaters and library code in it are ignored (nothing in a config is executed)
|
|
38
|
+
|
|
39
|
+
API: POST /api/lint | /api/fix | /api/transpile (multipart: sql and/or file..., dialect, rules, exclude_rules,
|
|
40
|
+
templater, config (file); transpile: read, write), GET /api/meta (dialects + rules).
|
|
41
|
+
/api/lint with format=sarif returns SARIF 2.1.0.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
UI = Path(__file__).with_name("ui") # wheel: packaged ui/; source checkout: repo-root ui/
|
|
45
|
+
if not UI.is_dir():
|
|
46
|
+
UI = Path(__file__).parents[1] / "ui"
|
|
47
|
+
DEADLINE = float(os.environ.get("LOCALTOOLS_SQL_DEADLINE") or 120)
|
|
48
|
+
JOBS = int(os.environ.get("LOCALTOOLS_SQL_JOBS") or max(1, min(4, (os.cpu_count() or 2) // 2)))
|
|
49
|
+
SLOTS = threading.BoundedSemaphore(JOBS)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def text(data, name):
|
|
53
|
+
try:
|
|
54
|
+
return engine.newlines(data.decode("utf-8-sig"))
|
|
55
|
+
except UnicodeDecodeError:
|
|
56
|
+
raise HTTPError(400, f"{name}: not UTF-8 text")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def read_form(req):
|
|
60
|
+
ctype, opts = multipart.parse_options_header(req.headers.get("Content-Type", ""))
|
|
61
|
+
if ctype != "multipart/form-data" or "boundary" not in opts:
|
|
62
|
+
raise HTTPError(400, "expected multipart/form-data")
|
|
63
|
+
size = int(req.headers.get("Content-Length") or 0)
|
|
64
|
+
if size > engine.MAX_BYTES * 2 + (1 << 20):
|
|
65
|
+
raise HTTPError(413, f"upload too large ({engine.MAX_BYTES >> 10} KB of SQL max)")
|
|
66
|
+
try:
|
|
67
|
+
parts = list(multipart.MultipartParser(req.rfile, opts["boundary"], size))
|
|
68
|
+
except multipart.MultipartError as e:
|
|
69
|
+
raise HTTPError(400, f"malformed upload: {e}")
|
|
70
|
+
try:
|
|
71
|
+
fields = {p.name: p.value for p in parts if not p.filename}
|
|
72
|
+
files = [(p.filename, text(p.raw, p.filename)) for p in parts if p.filename and p.name == "file"]
|
|
73
|
+
cfg = next(((p.filename, text(p.raw, p.filename)) for p in parts if p.filename and p.name == "config"), None)
|
|
74
|
+
finally:
|
|
75
|
+
for p in parts:
|
|
76
|
+
p.close()
|
|
77
|
+
if fields.get("sql", "").strip():
|
|
78
|
+
files.insert(0, (fields.get("name") or "pasted.sql", engine.newlines(fields["sql"])))
|
|
79
|
+
if not files:
|
|
80
|
+
raise HTTPError(400, "paste SQL or upload at least one .sql file")
|
|
81
|
+
if cfg:
|
|
82
|
+
fields["config_name"], fields["config"] = cfg
|
|
83
|
+
return files, fields
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def client_gone(req):
|
|
87
|
+
try:
|
|
88
|
+
r, _, _ = select.select([req.connection], [], [], 0)
|
|
89
|
+
return bool(r) and req.connection.recv(1, socket.MSG_PEEK) == b""
|
|
90
|
+
except OSError:
|
|
91
|
+
return True
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def run_worker(req, job):
|
|
95
|
+
"""One worker process per request: killed at the deadline, when the client disconnects, and (stdin EOF)
|
|
96
|
+
when this server exits. At most JOBS run at once; more get 503."""
|
|
97
|
+
if not SLOTS.acquire(blocking=False):
|
|
98
|
+
raise HTTPError(503, f"busy: {JOBS} SQL job(s) already running; try again when they finish")
|
|
99
|
+
try:
|
|
100
|
+
with tempfile.TemporaryDirectory(prefix="lt-sqlf-") as d:
|
|
101
|
+
jp = Path(d, "job.json")
|
|
102
|
+
jp.write_text(json.dumps(job), encoding="utf-8")
|
|
103
|
+
cmd = [sys.executable, *(["-I"] if sys.flags.isolated else []), "-m", __package__, "_worker", str(jp)]
|
|
104
|
+
p = subprocess.Popen(cmd, cwd=d, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
105
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
|
106
|
+
end = time.monotonic() + DEADLINE
|
|
107
|
+
try:
|
|
108
|
+
while p.poll() is None:
|
|
109
|
+
if time.monotonic() > end:
|
|
110
|
+
raise HTTPError(422, f"sqlfluff did not finish within {DEADLINE:g} s on this input; "
|
|
111
|
+
"split it, narrow the rules, or use the CLI (no deadline)")
|
|
112
|
+
if client_gone(req):
|
|
113
|
+
raise HTTPError(499, "client disconnected")
|
|
114
|
+
try:
|
|
115
|
+
p.wait(0.2)
|
|
116
|
+
except subprocess.TimeoutExpired:
|
|
117
|
+
pass
|
|
118
|
+
finally:
|
|
119
|
+
if p.poll() is None:
|
|
120
|
+
p.kill()
|
|
121
|
+
p.wait()
|
|
122
|
+
p.stdin.close()
|
|
123
|
+
out = jp.with_suffix(".out")
|
|
124
|
+
if not out.exists():
|
|
125
|
+
raise HTTPError(422, f"sqlfluff crashed on this input (exit code {p.returncode}); it may be nested too deeply")
|
|
126
|
+
res = json.loads(out.read_text(encoding="utf-8"))
|
|
127
|
+
finally:
|
|
128
|
+
SLOTS.release()
|
|
129
|
+
if "error" in res:
|
|
130
|
+
raise HTTPError(res["code"], res["error"])
|
|
131
|
+
return res
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def job(op, files, f):
|
|
135
|
+
try:
|
|
136
|
+
engine.check_limits(files)
|
|
137
|
+
except engine.SQLError as e:
|
|
138
|
+
raise HTTPError(413, str(e))
|
|
139
|
+
keys = ("read", "write") if op == "transpile" else ("dialect", "rules", "exclude_rules", "templater", "config", "config_name")
|
|
140
|
+
return {"op": op, "files": files, **{k: f[k] for k in keys if k in f}}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def api(op):
|
|
144
|
+
def handler(req):
|
|
145
|
+
files, f = read_form(req)
|
|
146
|
+
res = run_worker(req, job(op, files, f))
|
|
147
|
+
if op == "lint" and f.get("format") == "sarif":
|
|
148
|
+
body = json.dumps(engine.sarif(res["files"], sqlfluff_version()), indent=2).encode()
|
|
149
|
+
req.send_response(200)
|
|
150
|
+
req.send_header("Content-Type", "application/sarif+json")
|
|
151
|
+
req.send_header("Content-Disposition", 'attachment; filename="sqlfluff.sarif"')
|
|
152
|
+
req.send_header("Content-Length", str(len(body)))
|
|
153
|
+
req.end_headers()
|
|
154
|
+
return req.wfile.write(body)
|
|
155
|
+
req.send_json(200, res)
|
|
156
|
+
return handler
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def sqlfluff_version():
|
|
160
|
+
from importlib.metadata import version
|
|
161
|
+
return version("sqlfluff")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
META = {}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def api_meta(req):
|
|
168
|
+
if not META:
|
|
169
|
+
META.update(engine.dialects(), rules=engine.rules(), versions={"sqlfluff": sqlfluff_version()},
|
|
170
|
+
limits={"bytes": engine.MAX_BYTES, "statements": engine.MAX_STATEMENTS, "deadline_s": DEADLINE},
|
|
171
|
+
param_styles=engine.PARAM_STYLES)
|
|
172
|
+
req.send_json(200, META)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
# ---------- CLI ----------
|
|
176
|
+
def read_inputs(paths):
|
|
177
|
+
out = []
|
|
178
|
+
for p in paths:
|
|
179
|
+
if p == "-":
|
|
180
|
+
out.append(("stdin.sql", engine.newlines(sys.stdin.buffer.read().decode("utf-8-sig"))))
|
|
181
|
+
else:
|
|
182
|
+
try:
|
|
183
|
+
out.append((p, engine.newlines(Path(p).read_bytes().decode("utf-8-sig"))))
|
|
184
|
+
except OSError as e:
|
|
185
|
+
raise ValueError(f"{p}: {e.strerror}")
|
|
186
|
+
except UnicodeDecodeError:
|
|
187
|
+
raise ValueError(f"{p}: not UTF-8 text")
|
|
188
|
+
return out
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def lint_parser(prog):
|
|
192
|
+
p = argparse.ArgumentParser(prog=f"localtools-sql-formatter {prog}")
|
|
193
|
+
p.add_argument("files", nargs="+")
|
|
194
|
+
p.add_argument("--dialect", default="")
|
|
195
|
+
p.add_argument("--rules", default="")
|
|
196
|
+
p.add_argument("--exclude-rules", default="")
|
|
197
|
+
p.add_argument("--templater", default="")
|
|
198
|
+
p.add_argument("--config")
|
|
199
|
+
return p
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def cli_config(a):
|
|
203
|
+
cfg = Path(a.config).read_text(encoding="utf-8-sig") if a.config else ""
|
|
204
|
+
return engine.make_config(a.dialect, a.rules, a.exclude_rules, a.templater, cfg, Path(a.config).name if a.config else "")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def human(results):
|
|
208
|
+
for f in results:
|
|
209
|
+
for v in f.get("violations", f.get("remaining", [])):
|
|
210
|
+
print(f"{f['name']}:{v['line']}:{v['col']}: {v['code']} {v['description']}" + (" (warning)" if v["warning"] else ""))
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def utf8():
|
|
214
|
+
for s in (sys.stdout, sys.stderr):
|
|
215
|
+
s.reconfigure(encoding="utf-8", newline="") # SQL out exactly as fixed (no CRLF translation on Windows)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def cli_lint(args):
|
|
219
|
+
p = lint_parser("lint")
|
|
220
|
+
p.add_argument("--format", default="human", choices=["human", "json", "sarif"])
|
|
221
|
+
a = p.parse_args(args)
|
|
222
|
+
utf8()
|
|
223
|
+
cfg, dropped = cli_config(a)
|
|
224
|
+
for k in dropped:
|
|
225
|
+
print(f"note: ignored config key {k}", file=sys.stderr)
|
|
226
|
+
res = engine.lint(read_inputs(a.files), cfg)
|
|
227
|
+
if a.format == "json":
|
|
228
|
+
print(json.dumps(res, indent=2, ensure_ascii=False))
|
|
229
|
+
elif a.format == "sarif":
|
|
230
|
+
print(json.dumps(engine.sarif(res, sqlfluff_version()), indent=2, ensure_ascii=False))
|
|
231
|
+
else:
|
|
232
|
+
human(res)
|
|
233
|
+
n = sum(len(f["violations"]) for f in res)
|
|
234
|
+
print(f"{n} violation(s) in {len(res)} file(s)" if n else f"{len(res)} file(s) clean", file=sys.stderr)
|
|
235
|
+
return 1 if any(f["violations"] for f in res) else 0
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def cli_fix(args):
|
|
239
|
+
p = lint_parser("fix")
|
|
240
|
+
p.add_argument("--in-place", action="store_true")
|
|
241
|
+
p.add_argument("--diff", action="store_true")
|
|
242
|
+
a = p.parse_args(args)
|
|
243
|
+
utf8()
|
|
244
|
+
files = read_inputs(a.files)
|
|
245
|
+
if len(files) > 1 and not (a.in_place or a.diff):
|
|
246
|
+
raise ValueError("several files: add --in-place to rewrite them or --diff to print a diff")
|
|
247
|
+
if a.in_place and "-" in a.files:
|
|
248
|
+
raise ValueError("--in-place cannot rewrite stdin")
|
|
249
|
+
cfg, _ = cli_config(a)
|
|
250
|
+
res = engine.fix(files, cfg)
|
|
251
|
+
for f in res:
|
|
252
|
+
if a.in_place:
|
|
253
|
+
if f["changed"]:
|
|
254
|
+
Path(f["name"]).write_text(f["fixed"], encoding="utf-8", newline="")
|
|
255
|
+
print(f"{f['name']}: {'fixed' if f['changed'] else 'unchanged'}", file=sys.stderr)
|
|
256
|
+
elif a.diff:
|
|
257
|
+
sys.stdout.write(f["diff"])
|
|
258
|
+
else:
|
|
259
|
+
sys.stdout.write(f["fixed"])
|
|
260
|
+
if f["unparsable"]:
|
|
261
|
+
print(f"{f['name']}: not fixed, the file does not parse:", file=sys.stderr)
|
|
262
|
+
elif f["remaining"]:
|
|
263
|
+
print(f"{f['name']}: {len(f['remaining'])} violation(s) cannot be fixed automatically:", file=sys.stderr)
|
|
264
|
+
for v in f["remaining"]:
|
|
265
|
+
print(f" {v['line']}:{v['col']}: {v['code']} {v['description']}", file=sys.stderr)
|
|
266
|
+
return 1 if any(f["remaining"] for f in res) else 0
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def cli_transpile(args):
|
|
270
|
+
p = argparse.ArgumentParser(prog="localtools-sql-formatter transpile")
|
|
271
|
+
p.add_argument("files", nargs="+")
|
|
272
|
+
p.add_argument("--read", default="")
|
|
273
|
+
p.add_argument("--write", default="")
|
|
274
|
+
p.add_argument("--no-pretty", action="store_true")
|
|
275
|
+
a = p.parse_args(args)
|
|
276
|
+
utf8()
|
|
277
|
+
for f in engine.transpile(read_inputs(a.files), a.read, a.write, not a.no_pretty):
|
|
278
|
+
sys.stdout.write(f["sql"])
|
|
279
|
+
for w in f["warnings"]:
|
|
280
|
+
print(f"{f['name']}: warning: {w}", file=sys.stderr)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def cli_dialects(args):
|
|
284
|
+
d = engine.dialects()
|
|
285
|
+
print("lint/fix (sqlfluff):", " ".join(d["sqlfluff"]))
|
|
286
|
+
print("transpile (sqlglot):", " ".join(d["sqlglot"]))
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def cli_rules(args):
|
|
290
|
+
utf8()
|
|
291
|
+
for r in engine.rules():
|
|
292
|
+
print(f"{r['code']} {r['name']:<36} {r['description']}")
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def cli_worker(args):
|
|
296
|
+
engine.worker_main(args[0])
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
tool = Tool(slug="sql-formatter", port=47117, usage=USAGE, ui=UI,
|
|
300
|
+
routes={"POST /api/lint": api("lint"), "POST /api/fix": api("fix"), "POST /api/transpile": api("transpile"),
|
|
301
|
+
"GET /api/meta": api_meta},
|
|
302
|
+
cli={"lint": cli_lint, "fix": cli_fix, "transpile": cli_transpile,
|
|
303
|
+
"dialects": cli_dialects, "rules": cli_rules, "_worker": cli_worker})
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def main():
|
|
307
|
+
sys.exit(run(tool))
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
if __name__ == "__main__":
|
|
311
|
+
main()
|