localtools-postgres-migration-checker 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- localtools_postgres_migration_checker-1.0.0/.gitignore +8 -0
- localtools_postgres_migration_checker-1.0.0/LICENSE +21 -0
- localtools_postgres_migration_checker-1.0.0/PKG-INFO +53 -0
- localtools_postgres_migration_checker-1.0.0/README.md +40 -0
- localtools_postgres_migration_checker-1.0.0/localtools_postgres_migration_checker/__init__.py +0 -0
- localtools_postgres_migration_checker-1.0.0/localtools_postgres_migration_checker/__main__.py +257 -0
- localtools_postgres_migration_checker-1.0.0/localtools_postgres_migration_checker/engine.py +1429 -0
- localtools_postgres_migration_checker-1.0.0/localtools_postgres_migration_checker/limits.py +36 -0
- localtools_postgres_migration_checker-1.0.0/localtools_postgres_migration_checker/localtools.py +223 -0
- localtools_postgres_migration_checker-1.0.0/localtools_postgres_migration_checker/locks.py +40 -0
- localtools_postgres_migration_checker-1.0.0/localtools_postgres_migration_checker/parser.py +83 -0
- localtools_postgres_migration_checker-1.0.0/pyproject.toml +25 -0
- localtools_postgres_migration_checker-1.0.0/samples/river/LICENSE +374 -0
- localtools_postgres_migration_checker-1.0.0/samples/river/README.md +5 -0
- localtools_postgres_migration_checker-1.0.0/ui/app.js +18 -0
- localtools_postgres_migration_checker-1.0.0/ui/index.html +16 -0
- localtools_postgres_migration_checker-1.0.0/ui/kit/fonts/OFL.txt +93 -0
- localtools_postgres_migration_checker-1.0.0/ui/kit/fonts/jetbrains-mono-latin-bold.woff2 +0 -0
- localtools_postgres_migration_checker-1.0.0/ui/kit/fonts/jetbrains-mono-latin-regular.woff2 +0 -0
- localtools_postgres_migration_checker-1.0.0/ui/kit/localtools.css +189 -0
- localtools_postgres_migration_checker-1.0.0/ui/kit/localtools.js +191 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Chaitanya Prabuddha
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: localtools-postgres-migration-checker
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Check PostgreSQL migrations for the lock each statement takes, what it blocks, and a safe rewrite (localtools, offline)
|
|
5
|
+
Project-URL: Source, https://github.com/chayprabs/localtools
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Requires-Dist: multipart>=1.2
|
|
10
|
+
Requires-Dist: postgast==0.2.0
|
|
11
|
+
Requires-Dist: protobuf<8,>=5.29
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# Postgres Migration Checker
|
|
15
|
+
|
|
16
|
+
Offline static analysis of PostgreSQL migrations: estimated per-statement locks, conflicting traffic, scans/rewrites, transaction conflicts and concrete safer-step examples. SQL is never executed.
|
|
17
|
+
|
|
18
|
+
Install the portable app (no administrator or external runtime):
|
|
19
|
+
|
|
20
|
+
```powershell
|
|
21
|
+
irm https://raw.githubusercontent.com/chayprabs/localtools/main/postgres-migration-checker/install.ps1 | iex
|
|
22
|
+
```
|
|
23
|
+
```sh
|
|
24
|
+
curl -fsSL https://raw.githubusercontent.com/chayprabs/localtools/main/postgres-migration-checker/install.sh | bash
|
|
25
|
+
# Or Python 3.10+: pipx install localtools-postgres-migration-checker
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Run `localtools-postgres-migration-checker` for the UI. Paste UTF-8 SQL or choose files in execution order. Each chosen file can be removed. A new paste or upload replaces the previous input. Copy/download JSON, Markdown or SARIF. Views load 50 statements at a time.
|
|
29
|
+
|
|
30
|
+
```sh
|
|
31
|
+
localtools-postgres-migration-checker check migration.sql --format json
|
|
32
|
+
localtools-postgres-migration-checker check V1__users.sql V2__index.sql --pg 17 --transaction auto
|
|
33
|
+
localtools-postgres-migration-checker check - --transaction always --fail-on medium --format markdown -o report.md
|
|
34
|
+
curl -F 'sql=CREATE INDEX i ON users(email);' http://127.0.0.1:47116/api/check
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
CLI exit 1 means risk at/above `--fail-on` (default high), or an input/runtime error; `--fail-on never` retains findings and exits 0 for a completed report. API fields: `sql` (takes precedence over files), `name`, repeatable `file`, `pg` (11–18, default 18), `transaction` (auto/always/never), `format` (json/markdown/sarif/human). Parse failures are high-risk findings with line/column, so other statements remain visible. Invalid requests return JSON errors with 400/413/422/503.
|
|
38
|
+
|
|
39
|
+
All paths share limits: 100 files, 4 MiB aggregate UTF-8 SQL, 5,000 statements, 16 MiB report, 30 seconds per check, two concurrent checks. Workers have a 512 MiB memory budget (Windows job process memory / Unix address space). The API allows 256 KiB form overhead. Inputs over limits are rejected; deep nesting can be rejected earlier by the PostgreSQL parser. Reports write atomically. No input history is stored. Parent-owned job folders are deleted after completion/cancellation; folders left by a hard crash are swept after one hour at the next start.
|
|
40
|
+
|
|
41
|
+
## Engine and scope
|
|
42
|
+
|
|
43
|
+
Pinned **postgast 0.2.0** (BSD-2-Clause) calls the real **libpg_query PostgreSQL 18.4** parser/deparser. Typed PostgreSQL AST nodes drive the rules; no regex statement splitting. PostgreSQL 11–18 selects known version-dependent rules, not version-specific grammar validation. The abandoned scaffold used GPL pglast: it is not bundled or imported. Squawk is a good general SQL linter, but its CLI does not supply this tool's per-statement lock model, so a separate linter sidecar is unnecessary.
|
|
44
|
+
|
|
45
|
+
The portable ZIP bundles CPython 3.13 and all dependencies. PyPI has postgast wheels for Windows x64, Linux x64 and macOS ARM64. **A plain Intel macOS PyPI install is unavailable**: upstream has no Intel wheel, and its published source archive lacks build headers. Use the self-contained Intel Mac ZIP/one-line installer, or Python 3.13 with `pipx install localtools-postgres-migration-checker --python python3.13 --pip-args="--find-links https://github.com/chayprabs/localtools/releases/download/postgres-migration-checker-v1.0.0/postgast-0.2.0-cp313-cp313-macosx_10_13_x86_64.whl"`. That BSD dependency wheel is built unmodified from the complete upstream v0.2.0 Git tag and submodules (the archive is not used).
|
|
46
|
+
|
|
47
|
+
Locks are **static estimates**, not live observations. Existing schema/data, relations reached through views/triggers/functions/rules, partitions/default partitions, inherited tables, dependencies, search_path resolution and actual durations are unknown. Schema-qualified and unqualified names remain distinct. Newly created tables and validated checks are inferred from the supplied ordered SQL; invalid/rolled-back statements can invalidate those assumptions. Review all examples before use: they can need app changes, synchronization triggers, backups and adaptation to partitioning or schema. No automatic SQL patching is offered.
|
|
48
|
+
|
|
49
|
+
Framework detection reads real SQL comments: goose Up/Down/NO TRANSACTION, dbmate up/down/transaction:false, Liquibase formatted SQL changesets/runInTransaction:false, and Flyway `V…__…sql`/`R__…sql` filenames. Down sections are omitted. Explicit BEGIN/COMMIT is tracked. Rails/Django source code is not SQL; export SQL first and choose the runner transaction mode. Resolve Flyway/Liquibase placeholders before checking. psql command lines/COPY data are skipped with notes. Bodies of functions/DO/dynamic SQL and unmodelled statement kinds are explicitly outside analysis. Runner configuration files, database execution, live schema inspection and lock-time predictions are cut.
|
|
50
|
+
|
|
51
|
+
Primary references: [explicit locking / conflict matrix](https://www.postgresql.org/docs/18/explicit-locking.html), [ALTER TABLE lock exceptions and rewrite conditions](https://www.postgresql.org/docs/18/sql-altertable.html), [concurrent index caveats](https://www.postgresql.org/docs/18/sql-createindex.html), [postgast engine/license](https://github.com/eddieland/postgast/tree/v0.2.0), [libpg_query](https://github.com/pganalyze/libpg_query). These rules are exercised by the built-executable tests in `tests/`.
|
|
52
|
+
|
|
53
|
+
`localtools-postgres-migration-checker update` / `uninstall` use the portable installers; package-manager installations print their appropriate commands. MIT license; bundled dependency notices in THIRD_PARTY_NOTICES.md. Bug reports: the public [localtools issue form](https://github.com/chayprabs/localtools/issues/new?template=postgres-migration-checker.yml).
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Postgres Migration Checker
|
|
2
|
+
|
|
3
|
+
Offline static analysis of PostgreSQL migrations: estimated per-statement locks, conflicting traffic, scans/rewrites, transaction conflicts and concrete safer-step examples. SQL is never executed.
|
|
4
|
+
|
|
5
|
+
Install the portable app (no administrator or external runtime):
|
|
6
|
+
|
|
7
|
+
```powershell
|
|
8
|
+
irm https://raw.githubusercontent.com/chayprabs/localtools/main/postgres-migration-checker/install.ps1 | iex
|
|
9
|
+
```
|
|
10
|
+
```sh
|
|
11
|
+
curl -fsSL https://raw.githubusercontent.com/chayprabs/localtools/main/postgres-migration-checker/install.sh | bash
|
|
12
|
+
# Or Python 3.10+: pipx install localtools-postgres-migration-checker
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
Run `localtools-postgres-migration-checker` for the UI. Paste UTF-8 SQL or choose files in execution order. Each chosen file can be removed. A new paste or upload replaces the previous input. Copy/download JSON, Markdown or SARIF. Views load 50 statements at a time.
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
localtools-postgres-migration-checker check migration.sql --format json
|
|
19
|
+
localtools-postgres-migration-checker check V1__users.sql V2__index.sql --pg 17 --transaction auto
|
|
20
|
+
localtools-postgres-migration-checker check - --transaction always --fail-on medium --format markdown -o report.md
|
|
21
|
+
curl -F 'sql=CREATE INDEX i ON users(email);' http://127.0.0.1:47116/api/check
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
CLI exit 1 means risk at/above `--fail-on` (default high), or an input/runtime error; `--fail-on never` retains findings and exits 0 for a completed report. API fields: `sql` (takes precedence over files), `name`, repeatable `file`, `pg` (11–18, default 18), `transaction` (auto/always/never), `format` (json/markdown/sarif/human). Parse failures are high-risk findings with line/column, so other statements remain visible. Invalid requests return JSON errors with 400/413/422/503.
|
|
25
|
+
|
|
26
|
+
All paths share limits: 100 files, 4 MiB aggregate UTF-8 SQL, 5,000 statements, 16 MiB report, 30 seconds per check, two concurrent checks. Workers have a 512 MiB memory budget (Windows job process memory / Unix address space). The API allows 256 KiB form overhead. Inputs over limits are rejected; deep nesting can be rejected earlier by the PostgreSQL parser. Reports write atomically. No input history is stored. Parent-owned job folders are deleted after completion/cancellation; folders left by a hard crash are swept after one hour at the next start.
|
|
27
|
+
|
|
28
|
+
## Engine and scope
|
|
29
|
+
|
|
30
|
+
Pinned **postgast 0.2.0** (BSD-2-Clause) calls the real **libpg_query PostgreSQL 18.4** parser/deparser. Typed PostgreSQL AST nodes drive the rules; no regex statement splitting. PostgreSQL 11–18 selects known version-dependent rules, not version-specific grammar validation. The abandoned scaffold used GPL pglast: it is not bundled or imported. Squawk is a good general SQL linter, but its CLI does not supply this tool's per-statement lock model, so a separate linter sidecar is unnecessary.
|
|
31
|
+
|
|
32
|
+
The portable ZIP bundles CPython 3.13 and all dependencies. PyPI has postgast wheels for Windows x64, Linux x64 and macOS ARM64. **A plain Intel macOS PyPI install is unavailable**: upstream has no Intel wheel, and its published source archive lacks build headers. Use the self-contained Intel Mac ZIP/one-line installer, or Python 3.13 with `pipx install localtools-postgres-migration-checker --python python3.13 --pip-args="--find-links https://github.com/chayprabs/localtools/releases/download/postgres-migration-checker-v1.0.0/postgast-0.2.0-cp313-cp313-macosx_10_13_x86_64.whl"`. That BSD dependency wheel is built unmodified from the complete upstream v0.2.0 Git tag and submodules (the archive is not used).
|
|
33
|
+
|
|
34
|
+
Locks are **static estimates**, not live observations. Existing schema/data, relations reached through views/triggers/functions/rules, partitions/default partitions, inherited tables, dependencies, search_path resolution and actual durations are unknown. Schema-qualified and unqualified names remain distinct. Newly created tables and validated checks are inferred from the supplied ordered SQL; invalid/rolled-back statements can invalidate those assumptions. Review all examples before use: they can need app changes, synchronization triggers, backups and adaptation to partitioning or schema. No automatic SQL patching is offered.
|
|
35
|
+
|
|
36
|
+
Framework detection reads real SQL comments: goose Up/Down/NO TRANSACTION, dbmate up/down/transaction:false, Liquibase formatted SQL changesets/runInTransaction:false, and Flyway `V…__…sql`/`R__…sql` filenames. Down sections are omitted. Explicit BEGIN/COMMIT is tracked. Rails/Django source code is not SQL; export SQL first and choose the runner transaction mode. Resolve Flyway/Liquibase placeholders before checking. psql command lines/COPY data are skipped with notes. Bodies of functions/DO/dynamic SQL and unmodelled statement kinds are explicitly outside analysis. Runner configuration files, database execution, live schema inspection and lock-time predictions are cut.
|
|
37
|
+
|
|
38
|
+
Primary references: [explicit locking / conflict matrix](https://www.postgresql.org/docs/18/explicit-locking.html), [ALTER TABLE lock exceptions and rewrite conditions](https://www.postgresql.org/docs/18/sql-altertable.html), [concurrent index caveats](https://www.postgresql.org/docs/18/sql-createindex.html), [postgast engine/license](https://github.com/eddieland/postgast/tree/v0.2.0), [libpg_query](https://github.com/pganalyze/libpg_query). These rules are exercised by the built-executable tests in `tests/`.
|
|
39
|
+
|
|
40
|
+
`localtools-postgres-migration-checker update` / `uninstall` use the portable installers; package-manager installations print their appropriate commands. MIT license; bundled dependency notices in THIRD_PARTY_NOTICES.md. Bug reports: the public [localtools issue form](https://github.com/chayprabs/localtools/issues/new?template=postgres-migration-checker.yml).
|
|
File without changes
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import select
|
|
6
|
+
import shutil
|
|
7
|
+
import socket
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import tempfile
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
import multipart
|
|
16
|
+
from . import engine
|
|
17
|
+
from .localtools import HTTPError, Tool, main as run
|
|
18
|
+
from .limits import limit_memory
|
|
19
|
+
|
|
20
|
+
USAGE = '''localtools-postgres-migration-checker: offline PostgreSQL migration safety estimates.
|
|
21
|
+
|
|
22
|
+
Usage:
|
|
23
|
+
localtools-postgres-migration-checker [serve] [--port N] [--no-open]
|
|
24
|
+
localtools-postgres-migration-checker check FILE... [--pg 11..18] [--transaction auto|always|never]
|
|
25
|
+
[--format human|json|markdown|sarif] [--fail-on high|medium|low|never] [-o FILE]
|
|
26
|
+
localtools-postgres-migration-checker update | uninstall | --version | --help
|
|
27
|
+
|
|
28
|
+
FILE '-' reads UTF-8 SQL from stdin. Files run in supplied order. No SQL is executed.
|
|
29
|
+
Limits: 100 files, 4 MiB aggregate SQL, 5000 statements, 16 MiB report, 30 seconds, 2 concurrent checks.
|
|
30
|
+
POST /api/check: multipart sql, name, file (repeatable), pg, transaction, format.
|
|
31
|
+
'''
|
|
32
|
+
UI = Path(__file__).with_name('ui')
|
|
33
|
+
if not UI.is_dir():
|
|
34
|
+
UI = Path(__file__).parents[1] / 'ui'
|
|
35
|
+
DEADLINE = 30
|
|
36
|
+
MAX_REPORT = 16 << 20
|
|
37
|
+
SLOTS = threading.BoundedSemaphore(2)
|
|
38
|
+
PREFIX = 'lt-pgmc-'
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def sweep():
|
|
42
|
+
for p in Path(tempfile.gettempdir()).glob(PREFIX + '*'):
|
|
43
|
+
try:
|
|
44
|
+
if (re.fullmatch(r'lt-pgmc-[a-z0-9_]{8}', p.name) and not p.is_symlink()
|
|
45
|
+
and not (getattr(p.lstat(), 'st_file_attributes', 0) & 1024)
|
|
46
|
+
and p.is_dir() and time.time() - p.stat().st_mtime > 3600):
|
|
47
|
+
shutil.rmtree(p)
|
|
48
|
+
except OSError:
|
|
49
|
+
pass
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def decode(data, name):
|
|
53
|
+
if b'\x00' in data:
|
|
54
|
+
raise HTTPError(400, f'{name}: NUL bytes are not valid SQL text')
|
|
55
|
+
try:
|
|
56
|
+
return engine.newlines(data.decode('utf-8-sig'))
|
|
57
|
+
except UnicodeDecodeError:
|
|
58
|
+
raise HTTPError(400, f'{name}: expected UTF-8 SQL') from None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def read_form(req):
|
|
62
|
+
ctype, opts = multipart.parse_options_header(req.headers.get('Content-Type', ''))
|
|
63
|
+
if ctype != 'multipart/form-data' or 'boundary' not in opts:
|
|
64
|
+
raise HTTPError(400, 'expected multipart/form-data')
|
|
65
|
+
try:
|
|
66
|
+
size = int(req.headers.get('Content-Length', '0'))
|
|
67
|
+
except ValueError:
|
|
68
|
+
raise HTTPError(400, 'invalid Content-Length') from None
|
|
69
|
+
if size <= 0:
|
|
70
|
+
raise HTTPError(400, 'send SQL or a .sql file')
|
|
71
|
+
if size > engine.MAX_BYTES + (256 << 10):
|
|
72
|
+
raise HTTPError(413, 'limit: 4 MiB aggregate SQL plus 256 KiB form overhead')
|
|
73
|
+
parts, files, fields = [], [], {}
|
|
74
|
+
try:
|
|
75
|
+
for p in multipart.MultipartParser(req.rfile, opts['boundary'], size,
|
|
76
|
+
part_limit=engine.MAX_FILES + 8, memory_limit=1 << 20, partsize_limit=engine.MAX_BYTES):
|
|
77
|
+
parts.append(p)
|
|
78
|
+
if p.filename:
|
|
79
|
+
if p.name != 'file':
|
|
80
|
+
raise HTTPError(400, 'uploaded SQL must use the file field')
|
|
81
|
+
name = p.filename.replace('\\', '/').split('/')[-1][:200]
|
|
82
|
+
files.append((name, decode(p.raw, name)))
|
|
83
|
+
else:
|
|
84
|
+
if p.name not in ('sql', 'name', 'pg', 'transaction', 'format') or p.name in fields:
|
|
85
|
+
raise HTTPError(400, 'unknown or repeated form field')
|
|
86
|
+
fields[p.name] = decode(p.raw, p.name)
|
|
87
|
+
if fields.get('sql', '').strip():
|
|
88
|
+
files = [(fields.get('name', 'pasted.sql')[:200], fields['sql'])]
|
|
89
|
+
return files, fields
|
|
90
|
+
except multipart.ParserLimitReached:
|
|
91
|
+
raise HTTPError(413, 'upload exceeds file, field, or 4 MiB SQL limit') from None
|
|
92
|
+
except multipart.MultipartError:
|
|
93
|
+
raise HTTPError(400, 'malformed multipart upload or field limit exceeded') from None
|
|
94
|
+
finally:
|
|
95
|
+
for p in parts:
|
|
96
|
+
p.close()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def disconnected(req):
|
|
100
|
+
if req is None:
|
|
101
|
+
return False
|
|
102
|
+
try:
|
|
103
|
+
ready, _, _ = select.select([req.connection], [], [], 0)
|
|
104
|
+
return bool(ready) and req.connection.recv(1, socket.MSG_PEEK) == b''
|
|
105
|
+
except OSError:
|
|
106
|
+
return True
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def worker(args):
|
|
110
|
+
# EOF is reliable on all platforms and kills the worker if its parent dies.
|
|
111
|
+
def parent_watch():
|
|
112
|
+
os.read(sys.stdin.fileno(), 1)
|
|
113
|
+
os._exit(1)
|
|
114
|
+
threading.Thread(target=parent_watch, daemon=True).start()
|
|
115
|
+
path = Path(args[0])
|
|
116
|
+
try:
|
|
117
|
+
memory_job = limit_memory()
|
|
118
|
+
job = json.loads(path.read_text(encoding='utf-8'))
|
|
119
|
+
rep = engine.check(job['files'], pg=job['pg'], txn_mode=job['transaction'])
|
|
120
|
+
fmt = job['format']
|
|
121
|
+
output = (engine.markdown(rep) if fmt == 'markdown' else engine.human(rep) if fmt == 'human'
|
|
122
|
+
else engine.dumps(engine.sarif(rep, '1.0.0')) if fmt == 'sarif' else engine.dumps(rep))
|
|
123
|
+
if len(output.encode('utf-8')) > MAX_REPORT:
|
|
124
|
+
raise ValueError('report exceeds 16 MiB; split the migration')
|
|
125
|
+
res = {'output': output, 'failed': engine.failing(rep, job.get('fail_on', 'high'))}
|
|
126
|
+
except MemoryError:
|
|
127
|
+
res = {'error': 'SQL exceeds the 512 MiB parser worker memory budget; split it', 'code': 422}
|
|
128
|
+
except (ValueError, RecursionError) as e:
|
|
129
|
+
res = {'error': str(e) if not isinstance(e, RecursionError) else 'SQL is nested too deeply', 'code': 422}
|
|
130
|
+
except Exception:
|
|
131
|
+
res = {'error': 'unsupported SQL structure; split the migration or report this input', 'code': 422}
|
|
132
|
+
path.with_suffix('.out').write_text(json.dumps(res, ensure_ascii=False), encoding='utf-8')
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def execute(files, fields, req=None):
|
|
136
|
+
try:
|
|
137
|
+
pg = int(fields.get('pg') or engine.PG_DEFAULT)
|
|
138
|
+
except ValueError:
|
|
139
|
+
raise HTTPError(400, 'pg must be an integer from 11 to 18') from None
|
|
140
|
+
txn = fields.get('transaction', 'auto')
|
|
141
|
+
fmt = fields.get('format', 'json')
|
|
142
|
+
if not engine.PG_MIN <= pg <= engine.PG_MAX or txn not in engine.TXN_MODES:
|
|
143
|
+
raise HTTPError(400, 'pg must be 11-18; transaction must be auto, always or never')
|
|
144
|
+
if fmt not in ('json', 'markdown', 'sarif', 'human'):
|
|
145
|
+
raise HTTPError(400, 'format must be json, markdown, sarif or human')
|
|
146
|
+
if not files or not any(t.strip() for _, t in files):
|
|
147
|
+
raise HTTPError(400, 'paste SQL or upload a non-empty .sql file')
|
|
148
|
+
if len(files) > engine.MAX_FILES or sum(len(t.encode('utf-8')) for _, t in files) > engine.MAX_BYTES:
|
|
149
|
+
raise HTTPError(413, 'limit: 100 files and 4 MiB aggregate UTF-8 SQL')
|
|
150
|
+
if not SLOTS.acquire(blocking=False):
|
|
151
|
+
raise HTTPError(503, 'busy: two checks are running; retry when one finishes')
|
|
152
|
+
try:
|
|
153
|
+
with tempfile.TemporaryDirectory(prefix=PREFIX) as d:
|
|
154
|
+
path = Path(d, 'job.json')
|
|
155
|
+
path.write_text(json.dumps({'files': files, 'pg': pg, 'transaction': txn, 'format': fmt,
|
|
156
|
+
'fail_on': fields.get('fail_on', 'high')}), encoding='utf-8')
|
|
157
|
+
cmd = [sys.executable, *(['-I'] if sys.flags.isolated else []), '-m', __package__, '_worker', str(path)]
|
|
158
|
+
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
|
159
|
+
creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
|
|
160
|
+
end = time.monotonic() + DEADLINE
|
|
161
|
+
try:
|
|
162
|
+
while p.poll() is None:
|
|
163
|
+
if time.monotonic() > end:
|
|
164
|
+
raise HTTPError(422, 'check exceeded 30 seconds; split the migration')
|
|
165
|
+
if disconnected(req):
|
|
166
|
+
raise HTTPError(499, 'client disconnected')
|
|
167
|
+
try:
|
|
168
|
+
p.wait(.1)
|
|
169
|
+
except subprocess.TimeoutExpired:
|
|
170
|
+
pass
|
|
171
|
+
finally:
|
|
172
|
+
if p.poll() is None:
|
|
173
|
+
p.kill()
|
|
174
|
+
p.wait()
|
|
175
|
+
p.stdin.close()
|
|
176
|
+
out = path.with_suffix('.out')
|
|
177
|
+
if not out.is_file() or out.stat().st_size > MAX_REPORT * 2 + 1024:
|
|
178
|
+
raise HTTPError(422, 'parser could not process this SQL; nesting may be too deep')
|
|
179
|
+
res = json.loads(out.read_text(encoding='utf-8'))
|
|
180
|
+
finally:
|
|
181
|
+
SLOTS.release()
|
|
182
|
+
if 'error' in res:
|
|
183
|
+
raise HTTPError(res['code'], res['error'])
|
|
184
|
+
return res
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def api_check(req):
|
|
188
|
+
files, fields = read_form(req)
|
|
189
|
+
res = execute(files, fields, req)
|
|
190
|
+
fmt = fields.get('format', 'json')
|
|
191
|
+
types = {'json': 'application/json', 'sarif': 'application/sarif+json', 'markdown': 'text/markdown', 'human': 'text/plain'}
|
|
192
|
+
req.send_body(200, res['output'].encode('utf-8'), types[fmt] + '; charset=utf-8')
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def cli_check(args):
|
|
196
|
+
p = argparse.ArgumentParser(prog='localtools-postgres-migration-checker check')
|
|
197
|
+
p.add_argument('files', nargs='+')
|
|
198
|
+
p.add_argument('--pg', type=int, default=engine.PG_DEFAULT)
|
|
199
|
+
p.add_argument('--transaction', choices=engine.TXN_MODES, default='auto')
|
|
200
|
+
p.add_argument('--format', choices=('human', 'json', 'markdown', 'sarif'), default='human')
|
|
201
|
+
p.add_argument('--fail-on', choices=('high', 'medium', 'low', 'never'), default='high')
|
|
202
|
+
p.add_argument('-o', '--output')
|
|
203
|
+
a = p.parse_args(args)
|
|
204
|
+
if len(a.files) > engine.MAX_FILES:
|
|
205
|
+
raise ValueError('limit: 100 files')
|
|
206
|
+
files, total = [], 0
|
|
207
|
+
for name in a.files:
|
|
208
|
+
try:
|
|
209
|
+
stream = sys.stdin.buffer if name == '-' else open(name, 'rb')
|
|
210
|
+
try:
|
|
211
|
+
data = stream.read(engine.MAX_BYTES - total + 1)
|
|
212
|
+
finally:
|
|
213
|
+
if name != '-':
|
|
214
|
+
stream.close()
|
|
215
|
+
total += len(data)
|
|
216
|
+
if total > engine.MAX_BYTES:
|
|
217
|
+
raise ValueError('limit: 4 MiB aggregate UTF-8 SQL')
|
|
218
|
+
files.append((Path(name).name if name != '-' else 'stdin.sql', decode(data, name)))
|
|
219
|
+
except OSError:
|
|
220
|
+
raise ValueError(f'cannot read {Path(name).name}') from None
|
|
221
|
+
res = execute(files, vars(a))
|
|
222
|
+
output = res['output']
|
|
223
|
+
if a.output:
|
|
224
|
+
atomic_write(Path(a.output), output)
|
|
225
|
+
else:
|
|
226
|
+
sys.stdout.reconfigure(encoding='utf-8', newline='')
|
|
227
|
+
print(output, end='')
|
|
228
|
+
return int(res['failed'])
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def atomic_write(path, output):
|
|
232
|
+
temp = None
|
|
233
|
+
try:
|
|
234
|
+
with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', newline='', dir=path.parent,
|
|
235
|
+
prefix='.pgmc-', delete=False) as f:
|
|
236
|
+
temp = Path(f.name)
|
|
237
|
+
f.write(output)
|
|
238
|
+
os.replace(temp, path)
|
|
239
|
+
except OSError:
|
|
240
|
+
raise ValueError('cannot write report; check the output folder and permissions') from None
|
|
241
|
+
finally:
|
|
242
|
+
if temp is not None:
|
|
243
|
+
temp.unlink(missing_ok=True)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
tool = Tool(slug='postgres-migration-checker', port=47116, usage=USAGE, ui=UI,
|
|
247
|
+
routes={'POST /api/check': api_check}, cli={'check': cli_check, '_worker': worker})
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def main():
|
|
251
|
+
if len(sys.argv) < 2 or sys.argv[1] != '_worker':
|
|
252
|
+
sweep()
|
|
253
|
+
sys.exit(run(tool))
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
if __name__ == '__main__':
|
|
257
|
+
main()
|