localtools-postgres-migration-checker 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1429 @@
1
+ """Check PostgreSQL migrations: parse with postgast (libpg_query, the real PostgreSQL 18 parser), then for every statement
2
+ report the locks it takes, what they block, whether it rewrites or scans the table, the transaction it runs in, and a
3
+ safe rewrite for risky patterns. Behaviour that changed between versions follows the target version (11-18)."""
4
+ import json
5
+ import re
6
+ from bisect import bisect_right
7
+
8
+ import postgast
9
+ from postgast import nodes as ast, pg_query_pb2 as pb
10
+ from .parser import parse_sql, ParseError, split, sql
11
+ AT, CT, OT, RO, TK, VK = (getattr(pb, n) for n in
12
+ ("AlterTableType", "ConstrType", "ObjectType", "ReindexObjectType", "TransactionStmtKind", "VariableSetKind"))
13
+
14
+ from .locks import AE, AS, EX, MODES, RE, RS, SH, SRE, SUE, blocks, blocks_writes, rank, strongest
15
+
16
+ PG_MIN, PG_MAX, PG_DEFAULT = 11, 18, 18
17
+ SEVERITIES = ("info", "low", "medium", "high")
18
+ MAX_BYTES = 4 << 20 # aggregate UTF-8 SQL bytes, CLI and API
19
+ MAX_STATEMENTS = 5000
20
+ MAX_FILES = 100
21
+ PREVIEW = 600 # characters of each statement echoed back
22
+ SLOW = ("scan", "rewrite", "index", "rows")
23
+ WORK = {
24
+ "none": "no modelled work; consult findings for unknown operations",
25
+ "catalog": "catalog change; normally brief once the lock is granted",
26
+ "scan": "scans the whole table while holding the lock",
27
+ "rewrite": "rewrites the whole table and rebuilds its indexes while holding the lock",
28
+ "index": "reads the whole table to build an index while holding the lock",
29
+ "rows": "time grows with the number of rows it touches",
30
+ }
31
+ TXN_MODES = ("auto", "always", "never")
32
+
33
+ RULES = { # id: short title (SARIF rule names, README table)
34
+ "parse-error": "Statement does not parse",
35
+ "version": "Not supported by the target PostgreSQL version",
36
+ "add-column-rewrite": "ADD COLUMN rewrites the table",
37
+ "add-column-not-null": "ADD COLUMN NOT NULL without a default fails on a non-empty table",
38
+ "add-column-default-volatility": "ADD COLUMN default may be volatile",
39
+ "alter-column-type": "ALTER COLUMN TYPE rewrites the table",
40
+ "set-not-null": "SET NOT NULL scans the table under ACCESS EXCLUSIVE",
41
+ "add-check": "CHECK constraint is validated under ACCESS EXCLUSIVE",
42
+ "add-not-null-constraint": "NOT NULL constraint is validated under ACCESS EXCLUSIVE",
43
+ "add-foreign-key": "FOREIGN KEY is validated while writes to both tables are blocked",
44
+ "add-unique": "UNIQUE / PRIMARY KEY builds an index under ACCESS EXCLUSIVE",
45
+ "add-exclusion": "EXCLUDE constraint builds an index under ACCESS EXCLUSIVE",
46
+ "create-index": "CREATE INDEX blocks writes while it builds",
47
+ "drop-index": "DROP INDEX takes ACCESS EXCLUSIVE on the table",
48
+ "reindex": "REINDEX blocks writes (and reads that use the index)",
49
+ "table-rewrite": "Statement rewrites the whole table under ACCESS EXCLUSIVE",
50
+ "long-lock": "Long operation under a lock that blocks traffic",
51
+ "attach-partition": "ATTACH PARTITION scans the new partition",
52
+ "detach-partition": "DETACH PARTITION takes ACCESS EXCLUSIVE on the parent",
53
+ "partition-of": "CREATE TABLE ... PARTITION OF takes ACCESS EXCLUSIVE on the parent",
54
+ "refresh-matview": "REFRESH MATERIALIZED VIEW blocks reads while it runs",
55
+ "rename": "Rename breaks queries of the running application",
56
+ "drop-column": "DROP COLUMN loses data and breaks code that still uses it",
57
+ "drop-table": "DROP / TRUNCATE loses data",
58
+ "update-all-rows": "UPDATE / DELETE without WHERE touches every row in one transaction",
59
+ "domain-constraint": "Domain constraint is validated against every column that uses it",
60
+ "enum-add-value": "ALTER TYPE ... ADD VALUE inside a transaction",
61
+ "no-transaction": "Statement cannot run inside a transaction block",
62
+ "lock-outside-transaction": "LOCK TABLE outside a transaction block",
63
+ "missing-lock-timeout": "No lock_timeout before a lock that blocks traffic",
64
+ "missing-statement-timeout": "No statement_timeout before a long-running statement",
65
+ "locks-accumulate": "Locks from earlier statements are held until COMMIT",
66
+ "not-analysed": "Statement contents are not analysed",
67
+ }
68
+
69
+
70
+ class St:
71
+ """Analysis of one statement."""
72
+
73
+ def __init__(self, kind):
74
+ self.kind, self.locks, self.work, self.findings, self.notx = kind, [], "none", [], ""
75
+
76
+ def lock(self, target, mode, new=False):
77
+ self.locks.append({"target": target, "mode": mode, "new": bool(new)})
78
+
79
+ def find(self, rule, sev, msg, fix=""):
80
+ self.findings.append({"rule": rule, "severity": sev, "message": msg, "fix": fix.strip("\n")})
81
+
82
+ def slow(self, work):
83
+ order = ("none", "catalog", "rows", "scan", "index", "rewrite")
84
+ if order.index(work) > order.index(self.work):
85
+ self.work = work
86
+
87
+
88
+ class SQLError(ValueError):
89
+ pass
90
+
91
+
92
+ # ---------- names and deparsing ----------
93
+ def qi(name):
94
+ return '"' + name.replace('"', '""') + '"'
95
+
96
+
97
+ def rv(r):
98
+ """RangeVar -> display/key name ("public." dropped, so public.users and users are the same table)."""
99
+ parts = [p for p in (r.catalogname, r.schemaname) if p] + [r.relname]
100
+ return ".".join(qi(p) for p in parts)
101
+
102
+
103
+ def dotted(strings):
104
+ parts = [s.sval for s in strings or () if isinstance(s, ast.String)]
105
+ return ".".join(qi(p) for p in parts)
106
+
107
+
108
+ def ident(*parts, suffix):
109
+ """PostgreSQL-style generated constraint/index name, truncated to 63 bytes like the server does."""
110
+ base = "_".join(p.replace('"', "") for p in parts if p)
111
+ return qi((base[:63 - len(suffix) - 1] + "_" + suffix).lower())
112
+
113
+
114
+ def walk(node):
115
+ stack = [node]
116
+ while stack:
117
+ n = stack.pop()
118
+ if isinstance(n, (tuple, list)):
119
+ stack.extend(n)
120
+ elif isinstance(n, ast.AstNode):
121
+ yield n
122
+ for a in n.__match_args__:
123
+ v = getattr(n, a, None)
124
+ if isinstance(v, (ast.AstNode, tuple, list)):
125
+ stack.append(v)
126
+
127
+
128
+ VOLATILE = {"random", "random_normal", "clock_timestamp", "timeofday", "nextval", "setval", "currval", "lastval",
129
+ "gen_random_uuid", "uuid_generate_v1", "uuid_generate_v1mc", "uuid_generate_v4", "uuidv4", "uuidv7",
130
+ "gen_random_bytes", "txid_current", "pg_current_xact_id", "setseed", "pg_sleep", "dblink"}
131
+ NONVOLATILE = {"now", "transaction_timestamp", "statement_timestamp", "current_setting", "lower", "upper", "length",
132
+ "concat", "concat_ws", "md5", "abs", "round", "trunc", "floor", "ceil", "date_trunc", "make_interval",
133
+ "make_date", "make_timestamp", "make_timestamptz", "to_timestamp", "to_date", "to_char", "to_json",
134
+ "to_jsonb", "json_build_object", "jsonb_build_object", "json_build_array", "jsonb_build_array",
135
+ "array_fill", "replace", "substring", "btrim", "ltrim", "rtrim", "greatest", "least", "timezone",
136
+ "date_part", "extract", "current_date", "current_timestamp", "localtimestamp", "jsonb_object",
137
+ "inet", "int4range", "int8range", "tstzrange", "tsrange", "daterange", "numrange", "point"}
138
+ SERIAL = {"serial", "serial4", "bigserial", "serial8", "smallserial", "serial2"}
139
+
140
+
141
+ def volatility(expr):
142
+ """'volatile', 'unknown' (a function we do not know) or 'stable' (constants, casts, now(), ...)."""
143
+ seen = "stable"
144
+ for n in walk(expr):
145
+ if isinstance(n, ast.FuncCall):
146
+ f = n.funcname[-1].sval.lower()
147
+ if len(n.funcname) > 1 and n.funcname[-2].sval != 'pg_catalog':
148
+ seen = 'unknown'
149
+ continue
150
+ if f in VOLATILE:
151
+ return "volatile"
152
+ if f not in NONVOLATILE:
153
+ seen = "unknown"
154
+ elif isinstance(n, ast.SubLink):
155
+ seen = "unknown"
156
+ return seen
157
+
158
+
159
+ def type_name(tn):
160
+ return sql(tn) if tn is not None else "?"
161
+
162
+
163
+ def is_serial(tn):
164
+ return tn is not None and len(tn.names or ()) == 1 and tn.names[0].sval.lower() in SERIAL
165
+
166
+
167
+ # ---------- preprocessing: framework markers, psql meta-commands, COPY data, Flyway placeholders ----------
168
+ TOKEN = re.compile(r"--|/\*|['\";\n$]")
169
+ DOLLAR = re.compile(r"\$([A-Za-z_\u0080-\uffff][A-Za-z0-9_\u0080-\uffff]*)?\$")
170
+ COPY_STDIN = re.compile(r"^\s*COPY\b.*\bFROM\s+STDIN\b", re.I | re.S)
171
+
172
+
173
+ def _blank(chars, a, b):
174
+ for i in range(a, b):
175
+ if chars[i] != "\n":
176
+ chars[i] = " "
177
+
178
+
179
+ def scrub(text):
180
+ """Blank (same length, newlines kept) psql meta-command lines and COPY ... FROM stdin data blocks, which are not SQL.
181
+ Returns (text, notes). Quotes, comments and dollar quotes are respected so their contents are never touched."""
182
+ spans, notes = [], []
183
+ i, n, stmt, line_start = 0, len(text), 0, True
184
+ while i < n:
185
+ if line_start:
186
+ j = i
187
+ while j < n and text[j] in " \t":
188
+ j += 1
189
+ if j < n and text[j] == "\\":
190
+ e = text.find("\n", j)
191
+ e = n if e < 0 else e
192
+ spans.append((i, e))
193
+ notes.append((j, f"psql meta-command skipped: {text[j:e].strip()[:60]}"))
194
+ i, stmt = e, e
195
+ continue
196
+ line_start = False
197
+ m = TOKEN.search(text, i)
198
+ if not m:
199
+ break
200
+ t, i = m.group(), m.end()
201
+ if t == "\n":
202
+ line_start = True
203
+ elif t == "--":
204
+ e = text.find("\n", i)
205
+ i = n if e < 0 else e
206
+ elif t == "/*":
207
+ depth = 1
208
+ while depth and i < n:
209
+ m2 = re.compile(r"/\*|\*/").search(text, i)
210
+ if not m2:
211
+ i = n
212
+ break
213
+ depth += 1 if m2.group() == "/*" else -1
214
+ i = m2.end()
215
+ elif t == "'":
216
+ esc = m.start() > 0 and text[m.start() - 1] in "eE" and (m.start() < 2 or not (text[m.start() - 2].isalnum() or text[m.start() - 2] == "_"))
217
+ while i < n:
218
+ c = text[i]
219
+ if esc and c == "\\":
220
+ i += 2
221
+ continue
222
+ i += 1
223
+ if c == "'":
224
+ if i < n and text[i] == "'":
225
+ i += 1
226
+ continue
227
+ break
228
+ elif t == '"':
229
+ while i < n:
230
+ c = text[i]
231
+ i += 1
232
+ if c == '"':
233
+ if i < n and text[i] == '"':
234
+ i += 1
235
+ continue
236
+ break
237
+ elif t == "$":
238
+ s = m.start()
239
+ if s and (text[s - 1].isalnum() or text[s - 1] in "_$"):
240
+ continue # $ inside an identifier
241
+ d = DOLLAR.match(text, s)
242
+ if d:
243
+ e = text.find(d.group(), d.end())
244
+ i = n if e < 0 else e + len(d.group())
245
+ elif t == ";":
246
+ if COPY_STDIN.match(text[stmt:m.start()]):
247
+ a = text.find("\n", i)
248
+ if a >= 0:
249
+ e = re.compile(r"^\\\.[ \t]*$", re.M).search(text, a + 1)
250
+ b = e.end() if e else n
251
+ spans.append((a + 1, b))
252
+ notes.append((a + 1, "COPY data block skipped"))
253
+ i = b
254
+ stmt = i
255
+ if not spans:
256
+ return text, notes
257
+ chars = list(text)
258
+ for a, b in spans:
259
+ _blank(chars, a, b)
260
+ return "".join(chars), notes
261
+
262
+
263
+ MARK = re.compile(r"^[ \t]*--[ \t]*(\+goose[ \t]+(up|down|no[ \t]+transaction)|migrate:(up|down)([^\n]*)"
264
+ r"|liquibase[ \t]+formatted[ \t]+sql|changeset[ \t]+[^\n]*)", re.I | re.M)
265
+ FLYWAY = re.compile(r"(^|[\\/])[VRU]\d*[^\\/]*__[^\\/]*\.sql$|(^|[\\/])R__[^\\/]*\.sql$", re.I)
266
+
267
+
268
+ def prepare(text, name, txn_mode="auto"):
269
+ """Returns (sql, units, framework, notes). units = [(start, end, how)]: regions that the runner wraps in one
270
+ transaction. Down sections (goose, dbmate) are blanked so only the up migration is checked."""
271
+ try:
272
+ comments = {t.start for t in postgast.scan(text).tokens if t.token == pb.SQL_COMMENT}
273
+ marks, last, byte = [], 0, 0
274
+ for m in MARK.finditer(text):
275
+ start = m.start() + len(m.group()) - len(m.group().lstrip())
276
+ byte += len(text[last:start].encode('utf-8'))
277
+ last = start
278
+ if byte in comments:
279
+ marks.append(m)
280
+ except postgast.PgQueryError:
281
+ marks = []
282
+ framework, units, chars, notes = "plain SQL", [], None, []
283
+ lower = [m.group(1).lower() for m in marks]
284
+ if any(l.startswith("+goose") for l in lower):
285
+ framework = "goose"
286
+ notx = any(re.match(r"\+goose\s+no\s+transaction", l) for l in lower)
287
+ chars = list(text)
288
+ up = None
289
+ for m, l in zip(marks, lower):
290
+ if l.startswith("+goose") and l.split()[1] in ("up", "down"):
291
+ if up is not None:
292
+ units.append((up, m.start(), "goose"))
293
+ up = m.end() if l.split()[1] == "up" else None
294
+ if l.split()[1] == "down":
295
+ nxt = next((x.start() for x, y in zip(marks, lower) if x.start() > m.start() and y.split()[:2] == ["+goose", "up"]), len(text))
296
+ _blank(chars, m.end(), nxt)
297
+ if up is not None:
298
+ units.append((up, len(text), "goose"))
299
+ if notx:
300
+ units = []
301
+ elif any(l.startswith("migrate:") for l in lower):
302
+ framework = "dbmate"
303
+ chars = list(text)
304
+ for m, l in zip(marks, lower):
305
+ if l.startswith("migrate:down"):
306
+ nxt = next((x.start() for x, y in zip(marks, lower) if x.start() > m.start() and y.startswith("migrate:up")), len(text))
307
+ _blank(chars, m.end(), nxt)
308
+ elif l.startswith("migrate:up") and not re.search(r"transaction:\s*false", l):
309
+ nxt = next((x.start() for x, y in zip(marks, lower) if x.start() > m.start() and y.startswith("migrate:")), len(text))
310
+ units.append((m.end(), nxt, "dbmate"))
311
+ elif any(l.startswith("liquibase") for l in lower):
312
+ framework = "Liquibase"
313
+ sets = [(m, l) for m, l in zip(marks, lower) if l.startswith("changeset")]
314
+ for k, (m, l) in enumerate(sets):
315
+ end = sets[k + 1][0].start() if k + 1 < len(sets) else len(text)
316
+ if not re.search(r"runintransaction:\s*false", l):
317
+ units.append((m.end(), end, "Liquibase changeset"))
318
+ elif FLYWAY.search(name.replace("\\", "/")):
319
+ framework = "Flyway"
320
+ units = [(0, len(text), "Flyway")]
321
+ if chars is not None:
322
+ text = "".join(chars)
323
+ if txn_mode == "always":
324
+ units = [(0, len(text), "runner")]
325
+ elif txn_mode == "never":
326
+ units = []
327
+ text, more = scrub(text)
328
+ return text, units, framework, notes + more
329
+
330
+
331
+ # ---------- parsing ----------
332
+ def _err_at(text, e):
333
+ """The parser boundary already converts native UTF-8 byte offsets."""
334
+ msg, pos = e.args[0], e.args[1] if len(e.args) > 1 else None
335
+ return msg, len(text.rstrip()) if pos is None else min(pos, len(text))
336
+
337
+
338
+ def parse(text):
339
+ """[(start, end, stmt | None, error | None)] with character offsets. One bad statement does not hide the others."""
340
+ try:
341
+ return [(r.stmt_location, r.stmt_location + r.stmt_len if r.stmt_len else len(text), r.stmt, None)
342
+ for r in parse_sql(text)]
343
+ except ParseError:
344
+ pass
345
+ except RecursionError:
346
+ raise SQLError("the SQL is nested too deeply to analyse")
347
+ out, base = [], text
348
+ try:
349
+ slices = split(base, with_parser=False, only_slices=True)
350
+ tail = None
351
+ except ParseError as e: # the scanner itself failed (unterminated quote or comment): split up to that point
352
+ msg, pos = _err_at(base, e)
353
+ try:
354
+ slices = split(base[:pos], with_parser=False, only_slices=True)
355
+ except ParseError:
356
+ slices = ()
357
+ last = slices[-1].start if slices else 0
358
+ slices = slices[:-1] if slices else ()
359
+ tail = (last + len(base[last:]) - len(base[last:].lstrip()), len(base), None, (msg, pos))
360
+ for s in slices:
361
+ piece = base[s]
362
+ try:
363
+ for r in parse_sql(piece):
364
+ out.append((s.start + r.stmt_location, s.start + (r.stmt_location + r.stmt_len if r.stmt_len else len(piece)), r.stmt, None))
365
+ except ParseError as e:
366
+ msg, pos = _err_at(piece, e)
367
+ out.append((s.start, s.stop, None, (msg, s.start + pos)))
368
+ except RecursionError:
369
+ out.append((s.start, s.stop, None, ("nested too deeply to analyse", s.start)))
370
+ if tail:
371
+ out.append(tail)
372
+ return out
373
+
374
+
375
+ # ---------- per-statement rules ----------
376
+ class Ctx:
377
+ def __init__(self, pg):
378
+ self.pg = pg
379
+ self.tables = {} # tables created in these migrations: key -> {"cols": {name: type}, "partkey": [cols] | None}
380
+ self.nn_checks = {} # (table, constraint) -> column, for CHECK (col IS NOT NULL) [NOT VALID]
381
+ self.nn_ok = set() # (table, column) proven NOT NULL by a validated check
382
+ self.cons = {} # (table, constraint) -> referenced table for foreign keys
383
+
384
+ def new(self, t):
385
+ return t in self.tables
386
+
387
+
388
+ def need(st, ctx, ver, what):
389
+ if ctx.pg < ver:
390
+ st.find("version", "high", f"{what} needs PostgreSQL {ver} or later; this statement fails on PostgreSQL {ctx.pg}.")
391
+
392
+
393
+ def not_null_fix(t, c, pg):
394
+ name = ident(t.split(".")[-1], c, suffix="not_null")
395
+ if pg >= 18:
396
+ return (f"ALTER TABLE {t} ADD CONSTRAINT {name} NOT NULL {c} NOT VALID; -- instant\n"
397
+ f"ALTER TABLE {t} VALIDATE CONSTRAINT {name}; -- scans under SHARE UPDATE EXCLUSIVE: reads and writes continue")
398
+ fix = (f"ALTER TABLE {t} ADD CONSTRAINT {name} CHECK ({c} IS NOT NULL) NOT VALID; -- instant\n"
399
+ f"ALTER TABLE {t} VALIDATE CONSTRAINT {name}; -- scans under SHARE UPDATE EXCLUSIVE: reads and writes continue\n")
400
+ if pg >= 12:
401
+ return fix + (f"ALTER TABLE {t} ALTER COLUMN {c} SET NOT NULL; -- no scan: the validated CHECK proves it (PostgreSQL 12+)\n"
402
+ f"ALTER TABLE {t} DROP CONSTRAINT {name};")
403
+ return fix + "-- PostgreSQL 11 always scans for SET NOT NULL; keep the validated CHECK instead (upgrade to 12+ to convert it)."
404
+
405
+
406
+ def backfill(t, c, expr):
407
+ return (f"-- backfill existing rows in batches, each in its own short transaction; repeat until it updates 0 rows:\n"
408
+ f"UPDATE {t} SET {c} = {expr} WHERE ctid IN (SELECT ctid FROM {t} WHERE {c} IS NULL LIMIT 5000);")
409
+
410
+
411
+ NOTX_FIX = ("-- run it in its own migration without a transaction: goose `-- +goose NO TRANSACTION`, dbmate\n"
412
+ "-- `-- migrate:up transaction:false`, Rails `disable_ddl_transaction!`, Django `atomic = False`,\n"
413
+ "-- Flyway `executeInTransaction=false`, Liquibase `runInTransaction:false`; and no BEGIN/COMMIT around it.")
414
+ REPACK = "-- core PostgreSQL has no online variant; pg_repack or pg_squeeze rebuild a table without a long ACCESS EXCLUSIVE lock."
415
+
416
+
417
+ def add_column(st, ctx, t, cd):
418
+ st.slow('catalog')
419
+ c, tn = qi(cd.colname), cd.type_name
420
+ cons = list(cd.constraints or ())
421
+ kinds = {k.contype for k in cons}
422
+ default = next((k.raw_expr for k in cons if k.contype == CT.CONSTR_DEFAULT), None)
423
+ gen = next((k for k in cons if k.contype == CT.CONSTR_GENERATED), None)
424
+ notnull = CT.CONSTR_NOTNULL in kinds or CT.CONSTR_PRIMARY in kinds
425
+ typ = type_name(tn)
426
+ new = ctx.new(t)
427
+ if new:
428
+ ctx.tables[t]["cols"][cd.colname] = typ
429
+ rewrite, why = False, ""
430
+ if gen is not None:
431
+ virtual = gen.generated_kind != "s"
432
+ if virtual:
433
+ need(st, ctx, 18, "A VIRTUAL generated column (the default kind without STORED)")
434
+ else:
435
+ rewrite, why = True, "a STORED generated column is computed for every existing row"
436
+ elif CT.CONSTR_IDENTITY in kinds:
437
+ rewrite, why = True, "an identity column is filled from its sequence for every existing row"
438
+ elif is_serial(tn):
439
+ rewrite, why = True, f"{typ} has a volatile default (nextval) that is evaluated for every existing row"
440
+ elif default is not None and not (isinstance(default, ast.A_Const) and default.isnull):
441
+ vol = volatility(default)
442
+ if vol == "volatile":
443
+ rewrite, why = True, f"the default {sql(default)} is volatile, so it is evaluated for every existing row"
444
+ elif ctx.pg < 11:
445
+ rewrite, why = True, "before PostgreSQL 11 any non-null default rewrites the table"
446
+ elif vol == "unknown" and not new:
447
+ st.find("add-column-default-volatility", "medium",
448
+ f"The default {sql(default)} calls a function this checker does not know. If it is VOLATILE, adding the "
449
+ f"column rewrites {t} under ACCESS EXCLUSIVE; if it is STABLE or IMMUTABLE the change is instant (PostgreSQL 11+).",
450
+ f"-- check: SELECT proname, provolatile FROM pg_proc WHERE proname = '...'; ('v' = volatile)\n"
451
+ f"ALTER TABLE {t} ADD COLUMN {c} {typ};\nALTER TABLE {t} ALTER COLUMN {c} SET DEFAULT {sql(default)};\n"
452
+ + backfill(t, c, sql(default)))
453
+ if rewrite:
454
+ st.slow("rewrite")
455
+ if not new:
456
+ if gen is not None:
457
+ fix = ((f"-- PostgreSQL 18+: a VIRTUAL generated column is computed on read; adding it is instant\n"
458
+ f"ALTER TABLE {t} ADD COLUMN {c} {typ} GENERATED ALWAYS AS ({sql(gen.raw_expr)}) VIRTUAL;\n" if ctx.pg >= 18 else "")
459
+ + f"-- or a plain column kept up to date by a trigger, backfilled in batches:\nALTER TABLE {t} ADD COLUMN {c} {typ};\n"
460
+ + backfill(t, c, sql(gen.raw_expr)))
461
+ elif CT.CONSTR_IDENTITY in kinds or is_serial(tn):
462
+ base = {"bigserial": "bigint", "serial8": "bigint", "smallserial": "smallint", "serial2": "smallint"}.get(
463
+ tn.names[0].sval.lower(), "integer") if is_serial(tn) else typ
464
+ fix = (f"ALTER TABLE {t} ADD COLUMN {c} {base}; -- instant\n"
465
+ f"-- coordinate new writes and backfill unique values in short transactions.\n"
466
+ f"-- validate NOT NULL before ADD IDENTITY; then create/align its sequence above every existing value.\n"
467
+ f"-- sequence ownership, privileges and concurrent inserts require a schema-specific plan.")
468
+ else:
469
+ fix = (f"ALTER TABLE {t} ADD COLUMN {c} {typ}; -- instant\n"
470
+ f"ALTER TABLE {t} ALTER COLUMN {c} SET DEFAULT {sql(default)}; -- new rows only, no rewrite\n"
471
+ + backfill(t, c, sql(default)))
472
+ if notnull:
473
+ fix += "\n" + not_null_fix(t, c, ctx.pg)
474
+ st.find("add-column-rewrite", "high", f"Adding {c} rewrites {t} while holding ACCESS EXCLUSIVE: {why}. "
475
+ "Reads and writes wait for the whole rewrite.", fix)
476
+ elif notnull and default is None and gen is None and CT.CONSTR_IDENTITY not in kinds and not is_serial(tn) and not new:
477
+ st.find("add-column-not-null", "medium", f"Adding NOT NULL column {c} without a default fails with "
478
+ f"\"column contains null values\" if {t} has any rows.",
479
+ (f"ALTER TABLE {t} ADD COLUMN {c} {typ} NOT NULL DEFAULT <constant>; -- constant default: no rewrite on PostgreSQL 11+\n"
480
+ if ctx.pg >= 11 else "") + f"-- or add it nullable, backfill, then:\n" + not_null_fix(t, c, ctx.pg))
481
+ for k in cons:
482
+ if k.contype in (CT.CONSTR_PRIMARY, CT.CONSTR_UNIQUE):
483
+ st.slow("index")
484
+ if not new:
485
+ idx = ident(t.split(".")[-1], cd.colname, suffix="key")
486
+ st.find("add-unique", "high", f"The inline {'PRIMARY KEY' if k.contype == CT.CONSTR_PRIMARY else 'UNIQUE'} "
487
+ f"builds an index on {t} under ACCESS EXCLUSIVE.",
488
+ f"ALTER TABLE {t} ADD COLUMN {c} {typ};\nCREATE UNIQUE INDEX CONCURRENTLY {idx} ON {t} ({c}); -- outside a transaction\n"
489
+ f"ALTER TABLE {t} ADD CONSTRAINT {idx} {'PRIMARY KEY' if k.contype == CT.CONSTR_PRIMARY else 'UNIQUE'} USING INDEX {idx};")
490
+ elif k.contype == CT.CONSTR_FOREIGN:
491
+ ref = rv(k.pktable)
492
+ st.lock(ref, SRE, ctx.new(ref))
493
+ if default is not None and not new:
494
+ st.slow("scan")
495
+ elif k.contype == CT.CONSTR_CHECK and not new and (default is not None or gen is not None):
496
+ st.slow("scan")
497
+
498
+
499
+ def alter_type(st, ctx, t, cmd):
500
+ c, cd = qi(cmd.name), cmd.def_
501
+ typ = type_name(cd.type_name)
502
+ if ctx.new(t):
503
+ ctx.tables[t]["cols"][cmd.name] = typ
504
+ st.slow("rewrite")
505
+ return
506
+ st.slow("rewrite")
507
+ hint = (" unless the change is binary-coercible (varchar(n) to a longer varchar or text, "
508
+ "numeric(p,s) to a larger precision with the same scale, dropping a length limit, cidr to inet)")
509
+ tz = " (timestamp to timestamptz skips it on PostgreSQL 12+ only when the session TimeZone is UTC)" if "timestamp with time zone" in typ else ""
510
+ tmp = qi((cmd.name + "_new")[:63])
511
+ fix = (f"-- expand/contract instead of one long ACCESS EXCLUSIVE rewrite:\n"
512
+ f"ALTER TABLE {t} ADD COLUMN {tmp} {typ};\n"
513
+ f"-- keep new writes in sync with a trigger (SET NEW.{tmp} := NEW.{c}::{typ}), then backfill in batches:\n"
514
+ f"UPDATE {t} SET {tmp} = {c}::{typ} WHERE ctid IN (SELECT ctid FROM {t} WHERE {tmp} IS NULL AND {c} IS NOT NULL LIMIT 5000);\n"
515
+ f"-- finally, in one short transaction with lock_timeout set:\n"
516
+ f"ALTER TABLE {t} RENAME COLUMN {c} TO {qi((cmd.name + '_old')[:63])};\nALTER TABLE {t} RENAME COLUMN {tmp} TO {c};")
517
+ maybe = re.match(r"(text|varchar|character varying|numeric|varbit|bit varying|inet)\b", typ)
518
+ st.find("alter-column-type", "medium" if maybe else "high",
519
+ f"Changing {c} to {typ} rewrites {t} and rebuilds its indexes under ACCESS EXCLUSIVE{hint}{tz}.", fix)
520
+
521
+
522
+ def add_constraint(st, ctx, t, con, tname):
523
+ new, k = ctx.new(t), con.contype
524
+ name = con.conname
525
+ if k == CT.CONSTR_FOREIGN:
526
+ ref = rv(con.pktable)
527
+ st.lock(t, SRE, new)
528
+ st.lock(ref, SRE, ctx.new(ref))
529
+ if name:
530
+ ctx.cons[(t, name)] = ref
531
+ if con.skip_validation:
532
+ st.slow("catalog")
533
+ else:
534
+ st.slow("scan")
535
+ if not new:
536
+ name = name or ident(tname, *(a.sval for a in con.fk_attrs or ()), suffix="fkey")
537
+ con2 = sql(con).replace(" NOT VALID", "")
538
+ if not con.conname:
539
+ con2 = f"CONSTRAINT {name} " + con2
540
+ st.find("add-foreign-key", "high", f"Adding the foreign key scans {t} to validate every row while holding "
541
+ f"SHARE ROW EXCLUSIVE on {t} and {ref}: writes to both tables wait.",
542
+ f"ALTER TABLE {t} ADD {con2} NOT VALID; -- instant; new rows are checked from now on\n"
543
+ f"ALTER TABLE {t} VALIDATE CONSTRAINT {name}; -- SHARE UPDATE EXCLUSIVE: reads and writes continue")
544
+ return
545
+ st.lock(t, AE, new)
546
+ if k == CT.CONSTR_CHECK:
547
+ col = None
548
+ e = con.raw_expr
549
+ if isinstance(e, ast.NullTest) and e.nulltesttype == pb.IS_NOT_NULL and isinstance(e.arg, ast.ColumnRef) and len(e.arg.fields) == 1:
550
+ col = e.arg.fields[0].sval
551
+ if con.skip_validation:
552
+ st.slow("catalog")
553
+ if col and name:
554
+ ctx.nn_checks[(t, name)] = col
555
+ else:
556
+ st.slow("scan")
557
+ if col:
558
+ ctx.nn_ok.add((t, col))
559
+ if not new:
560
+ name = name or ident(tname, col or "", suffix="check")
561
+ st.find("add-check", "high", f"Adding the CHECK constraint scans all of {t} under ACCESS EXCLUSIVE: reads and writes wait.",
562
+ f"ALTER TABLE {t} ADD CONSTRAINT {name} CHECK ({sql(con.raw_expr)}) NOT VALID; -- instant\n"
563
+ f"ALTER TABLE {t} VALIDATE CONSTRAINT {name}; -- SHARE UPDATE EXCLUSIVE: reads and writes continue")
564
+ elif k == CT.CONSTR_NOTNULL:
565
+ need(st, ctx, 18, "ADD CONSTRAINT ... NOT NULL")
566
+ cols = [s.sval for s in con.keys or ()]
567
+ if con.skip_validation:
568
+ st.slow("catalog")
569
+ if name and cols:
570
+ ctx.nn_checks[(t, name)] = cols[0]
571
+ else:
572
+ st.slow("scan")
573
+ if not new:
574
+ st.find("add-not-null-constraint", "high", f"Adding the NOT NULL constraint scans all of {t} under ACCESS EXCLUSIVE.",
575
+ not_null_fix(t, qi(cols[0]) if cols else "?", ctx.pg))
576
+ elif k in (CT.CONSTR_PRIMARY, CT.CONSTR_UNIQUE):
577
+ what = "PRIMARY KEY" if k == CT.CONSTR_PRIMARY else "UNIQUE"
578
+ if con.nulls_not_distinct:
579
+ need(st, ctx, 15, "UNIQUE NULLS NOT DISTINCT")
580
+ if con.indexname:
581
+ st.slow("catalog" if k == CT.CONSTR_UNIQUE else "scan")
582
+ if k == CT.CONSTR_PRIMARY and not new:
583
+ st.find("add-unique", "low", f"PRIMARY KEY USING INDEX is instant only if the key columns are already NOT NULL; "
584
+ f"otherwise it scans {t} under ACCESS EXCLUSIVE to set NOT NULL.", "-- make each key column NOT NULL first:\n"
585
+ + not_null_fix(t, "<column>", ctx.pg))
586
+ return
587
+ st.slow("index")
588
+ if not new:
589
+ cols = [qi(s.sval) for s in con.keys or ()]
590
+ idx = qi(name) if name else ident(tname, *(s.sval for s in con.keys or ()), suffix="pkey" if k == CT.CONSTR_PRIMARY else "key")
591
+ index_extra = ''
592
+ if con.including:
593
+ index_extra += ' INCLUDE (' + ', '.join(qi(c.sval) for c in con.including) + ')'
594
+ index_extra += ' NULLS NOT DISTINCT' if con.nulls_not_distinct else ''
595
+ constraint_extra = (' DEFERRABLE' if con.deferrable else '') + (' INITIALLY DEFERRED' if con.initdeferred else '')
596
+ st.find("add-unique", "high", f"Adding the {what} builds a unique index on {t} while holding ACCESS EXCLUSIVE: reads and writes wait for the build.",
597
+ f"CREATE UNIQUE INDEX CONCURRENTLY {idx} ON {t} ({', '.join(cols)}){index_extra}; -- outside a transaction; reads and writes continue\n"
598
+ f"ALTER TABLE {t} ADD CONSTRAINT {idx} {what} USING INDEX {idx}{constraint_extra}; -- normally brief"
599
+ + ("\n-- the key columns must already be NOT NULL, or this step scans the table" if k == CT.CONSTR_PRIMARY else ""))
600
+ elif k == CT.CONSTR_EXCLUSION:
601
+ st.slow("index")
602
+ if not new:
603
+ st.find("add-exclusion", "high", f"Adding the EXCLUDE constraint builds an index on {t} under ACCESS EXCLUSIVE; "
604
+ "there is no CONCURRENTLY form for exclusion constraints.",
605
+ "-- run it when the table is quiet, with lock_timeout set; or enforce the rule in the application first.")
606
+ else:
607
+ st.slow("catalog")
608
+
609
+
610
+ REL_OPTS_SUE = re.compile(r"^(fillfactor|autovacuum_.*|toast\..*|toast_tuple_target|parallel_workers|vacuum_.*|"
611
+ r"log_autovacuum_min_duration|user_catalog_table|n_distinct.*)$")
612
+ TRIGGER_CMDS = {AT.AT_EnableTrig, AT.AT_EnableAlwaysTrig, AT.AT_EnableReplicaTrig, AT.AT_DisableTrig, AT.AT_EnableTrigAll,
613
+ AT.AT_DisableTrigAll, AT.AT_EnableTrigUser, AT.AT_DisableTrigUser}
614
+ SUE_CMDS = {AT.AT_SetStatistics, AT.AT_SetOptions, AT.AT_ResetOptions, AT.AT_ClusterOn, AT.AT_DropCluster,
615
+ AT.AT_ValidateConstraint, AT.AT_DetachPartitionFinalize}
616
+
617
+
618
+ def h_AlterTableStmt(s, ctx, st):
619
+ t = rv(s.relation)
620
+ tname = s.relation.relname
621
+ new = ctx.new(t)
622
+ kind = {OT.OBJECT_INDEX: "ALTER INDEX", OT.OBJECT_MATVIEW: "ALTER MATERIALIZED VIEW", OT.OBJECT_VIEW: "ALTER VIEW",
623
+ OT.OBJECT_SEQUENCE: "ALTER SEQUENCE", OT.OBJECT_FOREIGN_TABLE: "ALTER FOREIGN TABLE"}.get(s.objtype, "ALTER TABLE")
624
+ subs = []
625
+ for cmd in s.cmds:
626
+ sub = cmd.subtype
627
+ subs.append(AT.Name(sub)[3:])
628
+ before = len(st.locks)
629
+ if sub == AT.AT_AddColumn:
630
+ st.lock(t, AE, new)
631
+ add_column(st, ctx, t, cmd.def_)
632
+ elif sub == AT.AT_AlterColumnType:
633
+ st.lock(t, AE, new)
634
+ alter_type(st, ctx, t, cmd)
635
+ elif sub == AT.AT_SetNotNull:
636
+ st.lock(t, AE, new)
637
+ col = cmd.name
638
+ if new or (ctx.pg >= 12 and (t, col) in ctx.nn_ok):
639
+ st.slow("catalog")
640
+ else:
641
+ st.slow("scan")
642
+ st.find("set-not-null", "high", f"SET NOT NULL scans all of {t} under ACCESS EXCLUSIVE to check {qi(col)}: "
643
+ "reads and writes wait for the scan.", not_null_fix(t, qi(col), ctx.pg))
644
+ elif sub == AT.AT_AddConstraint:
645
+ add_constraint(st, ctx, t, cmd.def_, tname)
646
+ elif sub == AT.AT_ValidateConstraint:
647
+ st.lock(t, SUE, new)
648
+ st.slow("scan")
649
+ ref = ctx.cons.get((t, cmd.name))
650
+ if ref:
651
+ st.lock(ref, RS, ctx.new(ref))
652
+ col = ctx.nn_checks.get((t, cmd.name))
653
+ if col:
654
+ ctx.nn_ok.add((t, col))
655
+ elif sub == AT.AT_DropColumn:
656
+ st.lock(t, AE, new)
657
+ st.slow("catalog")
658
+ if not new:
659
+ st.find("drop-column", "medium", f"Dropping {qi(cmd.name)} is instant but irreversible, and every running query or "
660
+ "ORM model that still selects or inserts it fails afterwards.",
661
+ "-- 1. stop using the column in the application and deploy (Rails: self.ignored_columns; Django:\n"
662
+ "-- SeparateDatabaseAndState), 2. then drop it with lock_timeout set:\n"
663
+ f"SET lock_timeout = '5s';\nALTER TABLE {t} DROP COLUMN {qi(cmd.name)};")
664
+ elif sub in (AT.AT_SetLogged, AT.AT_SetUnLogged, AT.AT_SetTableSpace, AT.AT_SetAccessMethod, AT.AT_SetExpression):
665
+ st.lock(t, AE, new)
666
+ if sub == AT.AT_SetAccessMethod:
667
+ need(st, ctx, 15, "ALTER TABLE ... SET ACCESS METHOD")
668
+ if sub == AT.AT_SetExpression:
669
+ need(st, ctx, 17, "ALTER COLUMN ... SET EXPRESSION")
670
+ st.slow("rewrite")
671
+ if not new:
672
+ st.find("table-rewrite", "high", f"{AT.Name(sub)[3:]} rewrites (copies) all of {t} under ACCESS EXCLUSIVE: reads and writes wait.", REPACK)
673
+ elif sub in (AT.AT_SetRelOptions, AT.AT_ResetRelOptions):
674
+ names = [d.defname if not d.defnamespace else f"{d.defnamespace}.{d.defname}" for d in (cmd.def_.items if isinstance(cmd.def_, ast.List) else cmd.def_ or ())]
675
+ st.lock(t, SUE if names and all(REL_OPTS_SUE.match(n) for n in names) else AE, new)
676
+ st.slow("catalog")
677
+ elif sub in SUE_CMDS:
678
+ st.lock(t, SUE, new)
679
+ st.slow("catalog")
680
+ elif sub in TRIGGER_CMDS or sub == AT.AT_AlterConstraint:
681
+ st.lock(t, SRE, new)
682
+ st.slow("catalog")
683
+ elif sub == AT.AT_AttachPartition:
684
+ part = rv(cmd.def_.name)
685
+ if s.objtype == OT.OBJECT_INDEX:
686
+ st.lock(t, SUE, new)
687
+ st.lock(part, SUE, ctx.new(part))
688
+ st.slow("catalog")
689
+ continue
690
+ st.lock(t, SUE if ctx.pg >= 12 else AE, new)
691
+ st.lock(part, AE, ctx.new(part))
692
+ st.slow("scan")
693
+ if not ctx.new(part):
694
+ bound = sql(cmd.def_.bound) if cmd.def_.bound is not None else "DEFAULT"
695
+ st.find("attach-partition", "high" if ctx.pg < 12 else "medium",
696
+ f"ATTACH PARTITION scans {part} under ACCESS EXCLUSIVE to prove every row fits the bound"
697
+ + (f", and takes ACCESS EXCLUSIVE on the parent {t} before PostgreSQL 12." if ctx.pg < 12 else
698
+ f" (the parent {t} only gets SHARE UPDATE EXCLUSIVE on PostgreSQL 12+).")
699
+ + " A DEFAULT partition, if any, is scanned too.",
700
+ f"-- a validated CHECK that matches the bound lets ATTACH skip the scan:\n"
701
+ f"ALTER TABLE {part} ADD CONSTRAINT {ident(cmd.def_.name.relname, suffix='bound')} CHECK (<partition key condition for {bound}>) NOT VALID;\n"
702
+ f"ALTER TABLE {part} VALIDATE CONSTRAINT {ident(cmd.def_.name.relname, suffix='bound')}; -- reads and writes continue\n"
703
+ f"ALTER TABLE {t} ATTACH PARTITION {part} {bound};\n"
704
+ f"ALTER TABLE {part} DROP CONSTRAINT {ident(cmd.def_.name.relname, suffix='bound')};")
705
+ elif sub == AT.AT_DetachPartition:
706
+ part = rv(cmd.def_.name)
707
+ if cmd.def_.concurrent:
708
+ need(st, ctx, 14, "DETACH PARTITION ... CONCURRENTLY")
709
+ st.lock(t, SUE, new)
710
+ st.lock(part, SUE, ctx.new(part))
711
+ st.notx = "DETACH PARTITION ... CONCURRENTLY"
712
+ st.slow("catalog")
713
+ else:
714
+ st.lock(t, AE, new)
715
+ st.lock(part, AE, ctx.new(part))
716
+ st.slow("catalog")
717
+ if not new:
718
+ st.find("detach-partition", "medium", f"DETACH PARTITION takes ACCESS EXCLUSIVE on {t}: every query on the "
719
+ "partitioned table waits for it, and it waits for every running one.",
720
+ f"ALTER TABLE {t} DETACH PARTITION {part} CONCURRENTLY; -- PostgreSQL 14+, outside a transaction"
721
+ if ctx.pg >= 14 else "-- PostgreSQL 14+ has DETACH PARTITION ... CONCURRENTLY; on older versions set lock_timeout and retry.")
722
+ elif sub == AT.AT_AddIdentity:
723
+ st.lock(t, AE, new)
724
+ st.slow("catalog")
725
+ else: # defaults, DROP NOT NULL, storage, compression, owner, RLS, rules, inherit, replica identity...: ACCESS EXCLUSIVE, catalog only
726
+ st.lock(t, AE, new)
727
+ st.slow("catalog")
728
+ if len(st.locks) == before:
729
+ st.lock(t, AE, new)
730
+ st.kind = kind + " " + ", ".join(dict.fromkeys(x.replace("_", " ") for x in subs))
731
+
732
+
733
+ def h_IndexStmt(s, ctx, st):
734
+ t = rv(s.relation)
735
+ new = ctx.new(t)
736
+ st.kind = "CREATE " + ("UNIQUE " if s.unique else "") + "INDEX" + (" CONCURRENTLY" if s.concurrent else "")
737
+ st.slow("index")
738
+ if s.nulls_not_distinct:
739
+ need(st, ctx, 15, "NULLS NOT DISTINCT")
740
+ if s.concurrent:
741
+ st.lock(t, SUE, new)
742
+ st.notx = "CREATE INDEX CONCURRENTLY"
743
+ if ctx.tables.get(t, {}).get("partkey"):
744
+ st.find("version", "high", f"{t} is partitioned: CREATE INDEX CONCURRENTLY is not supported on partitioned tables.",
745
+ f"-- create the index on each partition CONCURRENTLY, then CREATE INDEX ON ONLY {t} and ALTER INDEX ... ATTACH PARTITION each one.")
746
+ return
747
+ st.lock(t, SH, new)
748
+ if not new:
749
+ s._pb.concurrent = True
750
+ fix = sql(s) + "; -- outside a transaction; reads and writes continue\n"
751
+ s._pb.concurrent = False
752
+ fix += "-- if it fails it leaves an INVALID index: DROP INDEX CONCURRENTLY it and retry."
753
+ st.find("create-index", "high", f"CREATE INDEX holds SHARE on {t} for the whole build: INSERT, UPDATE and DELETE wait until it finishes.", fix)
754
+
755
+
756
+ def drop_targets(s):
757
+ return [dotted(o.items) if isinstance(o, ast.List) else (dotted(o) if isinstance(o, (tuple, list)) else (dotted([o]) if isinstance(o, ast.String) else dotted(getattr(o, 'objname', ())) or type(o).__name__)) for o in s.objects or ()]
758
+
759
+
760
+ def h_DropStmt(s, ctx, st):
761
+ rt = s.remove_type
762
+ objs = drop_targets(s)
763
+ st.kind = "DROP " + OT.Name(rt)[7:].replace("_", " ") + (" CONCURRENTLY" if s.concurrent else "")
764
+ if rt in (OT.OBJECT_TABLE, OT.OBJECT_MATVIEW, OT.OBJECT_VIEW, OT.OBJECT_FOREIGN_TABLE, OT.OBJECT_SEQUENCE):
765
+ st.slow("catalog")
766
+ for o in objs:
767
+ st.lock(o, AE, ctx.new(o))
768
+ if rt == OT.OBJECT_TABLE and not ctx.new(o):
769
+ st.find("drop-table", "medium", f"DROP TABLE {o} deletes its data irreversibly, and takes ACCESS EXCLUSIVE on it "
770
+ "(and briefly on tables that reference it with foreign keys).",
771
+ "-- remove every use of the table from the application and deploy first; take a backup (pg_dump -t) if in doubt.")
772
+ ctx.tables.pop(o, None)
773
+ elif rt == OT.OBJECT_INDEX:
774
+ st.slow("catalog")
775
+ for o in objs:
776
+ if s.concurrent:
777
+ st.lock(f"table of index {o}", SUE)
778
+ st.notx = "DROP INDEX CONCURRENTLY"
779
+ else:
780
+ st.lock(f"table of index {o}", AE)
781
+ if not s.concurrent:
782
+ st.find("drop-index", "medium", "DROP INDEX takes ACCESS EXCLUSIVE on the index's table: it waits for every running query "
783
+ "on that table, and every new query waits behind it.",
784
+ "\n".join(f"DROP INDEX CONCURRENTLY {'IF EXISTS ' if s.missing_ok else ''}{o};" for o in objs)
785
+ + " -- outside a transaction, one index per statement")
786
+ elif len(objs) > 1:
787
+ st.find("version", "high", "DROP INDEX CONCURRENTLY drops one index per statement; this statement fails.",
788
+ "\n".join(f"DROP INDEX CONCURRENTLY {o};" for o in objs))
789
+ elif rt in (OT.OBJECT_TRIGGER, OT.OBJECT_POLICY, OT.OBJECT_RULE):
790
+ st.slow("catalog")
791
+ for o in s.objects or ():
792
+ parts = [x.sval for x in (o.items if isinstance(o, ast.List) else o)]
793
+ tbl = ".".join(qi(p) for p in parts[:-1] if p != "public")
794
+ st.lock(tbl, AE, ctx.new(tbl))
795
+ elif rt in (OT.OBJECT_SCHEMA, OT.OBJECT_DATABASE):
796
+ st.slow("catalog")
797
+ if rt == OT.OBJECT_DATABASE:
798
+ st.notx = "DROP DATABASE"
799
+ st.find("drop-table", "medium", f"DROP {OT.Name(rt)[7:]} {', '.join(objs)} deletes everything in it irreversibly"
800
+ + (" (CASCADE)." if s.behavior == pb.DROP_CASCADE else "."))
801
+ else:
802
+ st.slow("catalog")
803
+
804
+
805
+ def h_TruncateStmt(s, ctx, st):
806
+ st.kind = "TRUNCATE"
807
+ st.slow("catalog")
808
+ for r in s.relations:
809
+ t = rv(r)
810
+ st.lock(t, AE, ctx.new(t))
811
+ if not ctx.new(t):
812
+ st.find("drop-table", "medium", f"TRUNCATE deletes every row of {t} irreversibly and takes ACCESS EXCLUSIVE on it.")
813
+
814
+
815
+ def h_CreateStmt(s, ctx, st):
816
+ t = rv(s.relation)
817
+ st.kind = "CREATE TABLE" + (" PARTITION OF" if s.partbound is not None else "")
818
+ cols = {}
819
+ for e in s.table_elts or ():
820
+ if isinstance(e, ast.ColumnDef):
821
+ cols[e.colname] = type_name(e.type_name)
822
+ for k in e.constraints or ():
823
+ if k.contype == CT.CONSTR_FOREIGN:
824
+ ref = rv(k.pktable)
825
+ st.lock(ref, SRE, ctx.new(ref) or ref == t)
826
+ elif isinstance(e, ast.Constraint) and e.contype == CT.CONSTR_FOREIGN:
827
+ ref = rv(e.pktable)
828
+ st.lock(ref, SRE, ctx.new(ref) or ref == t)
829
+ existed = ctx.new(t)
830
+ partkey = [p.name for p in s.partspec.part_params if p.name] if s.partspec is not None else None
831
+ if not existed or not s.if_not_exists:
832
+ ctx.tables[t] = {"cols": cols, "partkey": partkey}
833
+ st.lock(t, AE, True)
834
+ st.slow("catalog")
835
+ if s.partbound is not None:
836
+ parent = rv(s.inh_relations[0])
837
+ st.lock(parent, AE, ctx.new(parent))
838
+ if not ctx.new(parent):
839
+ st.find("partition-of", "medium", f"CREATE TABLE ... PARTITION OF takes ACCESS EXCLUSIVE on {parent}: every query on it waits, "
840
+ "and it waits for every running query.",
841
+ f"CREATE TABLE {t} (LIKE {parent} INCLUDING DEFAULTS INCLUDING CONSTRAINTS);\n"
842
+ f"-- add a CHECK matching the bound (NOT VALID, then VALIDATE) so ATTACH does not scan, then:\n"
843
+ f"ALTER TABLE {parent} ATTACH PARTITION {t} {sql(s.partbound)}; -- SHARE UPDATE EXCLUSIVE on the parent (PostgreSQL 12+)"
844
+ if ctx.pg >= 12 else "-- before PostgreSQL 12 ATTACH PARTITION also takes ACCESS EXCLUSIVE on the parent; set lock_timeout and retry.")
845
+
846
+
847
+ def h_CreateTableAsStmt(s, ctx, st):
848
+ t = rv(s.into.rel)
849
+ st.kind = "CREATE MATERIALIZED VIEW" if s.objtype == OT.OBJECT_MATVIEW else "CREATE TABLE AS"
850
+ ctx.tables[t] = {"cols": {}, "partkey": None}
851
+ st.lock(t, AE, True)
852
+ st.slow("rows")
853
+ reads(s.query, ctx, st)
854
+
855
+
856
+ def reads(query, ctx, st, mode=AS):
857
+ for n in walk(query):
858
+ if isinstance(n, ast.RangeVar):
859
+ t = rv(n)
860
+ if not any(l["target"] == t for l in st.locks):
861
+ st.lock(t, mode, ctx.new(t))
862
+
863
+
864
+ def h_RefreshMatViewStmt(s, ctx, st):
865
+ t = rv(s.relation)
866
+ st.kind = "REFRESH MATERIALIZED VIEW" + (" CONCURRENTLY" if s.concurrent else "")
867
+ st.slow("rewrite")
868
+ if s.concurrent:
869
+ st.lock(t, EX, ctx.new(t))
870
+ return
871
+ st.lock(t, AE, ctx.new(t))
872
+ if not ctx.new(t):
873
+ st.find("refresh-matview", "high", f"REFRESH MATERIALIZED VIEW holds ACCESS EXCLUSIVE on {t} while the query reruns: reads of the view wait.",
874
+ f"REFRESH MATERIALIZED VIEW CONCURRENTLY {t}; -- EXCLUSIVE: reads continue; needs a UNIQUE index on the view")
875
+
876
+
877
+ def h_ViewStmt(s, ctx, st):
878
+ t = rv(s.view)
879
+ st.kind = "CREATE OR REPLACE VIEW" if s.replace else "CREATE VIEW"
880
+ st.lock(t, AE, not s.replace or ctx.new(t))
881
+ st.slow("catalog")
882
+ reads(s.query, ctx, st)
883
+
884
+
885
+ def h_RenameStmt(s, ctx, st):
886
+ rt = s.rename_type
887
+ st.kind = f"RENAME {OT.Name(rt)[7:].replace('_', ' ')}"
888
+ st.slow("catalog")
889
+ if s.relation is None:
890
+ return
891
+ t = rv(s.relation)
892
+ new = ctx.new(t)
893
+ if rt == OT.OBJECT_INDEX or (rt == OT.OBJECT_TABLE and s.relation_type == OT.OBJECT_INDEX):
894
+ st.lock(t, SUE if ctx.pg >= 12 else AE, new)
895
+ return
896
+ st.lock(t, AE, new)
897
+ if rt == OT.OBJECT_COLUMN:
898
+ if new:
899
+ ctx.tables[t]["cols"][s.newname] = ctx.tables[t]["cols"].pop(s.subname, "?")
900
+ else:
901
+ st.find("rename", "medium", f"Renaming {t}.{qi(s.subname)} to {qi(s.newname)} breaks every query of the running application "
902
+ "that still uses the old name, from the moment it commits until the new code is deployed.",
903
+ f"-- expand/contract: ADD COLUMN {qi(s.newname)}, keep both in sync with a trigger, backfill, switch the code to\n"
904
+ f"-- the new column, then drop {qi(s.subname)}. Or deploy code that tolerates both names before renaming.")
905
+ elif rt in (OT.OBJECT_TABLE, OT.OBJECT_VIEW, OT.OBJECT_MATVIEW, OT.OBJECT_FOREIGN_TABLE):
906
+ newt = ".".join(t.split(".")[:-1] + [qi(s.newname)])
907
+ if new:
908
+ ctx.tables[newt] = ctx.tables.pop(t)
909
+ elif rt == OT.OBJECT_TABLE:
910
+ st.find("rename", "medium", f"Renaming {t} to {qi(s.newname)} breaks every query of the running application that still uses the old name.",
911
+ f"ALTER TABLE {t} RENAME TO {qi(s.newname)};\n"
912
+ f"CREATE VIEW {t} AS SELECT * FROM {newt}; -- same transaction: old name keeps working (simple views are updatable)\n"
913
+ f"-- drop the view once every client uses {qi(s.newname)}")
914
+
915
+
916
+ def h_AlterObjectSchemaStmt(s, ctx, st):
917
+ st.kind = f"ALTER {OT.Name(s.object_type)[7:]} SET SCHEMA"
918
+ st.slow("catalog")
919
+ if s.relation is not None:
920
+ t = rv(s.relation)
921
+ st.lock(t, AE, ctx.new(t))
922
+ if not ctx.new(t) and s.object_type == OT.OBJECT_TABLE:
923
+ st.find("rename", "medium", f"Moving {t} to schema {qi(s.newschema)} breaks queries that use the old qualified name or rely on search_path.")
924
+
925
+
926
+ def h_CreateTrigStmt(s, ctx, st):
927
+ t = rv(s.relation)
928
+ st.kind = "CREATE OR REPLACE TRIGGER" if s.replace else ("CREATE CONSTRAINT TRIGGER" if s.isconstraint else "CREATE TRIGGER")
929
+ if s.replace:
930
+ need(st, ctx, 14, "CREATE OR REPLACE TRIGGER")
931
+ st.lock(t, SRE, ctx.new(t))
932
+ st.slow("catalog")
933
+
934
+
935
+ def h_CreatePolicyStmt(s, ctx, st, kind="CREATE POLICY"):
936
+ t = rv(s.table)
937
+ st.kind = kind
938
+ st.lock(t, AE, ctx.new(t))
939
+ st.slow("catalog")
940
+
941
+
942
+ def h_AlterPolicyStmt(s, ctx, st):
943
+ h_CreatePolicyStmt(s, ctx, st, "ALTER POLICY")
944
+
945
+
946
+ def h_RuleStmt(s, ctx, st):
947
+ t = rv(s.relation)
948
+ st.kind = "CREATE RULE"
949
+ st.lock(t, AE, ctx.new(t))
950
+ st.slow("catalog")
951
+
952
+
953
+ COMMENT_REL = {OT.OBJECT_TABLE, OT.OBJECT_COLUMN, OT.OBJECT_INDEX, OT.OBJECT_VIEW, OT.OBJECT_MATVIEW, OT.OBJECT_SEQUENCE,
954
+ OT.OBJECT_FOREIGN_TABLE, OT.OBJECT_TABCONSTRAINT, OT.OBJECT_TRIGGER, OT.OBJECT_POLICY, OT.OBJECT_RULE}
955
+
956
+
957
+ def h_CommentStmt(s, ctx, st):
958
+ st.kind = f"COMMENT ON {OT.Name(s.objtype)[7:].replace('_', ' ')}"
959
+ st.slow("catalog")
960
+ if s.objtype in COMMENT_REL and isinstance(s.object, (tuple, list, ast.List)):
961
+ parts = [x.sval for x in (s.object.items if isinstance(s.object, ast.List) else s.object) if isinstance(x, ast.String)]
962
+ if s.objtype in (OT.OBJECT_COLUMN, OT.OBJECT_TABCONSTRAINT, OT.OBJECT_TRIGGER, OT.OBJECT_POLICY, OT.OBJECT_RULE):
963
+ parts = parts[:-1]
964
+ t = ".".join(qi(p) for p in parts)
965
+ st.lock(t, SUE, ctx.new(t))
966
+
967
+
968
+ def h_CreateStatsStmt(s, ctx, st):
969
+ st.kind = "CREATE STATISTICS"
970
+ st.slow("catalog")
971
+ for r in s.relations or ():
972
+ if isinstance(r, ast.RangeVar):
973
+ st.lock(rv(r), SUE, ctx.new(rv(r)))
974
+
975
+
976
+ def h_VacuumStmt(s, ctx, st):
977
+ opts = {d.defname.lower() for d in s.options or ()}
978
+ full = "full" in opts and s.is_vacuumcmd
979
+ st.kind = ("VACUUM FULL" if full else "VACUUM") if s.is_vacuumcmd else "ANALYZE"
980
+ if s.is_vacuumcmd:
981
+ st.notx = st.kind
982
+ rels = [rv(v.relation) for v in s.rels or ()] or ["every table in the database"]
983
+ for t in rels:
984
+ st.lock(t, AE if full else SUE, ctx.new(t))
985
+ st.slow("rewrite" if full else "scan")
986
+ if full and not all(ctx.new(t) for t in rels):
987
+ st.find("table-rewrite", "high", f"VACUUM FULL rewrites {', '.join(rels)} under ACCESS EXCLUSIVE: reads and writes wait.", REPACK)
988
+
989
+
990
+ def h_ClusterStmt(s, ctx, st):
991
+ st.kind = "CLUSTER"
992
+ t = rv(s.relation) if s.relation is not None else "every clustered table"
993
+ if s.relation is None:
994
+ st.notx = "CLUSTER without a table"
995
+ st.lock(t, AE, ctx.new(t))
996
+ st.slow("rewrite")
997
+ if not ctx.new(t):
998
+ st.find("table-rewrite", "high", f"CLUSTER rewrites {t} in index order under ACCESS EXCLUSIVE: reads and writes wait.", REPACK)
999
+
1000
+
1001
+ def h_ReindexStmt(s, ctx, st):
1002
+ conc = any(d.defname.lower() == "concurrently" for d in s.params or ())
1003
+ what = {RO.REINDEX_OBJECT_INDEX: "INDEX", RO.REINDEX_OBJECT_TABLE: "TABLE", RO.REINDEX_OBJECT_SCHEMA: "SCHEMA",
1004
+ RO.REINDEX_OBJECT_SYSTEM: "SYSTEM", RO.REINDEX_OBJECT_DATABASE: "DATABASE"}[s.kind]
1005
+ st.kind = f"REINDEX {what}" + (" CONCURRENTLY" if conc else "")
1006
+ name = rv(s.relation) if s.relation is not None else (s.name or "current database")
1007
+ target = f"table of index {name}" if what == "INDEX" else name
1008
+ st.slow("index")
1009
+ if what in ("SCHEMA", "SYSTEM", "DATABASE"):
1010
+ st.notx = st.kind
1011
+ if conc:
1012
+ need(st, ctx, 12, "REINDEX CONCURRENTLY")
1013
+ st.notx = st.kind
1014
+ st.lock(target, SUE)
1015
+ return
1016
+ st.lock(target, SH)
1017
+ st.lock(name if what == "INDEX" else f"indexes of {name}", AE)
1018
+ st.find("reindex", "high", f"REINDEX {what} {name} blocks writes to the table and takes ACCESS EXCLUSIVE on the index"
1019
+ f"{'es' if what != 'INDEX' else ''}, so queries that would use it wait too, for the whole rebuild.",
1020
+ f"REINDEX {what} CONCURRENTLY {name}; -- PostgreSQL 12+, outside a transaction" if ctx.pg >= 12 else
1021
+ "-- before PostgreSQL 12: CREATE INDEX CONCURRENTLY a copy, DROP INDEX CONCURRENTLY the old one, ALTER INDEX ... RENAME.")
1022
+
1023
+
1024
+ def h_LockStmt(s, ctx, st):
1025
+ mode = MODES[s.mode - 1]
1026
+ st.kind = f"LOCK TABLE IN {mode} MODE"
1027
+ st.slow("catalog")
1028
+ for r in s.relations:
1029
+ st.lock(rv(r), mode, ctx.new(rv(r)))
1030
+
1031
+
1032
+ def h_AlterEnumStmt(s, ctx, st):
1033
+ st.kind = "ALTER TYPE ADD VALUE" if s.new_val and not s.old_val else "ALTER TYPE RENAME VALUE"
1034
+ st.slow("catalog")
1035
+ if not s.old_val and s.new_val:
1036
+ st.enum_add = True
1037
+
1038
+
1039
+ def h_AlterDomainStmt(s, ctx, st):
1040
+ st.kind = "ALTER DOMAIN"
1041
+ st.slow("catalog")
1042
+ d = dotted(s.type_name)
1043
+ if s.subtype == "C" and isinstance(s.def_, ast.Constraint) and not s.def_.skip_validation:
1044
+ st.slow("scan")
1045
+ name = s.def_.conname or ident(d.split(".")[-1], suffix="check")
1046
+ con = sql(s.def_) if s.def_.conname else f"CONSTRAINT {name} {sql(s.def_)}"
1047
+ st.find("domain-constraint", "high", f"Adding a constraint to domain {d} scans every table column that uses it, "
1048
+ "locking those tables against writes.",
1049
+ f"ALTER DOMAIN {d} ADD {con} NOT VALID; -- instant; new values are checked\n"
1050
+ f"ALTER DOMAIN {d} VALIDATE CONSTRAINT {name};")
1051
+ elif s.subtype in ("O", "V"):
1052
+ st.slow("scan")
1053
+
1054
+
1055
+ def h_dml(s, ctx, st, kind):
1056
+ t = rv(s.relation)
1057
+ st.kind = kind
1058
+ st.lock(t, RE, ctx.new(t))
1059
+ st.slow("rows")
1060
+ for part in (getattr(s, "fromClause", None), getattr(s, "usingClause", None), getattr(s, "selectStmt", None),
1061
+ getattr(s, "whereClause", None), getattr(s, "sourceRelation", None)):
1062
+ if part is not None:
1063
+ reads(part, ctx, st)
1064
+ if kind in ("UPDATE", "DELETE") and s.where_clause is None and not ctx.new(t):
1065
+ verb = "updates" if kind == "UPDATE" else "deletes"
1066
+ st.find("update-all-rows", "medium", f"{kind} without WHERE {verb} every row of {t} in one transaction: it row-locks them all "
1067
+ "(blocking concurrent writes to them), generates WAL and dead tuples for the whole table.",
1068
+ f"-- batch it; each batch in its own short transaction, repeat until 0 rows:\n"
1069
+ + (f"UPDATE {t} SET ... WHERE ctid IN (SELECT ctid FROM {t} WHERE <not yet updated> LIMIT 5000);" if kind == "UPDATE"
1070
+ else f"DELETE FROM {t} WHERE ctid IN (SELECT ctid FROM {t} LIMIT 5000);"))
1071
+
1072
+
1073
+ def h_InsertStmt(s, ctx, st):
1074
+ h_dml(s, ctx, st, "INSERT")
1075
+
1076
+
1077
+ def h_UpdateStmt(s, ctx, st):
1078
+ h_dml(s, ctx, st, "UPDATE")
1079
+
1080
+
1081
+ def h_DeleteStmt(s, ctx, st):
1082
+ h_dml(s, ctx, st, "DELETE")
1083
+
1084
+
1085
+ def h_MergeStmt(s, ctx, st):
1086
+ need(st, ctx, 15, "MERGE")
1087
+ h_dml(s, ctx, st, "MERGE")
1088
+
1089
+
1090
+ def h_CopyStmt(s, ctx, st):
1091
+ st.kind = "COPY FROM" if s.is_from else "COPY TO"
1092
+ st.slow("rows")
1093
+ if s.relation is not None:
1094
+ t = rv(s.relation)
1095
+ st.lock(t, RE if s.is_from else AS, ctx.new(t))
1096
+
1097
+
1098
+ def h_SelectStmt(s, ctx, st):
1099
+ st.kind = "SELECT"
1100
+ st.slow("rows")
1101
+ mode = RS if s.locking_clause else AS
1102
+ reads(s, ctx, st, mode)
1103
+
1104
+
1105
+ def h_DoStmt(s, ctx, st):
1106
+ st.kind = "DO block"
1107
+ st.find("not-analysed", "medium", "The body of a DO block is not analysed: check its statements by hand.")
1108
+
1109
+
1110
+ NO_TABLE = { # statements that change catalogs only and lock no user table
1111
+ "CreateFunctionStmt": "CREATE FUNCTION", "CreateSeqStmt": "CREATE SEQUENCE", "AlterSeqStmt": "ALTER SEQUENCE",
1112
+ "CreateEnumStmt": "CREATE TYPE (enum)", "CompositeTypeStmt": "CREATE TYPE", "CreateDomainStmt": "CREATE DOMAIN",
1113
+ "CreateSchemaStmt": "CREATE SCHEMA", "CreateExtensionStmt": "CREATE EXTENSION", "AlterExtensionStmt": "ALTER EXTENSION",
1114
+ "GrantStmt": "GRANT/REVOKE", "GrantRoleStmt": "GRANT role", "CreateRoleStmt": "CREATE ROLE", "AlterRoleStmt": "ALTER ROLE",
1115
+ "AlterRoleSetStmt": "ALTER ROLE SET", "DropRoleStmt": "DROP ROLE", "AlterOwnerStmt": "ALTER OWNER", "DefineStmt": "CREATE (definition)",
1116
+ "AlterFunctionStmt": "ALTER FUNCTION", "CreateRangeStmt": "CREATE TYPE (range)", "AlterDefaultPrivilegesStmt": "ALTER DEFAULT PRIVILEGES",
1117
+ "CreatedbStmt": "CREATE DATABASE", "DropdbStmt": "DROP DATABASE", "AlterSystemStmt": "ALTER SYSTEM", "CreateCastStmt": "CREATE CAST",
1118
+ "CreateOpClassStmt": "CREATE OPERATOR CLASS", "CreatePublicationStmt": "CREATE PUBLICATION", "AlterPublicationStmt": "ALTER PUBLICATION",
1119
+ "CreateSubscriptionStmt": "CREATE SUBSCRIPTION", "CreateEventTrigStmt": "CREATE EVENT TRIGGER", "CreateConversionStmt": "CREATE CONVERSION",
1120
+ "CreateTableSpaceStmt": "CREATE TABLESPACE", "DropTableSpaceStmt": "DROP TABLESPACE", "CreateForeignServerStmt": "CREATE SERVER",
1121
+ "CreateFdwStmt": "CREATE FOREIGN DATA WRAPPER", "CreateUserMappingStmt": "CREATE USER MAPPING", "ImportForeignSchemaStmt": "IMPORT FOREIGN SCHEMA",
1122
+ "AlterCollationStmt": "ALTER COLLATION", "AlterTSConfigurationStmt": "ALTER TEXT SEARCH CONFIGURATION", "CreatePLangStmt": "CREATE LANGUAGE",
1123
+ "AlterEventTrigStmt": "ALTER EVENT TRIGGER", "NotifyStmt": "NOTIFY", "ListenStmt": "LISTEN", "DiscardStmt": "DISCARD",
1124
+ "CheckPointStmt": "CHECKPOINT", "ExplainStmt": "EXPLAIN", "DeallocateStmt": "DEALLOCATE", "PrepareStmt": "PREPARE",
1125
+ "CreateTransformStmt": "CREATE TRANSFORM", "CreateAmStmt": "CREATE ACCESS METHOD", "AlterTypeStmt": "ALTER TYPE", "AlterOpFamilyStmt": "ALTER OPERATOR FAMILY",
1126
+ "CreateOpFamilyStmt": "CREATE OPERATOR FAMILY", "ReassignOwnedStmt": "REASSIGN OWNED", "DropOwnedStmt": "DROP OWNED",
1127
+ "SecLabelStmt": "SECURITY LABEL", "AlterDatabaseSetStmt": "ALTER DATABASE SET", "AlterDatabaseStmt": "ALTER DATABASE",
1128
+ }
1129
+ NOTX_TYPES = {"CreatedbStmt": "CREATE DATABASE", "DropdbStmt": "DROP DATABASE", "AlterSystemStmt": "ALTER SYSTEM",
1130
+ "CreateTableSpaceStmt": "CREATE TABLESPACE", "DropTableSpaceStmt": "DROP TABLESPACE",
1131
+ "CreateSubscriptionStmt": "CREATE SUBSCRIPTION"}
1132
+ HANDLERS = {k[2:]: v for k, v in globals().items() if k.startswith("h_") and k != "h_dml"}
1133
+
1134
+
1135
+ def analyse(stmt, ctx):
1136
+ name = type(stmt).__name__
1137
+ st = St(NO_TABLE.get(name, name.removesuffix("Stmt")))
1138
+ if name in HANDLERS:
1139
+ HANDLERS[name](stmt, ctx, st)
1140
+ elif name in NO_TABLE:
1141
+ st.notx = NOTX_TYPES.get(name, "")
1142
+ if name not in ('CreateFunctionStmt', 'CreateEnumStmt', 'CreateSeqStmt', 'CreateRoleStmt',
1143
+ 'GrantRoleStmt', 'AlterRoleStmt', 'AlterRoleSetStmt', 'NotifyStmt', 'ListenStmt',
1144
+ 'CreateDomainStmt', 'AlterDefaultPrivilegesStmt', 'CreateEventTrigStmt'):
1145
+ st.find("not-analysed", "medium", "This operation can reach schema objects or execute nested SQL; its table locks are not modelled.")
1146
+ if name == "CreateSubscriptionStmt":
1147
+ st.notx = "" if any(d.defname == "create_slot" and sql(d.arg) in ("false", "'false'") for d in stmt.options or ()) else st.notx
1148
+ else:
1149
+ st.find("not-analysed", "medium", f"{st.kind} statements are not modelled; check its locks in the PostgreSQL docs.")
1150
+ return st
1151
+
1152
+
1153
+ # ---------- the run over files ----------
1154
+ def timeout_value(args):
1155
+ if not args:
1156
+ return None
1157
+ v = args[0]
1158
+ v = getattr(v, "val", v)
1159
+ raw = str(getattr(v, "sval", None) or getattr(v, "ival", "")).strip().lower()
1160
+ return None if re.fullmatch(r"0+(\.0*)?\s*(us|ms|s|min|h|d)?", raw or "0") else raw
1161
+
1162
+
1163
+ def set_config(stmt):
1164
+ """SELECT set_config('lock_timeout', '5s', is_local) -> (name, value, local)."""
1165
+ if type(stmt).__name__ != "SelectStmt" or not stmt.target_list or len(stmt.target_list) != 1:
1166
+ return None
1167
+ f = stmt.target_list[0].val
1168
+ if not isinstance(f, ast.FuncCall) or f.funcname[-1].sval != "set_config" or len(f.args or ()) != 3:
1169
+ return None
1170
+ try:
1171
+ name, val, loc = (a.val for a in f.args)
1172
+ return name.sval.lower(), timeout_value([val]), bool(getattr(loc, "boolval", False))
1173
+ except AttributeError:
1174
+ return None
1175
+
1176
+
1177
+ class Lines:
1178
+ def __init__(self, text):
1179
+ self.starts = [0] + [m.end() for m in re.finditer("\n", text)]
1180
+
1181
+ def at(self, off):
1182
+ i = bisect_right(self.starts, off) - 1
1183
+ return i + 1, off - self.starts[i] + 1
1184
+
1185
+
1186
+ def check(files, pg=PG_DEFAULT, txn_mode="auto", assume_timeouts=False, max_statements=None):
1187
+ """files: [(name, text)] in the order they run. Returns the full report dict."""
1188
+ if not PG_MIN <= pg <= PG_MAX:
1189
+ raise SQLError(f"PostgreSQL version must be {PG_MIN}-{PG_MAX}")
1190
+ if txn_mode not in TXN_MODES:
1191
+ raise SQLError(f"transaction mode must be one of {', '.join(TXN_MODES)}")
1192
+ if not files or not any(t.strip() for _, t in files):
1193
+ raise SQLError('paste SQL or upload a non-empty .sql file')
1194
+ if len(files) > MAX_FILES or sum(len(t.encode('utf-8')) for _, t in files) > MAX_BYTES:
1195
+ raise SQLError('limit: 100 files and 4 MiB aggregate UTF-8 SQL')
1196
+ ctx, out, total = Ctx(pg), [], 0
1197
+ for name, text in files:
1198
+ text = newlines(text)
1199
+ prepared, units, framework, notes = prepare(text, name, txn_mode)
1200
+ lines = Lines(prepared)
1201
+ parsed = parse(prepared)
1202
+ total += len(parsed)
1203
+ if total > (max_statements or MAX_STATEMENTS):
1204
+ raise SQLError(f"limit: {max_statements or MAX_STATEMENTS} statements per check")
1205
+ out.append(run_file(name, prepared, units, framework, notes, lines, parsed, ctx, assume_timeouts))
1206
+ return report(out, pg)
1207
+
1208
+
1209
+ def run_file(name, text, units, framework, notes, lines, parsed, ctx, assume_timeouts):
1210
+ rows = []
1211
+ lt, st_to = {"session": None, "local": None}, {"session": None, "local": None}
1212
+ txn = None # {"how", "line", "held": {target: (mode, line)}, "implicit"}
1213
+ cur_unit, warned_lt, warned_st = None, False, False
1214
+
1215
+ def end_txn():
1216
+ nonlocal txn
1217
+ txn = None
1218
+ lt["local"] = st_to["local"] = None
1219
+
1220
+ for start, end, stmt, err in parsed:
1221
+ line, col = lines.at(start)
1222
+ # implicit transactions of the migration runner (goose, dbmate, Flyway, Liquibase changesets, --transaction always)
1223
+ unit = next((u for u in units if u[0] <= start < u[1]), None)
1224
+ if unit is not cur_unit:
1225
+ if cur_unit and txn and txn["implicit"]:
1226
+ end_txn()
1227
+ cur_unit = unit
1228
+ if unit and not txn:
1229
+ txn = {"how": f"implicit ({unit[2]})", "line": line, "held": {}, "implicit": True}
1230
+ text_of = text[start:end].strip()
1231
+ row = {"n": len(rows) + 1, "line": line, "col": col, "end_line": lines.at(max(start, end - 1))[0],
1232
+ "sql": text_of[:PREVIEW] + ("..." if len(text_of) > PREVIEW else "")}
1233
+ if err:
1234
+ el, ec = lines.at(err[1])
1235
+ row.update(kind="(parse error)", locks=[], work="none", blocks="nothing", transaction=txn["how"] if txn else "",
1236
+ findings=[{"rule": "parse-error", "severity": "high", "message": f"line {el}, column {ec}: {err[0]}",
1237
+ "fix": "", "line": el, "col": ec}])
1238
+ rows.append(row)
1239
+ continue
1240
+ kind = type(stmt).__name__
1241
+ if kind == "TransactionStmt":
1242
+ k = stmt.kind
1243
+ row.update(kind=TK.Name(k)[11:].replace("_", " "), locks=[], work="none", blocks="nothing", findings=[])
1244
+ if k in (TK.TRANS_STMT_BEGIN, TK.TRANS_STMT_START):
1245
+ if not txn:
1246
+ txn = {"how": f"BEGIN at line {line}", "line": line, "held": {}, "implicit": False}
1247
+ if k in (TK.TRANS_STMT_ROLLBACK, TK.TRANS_STMT_ROLLBACK_TO):
1248
+ ctx.tables.clear()
1249
+ ctx.nn_checks.clear()
1250
+ ctx.nn_ok.clear()
1251
+ ctx.cons.clear()
1252
+ if k in (TK.TRANS_STMT_COMMIT, TK.TRANS_STMT_ROLLBACK, TK.TRANS_STMT_COMMIT_PREPARED, TK.TRANS_STMT_PREPARE):
1253
+ if txn:
1254
+ row["note"] = "releases " + (", ".join(f"{m} on {t}" for t, (m, _) in txn["held"].items()) or "no table locks")
1255
+ end_txn()
1256
+ row["transaction"] = txn["how"] if txn else ""
1257
+ rows.append(row)
1258
+ continue
1259
+ if kind == "VariableSetStmt" or set_config(stmt):
1260
+ sc = set_config(stmt)
1261
+ if sc:
1262
+ nm, val, local = sc
1263
+ else:
1264
+ nm = (stmt.name or "").lower()
1265
+ local = stmt.is_local
1266
+ val = timeout_value(stmt.args) if stmt.kind == VK.VAR_SET_VALUE else None
1267
+ if stmt.kind == VK.VAR_RESET_ALL:
1268
+ lt["session"] = lt["local"] = st_to["session"] = st_to["local"] = None
1269
+ for var, store in (("lock_timeout", lt), ("statement_timeout", st_to)):
1270
+ if nm == var:
1271
+ if local:
1272
+ if txn:
1273
+ store["local"] = val
1274
+ else:
1275
+ store["session"] = val
1276
+ if txn:
1277
+ store["local"] = None
1278
+ row.update(kind="SET " + nm if nm else "SET", locks=[], work="none", blocks="nothing", findings=[],
1279
+ transaction=txn["how"] if txn else "")
1280
+ rows.append(row)
1281
+ continue
1282
+ st = analyse(stmt, ctx)
1283
+ if kind == "LockStmt" and not txn:
1284
+ st.find("lock-outside-transaction", "high", "LOCK TABLE can only be used in transaction blocks; outside one it fails.",
1285
+ "BEGIN;\n" + text_of + ";\n-- ... the statements that need the lock ...\nCOMMIT;")
1286
+ if getattr(st, "enum_add", False) and txn:
1287
+ if ctx.pg < 12:
1288
+ st.find("enum-add-value", "high", "ALTER TYPE ... ADD VALUE cannot run inside a transaction block before PostgreSQL 12: it fails.", NOTX_FIX)
1289
+ else:
1290
+ st.find("enum-add-value", "low", "The new enum value cannot be used until this transaction commits; "
1291
+ "statements later in the same transaction that use it fail.", "-- commit before using the new value (separate migration).")
1292
+ if st.notx and txn:
1293
+ st.find("no-transaction", "high", f"{st.notx} cannot run inside a transaction block ({txn['how']}): it fails with "
1294
+ "\"cannot run inside a transaction block\".", NOTX_FIX)
1295
+ effective_lt = assume_timeouts or (lt["local"] if txn and lt["local"] is not None else lt["session"])
1296
+ effective_st = assume_timeouts or (st_to["local"] if txn and st_to["local"] is not None else st_to["session"])
1297
+ existing = [l for l in st.locks if not l["new"]]
1298
+ top = strongest([l["mode"] for l in existing])
1299
+ if top and blocks_writes(top) and st.work in SLOW and not any(f["severity"] == "high" for f in st.findings):
1300
+ st.find("long-lock", "high", f"{st.kind} holds {top} on {', '.join(l['target'] for l in existing if l['mode'] == top)} "
1301
+ f"while it {WORK[st.work].split(' while')[0].replace('time grows', 'runs for a time that grows')}: {blocks(top)} wait.")
1302
+ if top and blocks_writes(top) and not effective_lt:
1303
+ st.find("missing-lock-timeout", "medium",
1304
+ f"No lock_timeout is set: if {top} on {existing[0]['target'] if len(existing) == 1 else 'these tables'} has to wait "
1305
+ f"for a long-running query, every later query on the table queues behind it ({blocks(top)} stall).",
1306
+ "" if warned_lt else "SET lock_timeout = '5s'; -- at the top of the migration (SET LOCAL inside a transaction); retry on timeout")
1307
+ warned_lt = True
1308
+ if existing and st.work in ("scan", "rewrite", "index") and not effective_st and not warned_st:
1309
+ st.find("missing-statement-timeout", "low", "No statement_timeout is set: nothing bounds how long this statement runs.",
1310
+ "SET statement_timeout = '30min'; -- an upper bound you are willing to hold these locks for")
1311
+ warned_st = True
1312
+ if txn:
1313
+ held = {t: (m, ln) for t, (m, ln) in txn["held"].items() if blocks_writes(m) and not ctx.new(t)}
1314
+ others = {t: v for t, v in held.items() if not any(l["target"] == t for l in st.locks)}
1315
+ if others and (st.work in SLOW or (top and blocks_writes(top))):
1316
+ desc = ", ".join(f"{m} on {t} (line {ln})" for t, (m, ln) in others.items())
1317
+ sev = "high" if st.work in SLOW and any(m == AE for m, _ in others.values()) else "medium"
1318
+ st.find("locks-accumulate", sev, f"Locks are held until COMMIT ({txn['how']}), so {desc} still blocks traffic "
1319
+ f"while this statement runs" + (" for a long time." if st.work in SLOW else " and waits for its own lock."),
1320
+ "-- split the migration so each risky step commits on its own (separate migration files or transactions);\n"
1321
+ "-- then a slow step only holds its own lock, and a lock wait cannot stall the tables changed earlier.")
1322
+ for l in st.locks:
1323
+ prev = txn["held"].get(l["target"])
1324
+ if not prev or rank(l["mode"]) > rank(prev[0]):
1325
+ txn["held"][l["target"]] = (l["mode"], line)
1326
+ unknown = any(f['rule'] == 'not-analysed' for f in st.findings)
1327
+ row.update(kind=st.kind, locks=st.locks, work=st.work, blocks='unknown (not modelled)' if unknown else blocks(top), transaction=txn["how"] if txn else "",
1328
+ findings=sorted(st.findings, key=lambda f: -SEVERITIES.index(f["severity"])))
1329
+ rows.append(row)
1330
+ for r in rows:
1331
+ r["risk"] = SEVERITIES[max((SEVERITIES.index(f["severity"]) for f in r["findings"]), default=0)]
1332
+ r["work_text"] = WORK.get(r["work"], "")
1333
+ return {"name": name, "framework": framework, "statements": rows,
1334
+ "notes": [{"line": lines.at(o)[0], "text": t} for o, t in notes]}
1335
+
1336
+
1337
+ def report(files, pg):
1338
+ counts = {s: 0 for s in SEVERITIES}
1339
+ n = 0
1340
+ for f in files:
1341
+ for r in f["statements"]:
1342
+ n += 1
1343
+ counts[r["risk"]] += 1
1344
+ risk = next((s for s in reversed(SEVERITIES) if counts[s]), "info")
1345
+ return {"pg": pg, "risk": risk, "statements": n, "counts": counts, "files": files,
1346
+ "scope": "Static estimates only. No database is contacted. Existing schema, data, partitions, triggers, function bodies, dependencies and actual lock duration are unknown. Review rewrite examples before running; they are not automatic fixes."}
1347
+
1348
+
1349
+ def newlines(text):
1350
+ return text.lstrip("\ufeff").replace("\r\n", "\n").replace("\r", "\n")
1351
+
1352
+
1353
+ def failing(rep, threshold):
1354
+ """True if any statement's risk is at or above threshold (high|medium|low|never)."""
1355
+ if threshold == "never":
1356
+ return False
1357
+ return any(rep["counts"][s] for s in SEVERITIES[SEVERITIES.index(threshold):])
1358
+
1359
+
1360
+ # ---------- output formats ----------
1361
+ LEVEL = {"high": "error", "medium": "warning", "low": "note", "info": "note"}
1362
+
1363
+
1364
+ def sarif(rep, version):
1365
+ used = sorted({f["rule"] for fl in rep["files"] for r in fl["statements"] for f in r["findings"]})
1366
+ results = []
1367
+ for fl in rep["files"]:
1368
+ for r in fl["statements"]:
1369
+ for f in r["findings"]:
1370
+ msg = f["message"] + (f"\n\nSafer:\n{f['fix']}" if f["fix"] else "")
1371
+ results.append({"ruleId": f["rule"], "level": LEVEL[f["severity"]], "message": {"text": msg},
1372
+ "properties": {"severity": f["severity"], "statement": r["kind"],
1373
+ "locks": [f"{l['mode']} on {l['target']}" for l in r["locks"]]},
1374
+ "locations": [{"physicalLocation": {
1375
+ "artifactLocation": {"uri": fl["name"].replace("\\", "/")},
1376
+ "region": {"startLine": f.get("line", r["line"]), "startColumn": f.get("col", r["col"]),
1377
+ **({} if "line" in f else {"endLine": r["end_line"]})}}}]})
1378
+ return {"$schema": "https://json.schemastore.org/sarif-2.1.0.json", "version": "2.1.0",
1379
+ "runs": [{"tool": {"driver": {"name": "localtools-postgres-migration-checker", "version": version,
1380
+ "informationUri": "https://github.com/chayprabs/localtools/tree/main/postgres-migration-checker",
1381
+ "rules": [{"id": r, "name": r, "shortDescription": {"text": RULES[r]}} for r in used]}},
1382
+ "properties": {"postgresVersion": rep["pg"]}, "results": results}]}
1383
+
1384
+
1385
+ def _md(s):
1386
+ return s.replace("|", "\\|").replace("\n", " ").replace('`', '&#96;')
1387
+
1388
+
1389
+ def markdown(rep):
1390
+ c = rep["counts"]
1391
+ out = [f"# Postgres migration check (PostgreSQL {rep['pg']})", "",
1392
+ f"**Risk: {rep['risk'].upper()}** · {rep['statements']} statements · {c['high']} high, {c['medium']} medium, "
1393
+ f"{c['low']} low, {c['info']} info", ""]
1394
+ for fl in rep["files"]:
1395
+ out += [f"## {fl['name']}", "", f"Runner: {fl['framework']}", ""]
1396
+ out += [f"- line {n['line']}: {n['text']}" for n in fl["notes"]] + ([""] if fl["notes"] else [])
1397
+ out += ["| # | Line | Statement | Locks | Blocks | Work | Risk |", "|---|---|---|---|---|---|---|"]
1398
+ for r in fl["statements"]:
1399
+ locks = "; ".join(f"{l['mode']} on {l['target']}" + (" (new)" if l["new"] else "") for l in r["locks"]) or "none"
1400
+ out.append(f"| {r['n']} | {r['line']} | `{_md(r['sql'][:80])}` | {_md(locks)} | {r['blocks']} | {r['work']} | {r['risk']} |")
1401
+ out.append("")
1402
+ for r in fl["statements"]:
1403
+ for f in r["findings"]:
1404
+ out += [f"### Line {f.get('line', r['line'])} · {f['severity']} · {f['rule']}", "", f["message"], ""]
1405
+ if f["fix"]:
1406
+ fence = '`' * max(3, 1 + max((len(m.group()) for m in re.finditer(r'`+', f['fix'])), default=0))
1407
+ out += [fence + "sql", f["fix"], fence, ""]
1408
+ return "\n".join(out) + "\n"
1409
+
1410
+
1411
+ def human(rep):
1412
+ out = []
1413
+ for fl in rep["files"]:
1414
+ for n in fl["notes"]:
1415
+ out.append(f"{fl['name']}:{n['line']}: note: {n['text']}")
1416
+ for r in fl["statements"]:
1417
+ locks = ", ".join(f"{l['mode']} on {l['target']}" for l in r["locks"] if not l["new"]) or "no lock on existing tables"
1418
+ out.append(f"{fl['name']}:{r['line']}:{r['col']}: {r['risk'].upper():6} {r['kind']}: {locks}"
1419
+ + (f"; blocks {r['blocks']}" if r["blocks"] != "nothing" and any(not l['new'] for l in r['locks']) else "")
1420
+ + (f"; {r['work']}" if r["work"] in SLOW else ""))
1421
+ for f in r["findings"]:
1422
+ out.append(f" [{f['rule']}] {f['message']}")
1423
+ for ln in f["fix"].splitlines():
1424
+ out.append(f" {ln}")
1425
+ return "\n".join(out) + ("\n" if out else "")
1426
+
1427
+
1428
+ def dumps(rep):
1429
+ return json.dumps(rep, indent=2, ensure_ascii=False)