pythia-postgres 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pythia_postgres/__init__.py +5 -0
- pythia_postgres/__main__.py +5 -0
- pythia_postgres/catalog.py +59 -0
- pythia_postgres/cli.py +8 -0
- pythia_postgres/engine.py +213 -0
- pythia_postgres/guard.py +54 -0
- pythia_postgres/queries/code-objects.sql +28 -0
- pythia_postgres/queries/columns.sql +15 -0
- pythia_postgres/queries/comment-current.sql +17 -0
- pythia_postgres/queries/current-schema.sql +5 -0
- pythia_postgres/queries/dependencies-catalog.sql +66 -0
- pythia_postgres/queries/dependents-catalog.sql +66 -0
- pythia_postgres/queries/dependents-text.sql +22 -0
- pythia_postgres/queries/function-body.sql +8 -0
- pythia_postgres/queries/grep.sql +30 -0
- pythia_postgres/queries/name-candidates.sql +32 -0
- pythia_postgres/queries/object-names.sql +56 -0
- pythia_postgres/queries/plpgsql-check-installed.sql +7 -0
- pythia_postgres/queries/plpgsql-check.sql +27 -0
- pythia_postgres/queries/plpgsql-unchecked.sql +16 -0
- pythia_postgres/queries/rehearsal-limits.sql +7 -0
- pythia_postgres/queries/resolve.sql +34 -0
- pythia_postgres/queries/row-set-count.sql +11 -0
- pythia_postgres/queries/row-set-sample.sql +8 -0
- pythia_postgres/queries/search-path-catalog.sql +7 -0
- pythia_postgres/queries/search-path-first.sql +13 -0
- pythia_postgres/queries/sequence-ddl.sql +16 -0
- pythia_postgres/queries/server-version.sql +6 -0
- pythia_postgres/queries/session-power.sql +13 -0
- pythia_postgres/queries/source.sql +28 -0
- pythia_postgres/queries/table-columns.sql +16 -0
- pythia_postgres/queries/table-constraints.sql +13 -0
- pythia_postgres/queries/table-extras.sql +43 -0
- pythia_postgres/queries/table-head.sql +18 -0
- pythia_postgres/read.py +299 -0
- pythia_postgres/rules.py +135 -0
- pythia_postgres/skills/pythia-pg-apply/SKILL.md +97 -0
- pythia_postgres/skills/pythia-pg-apply/reference/refusals.md +26 -0
- pythia_postgres/skills/pythia-pg-autopilot/SKILL.md +63 -0
- pythia_postgres/skills/pythia-pg-deploy/SKILL.md +73 -0
- pythia_postgres/skills/pythia-pg-explore/SKILL.md +63 -0
- pythia_postgres/skills/pythia-pg-impact/SKILL.md +57 -0
- pythia_postgres/skills/pythia-pg-verify/SKILL.md +56 -0
- pythia_postgres/skills/using-pythia-pg/SKILL.md +65 -0
- pythia_postgres/write.py +583 -0
- pythia_postgres-0.1.0.dist-info/METADATA +360 -0
- pythia_postgres-0.1.0.dist-info/RECORD +51 -0
- pythia_postgres-0.1.0.dist-info/WHEEL +5 -0
- pythia_postgres-0.1.0.dist-info/entry_points.txt +2 -0
- pythia_postgres-0.1.0.dist-info/licenses/LICENSE +21 -0
- pythia_postgres-0.1.0.dist-info/top_level.txt +1 -0
pythia_postgres/write.py
ADDED
|
@@ -0,0 +1,583 @@
|
|
|
1
|
+
"""The write tier for PostgreSQL: pythia's Do movement.
|
|
2
|
+
|
|
3
|
+
PostgreSQL runs DDL inside a transaction, so every write is rehearsed: run
|
|
4
|
+
in the transaction pythia-core opened, verified, and rolled back. A preview
|
|
5
|
+
shows what happened, not what would. The confirm run is the same rehearsal,
|
|
6
|
+
committed only when it still matches what was approved (gate.py in core).
|
|
7
|
+
|
|
8
|
+
What this module adds is PostgreSQL's own reading of a statement — a lexer
|
|
9
|
+
that knows its quoting ('…', E'…', "…", $tag$…$tag$, nested /* */), the
|
|
10
|
+
group each statement belongs to, the rows an UPDATE or DELETE touches — and
|
|
11
|
+
its own answers: what an object looked like before, which overload a CREATE
|
|
12
|
+
OR REPLACE really replaced, and what plpgsql_check says broke.
|
|
13
|
+
"""
|
|
14
|
+
import re
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
from pythia_core.output import cell
|
|
18
|
+
from pythia_core.queries import load_query
|
|
19
|
+
|
|
20
|
+
from . import read
|
|
21
|
+
from .catalog import QUERY_DIR, fold, q
|
|
22
|
+
|
|
23
|
+
# A rehearsal must not hold locks or rewrite a big table for long; the
|
|
24
|
+
# confirm run is the same rehearsal, so the write gets the same bounds.
|
|
25
|
+
STATEMENT_TIMEOUT = "30s"
|
|
26
|
+
LOCK_TIMEOUT = "5s"
|
|
27
|
+
SAMPLE_ROWS = 10
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
# --- reading a statement ---------------------------------------------------------
|
|
31
|
+
|
|
32
|
+
class Lexed:
|
|
33
|
+
def __init__(self, masked, cuts, meta, unterminated, uncommented):
|
|
34
|
+
self.masked = masked # same length; quotes and comments blanked
|
|
35
|
+
self.uncommented = uncommented # same length; only comments blanked
|
|
36
|
+
self.cuts = cuts # positions of statement-ending ;
|
|
37
|
+
self.meta = meta # a psql meta-command line (\i, \set ...)
|
|
38
|
+
self.unterminated = unterminated # ends inside a quote, comment or $$ body
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
_DOLLAR = re.compile(r"\$([A-Za-z_][A-Za-z0-9_]*)?\$")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _word_char(ch):
|
|
45
|
+
return ch.isalnum() or ch in "_$"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def lex(text):
|
|
49
|
+
"""Blank out string literals, quoted identifiers, dollar-quoted bodies and
|
|
50
|
+
comments (keeping every position and newline), and find the semicolons
|
|
51
|
+
that end statements. What is left is the code a keyword can be read in."""
|
|
52
|
+
out, bare = list(text), list(text)
|
|
53
|
+
cuts, meta, unterminated = [], False, False
|
|
54
|
+
n, i, line_start = len(text), 0, True
|
|
55
|
+
|
|
56
|
+
def blank(a, b, comment=False):
|
|
57
|
+
for k in range(a, min(b, n)):
|
|
58
|
+
if out[k] != "\n":
|
|
59
|
+
out[k] = " "
|
|
60
|
+
if comment:
|
|
61
|
+
bare[k] = " "
|
|
62
|
+
|
|
63
|
+
while i < n:
|
|
64
|
+
c = text[i]
|
|
65
|
+
if c == "\n":
|
|
66
|
+
line_start, i = True, i + 1
|
|
67
|
+
continue
|
|
68
|
+
if c in " \t\r\f":
|
|
69
|
+
i += 1
|
|
70
|
+
continue
|
|
71
|
+
if line_start and c == "\\":
|
|
72
|
+
meta = True
|
|
73
|
+
line_start = False
|
|
74
|
+
if text.startswith("--", i):
|
|
75
|
+
j = text.find("\n", i)
|
|
76
|
+
j = n if j < 0 else j
|
|
77
|
+
blank(i, j, comment=True)
|
|
78
|
+
i = j
|
|
79
|
+
elif text.startswith("/*", i):
|
|
80
|
+
depth, j = 1, i + 2 # PostgreSQL nests them
|
|
81
|
+
while j < n and depth:
|
|
82
|
+
if text.startswith("/*", j):
|
|
83
|
+
depth, j = depth + 1, j + 2
|
|
84
|
+
elif text.startswith("*/", j):
|
|
85
|
+
depth, j = depth - 1, j + 2
|
|
86
|
+
else:
|
|
87
|
+
j += 1
|
|
88
|
+
unterminated |= depth > 0
|
|
89
|
+
blank(i, j, comment=True)
|
|
90
|
+
i = j
|
|
91
|
+
elif c == "'" or (c in "eE" and text.startswith("'", i + 1)
|
|
92
|
+
and not (i and _word_char(text[i - 1]))):
|
|
93
|
+
backslashes = c != "'" # E'...' escapes with \
|
|
94
|
+
j, closed = i + (1 if c == "'" else 2), False
|
|
95
|
+
while j < n:
|
|
96
|
+
if backslashes and text[j] == "\\":
|
|
97
|
+
j += 2
|
|
98
|
+
continue
|
|
99
|
+
if text[j] == "'":
|
|
100
|
+
if text.startswith("''", j):
|
|
101
|
+
j += 2
|
|
102
|
+
continue
|
|
103
|
+
closed, j = True, j + 1
|
|
104
|
+
break
|
|
105
|
+
j += 1
|
|
106
|
+
unterminated |= not closed
|
|
107
|
+
blank(i, j)
|
|
108
|
+
i = j
|
|
109
|
+
elif c == '"':
|
|
110
|
+
j, closed = i + 1, False
|
|
111
|
+
while j < n:
|
|
112
|
+
if text[j] == '"':
|
|
113
|
+
if text.startswith('""', j):
|
|
114
|
+
j += 2
|
|
115
|
+
continue
|
|
116
|
+
closed, j = True, j + 1
|
|
117
|
+
break
|
|
118
|
+
j += 1
|
|
119
|
+
unterminated |= not closed
|
|
120
|
+
blank(i, j)
|
|
121
|
+
i = j
|
|
122
|
+
elif c == "$" and not (i and _word_char(text[i - 1])) \
|
|
123
|
+
and _DOLLAR.match(text, i):
|
|
124
|
+
tag = _DOLLAR.match(text, i).group(0) # $1 is a parameter, not this
|
|
125
|
+
end = text.find(tag, i + len(tag))
|
|
126
|
+
if end < 0:
|
|
127
|
+
unterminated = True
|
|
128
|
+
blank(i, n)
|
|
129
|
+
i = n
|
|
130
|
+
else:
|
|
131
|
+
blank(i, end + len(tag))
|
|
132
|
+
i = end + len(tag)
|
|
133
|
+
else:
|
|
134
|
+
if c == ";":
|
|
135
|
+
cuts.append(i)
|
|
136
|
+
i += 1
|
|
137
|
+
return Lexed("".join(out), cuts, meta, unterminated, "".join(bare))
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def split_statements(text):
|
|
141
|
+
"""The statements of a file, each without its ending semicolon; empty
|
|
142
|
+
ones (whitespace, comments) are dropped."""
|
|
143
|
+
lexed = lex(text)
|
|
144
|
+
parts, start = [], 0
|
|
145
|
+
for cut in lexed.cuts + [len(text)]:
|
|
146
|
+
if lexed.masked[start:cut].strip():
|
|
147
|
+
parts.append(text[start:cut].strip())
|
|
148
|
+
start = cut + 1
|
|
149
|
+
return parts
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def split_script(text):
|
|
153
|
+
"""(line, statement) for each statement of a script, in order, cut where
|
|
154
|
+
PostgreSQL would cut it — never inside a quote, a comment or a $$ body.
|
|
155
|
+
The line is where the statement's code starts, as an editor counts."""
|
|
156
|
+
lexed = lex(text)
|
|
157
|
+
out, start = [], 0
|
|
158
|
+
for cut in lexed.cuts + [len(text)]:
|
|
159
|
+
code = lexed.masked[start:cut]
|
|
160
|
+
if code.strip():
|
|
161
|
+
first = start + len(code) - len(code.lstrip())
|
|
162
|
+
out.append((text.count("\n", 0, first) + 1, text[start:cut].strip()))
|
|
163
|
+
start = cut + 1
|
|
164
|
+
return out
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _code(stmt):
|
|
168
|
+
"""The statement's code: literals and comments blanked, lower case, one
|
|
169
|
+
space between words."""
|
|
170
|
+
return " ".join(lex(stmt).masked.lower().split())
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
TXN = ("pythia-pg owns the transaction: it rolls a preview back and commits only "
|
|
174
|
+
"an approved write. Leave transaction control out of the file.")
|
|
175
|
+
NONTX = ("This statement cannot run inside a transaction, so pythia-pg can "
|
|
176
|
+
"neither rehearse it nor roll it back. Run it by hand, with a DBA "
|
|
177
|
+
"watching.")
|
|
178
|
+
CANNOT = ("Cannot classify this statement, so it is refused rather than guessed "
|
|
179
|
+
"at. apply takes one CREATE [OR REPLACE] FUNCTION / PROCEDURE / VIEW, "
|
|
180
|
+
"COMMENT ON, INSERT / UPDATE / DELETE, DDL, GRANT or SET statement per "
|
|
181
|
+
"file.")
|
|
182
|
+
|
|
183
|
+
RULES = (
|
|
184
|
+
(r"(begin|start transaction|commit|end|rollback|abort|savepoint|release|"
|
|
185
|
+
r"prepare transaction|set transaction|set session characteristics|lock)\b",
|
|
186
|
+
None, TXN),
|
|
187
|
+
(r"do\b", None,
|
|
188
|
+
"Anonymous blocks (DO) are refused: a DO block can run anything, so no "
|
|
189
|
+
"policy group honestly fits. Put the logic in a function and apply that."),
|
|
190
|
+
(r"call\b", None,
|
|
191
|
+
"CALL runs a procedure's body, which can do anything — no policy group "
|
|
192
|
+
"honestly fits. Run it by hand, or apply the statements it would run."),
|
|
193
|
+
(r"with\b", None,
|
|
194
|
+
"A WITH statement is refused: a data-modifying WITH hides its writes "
|
|
195
|
+
"inside a query. Write it as a plain INSERT, UPDATE or DELETE."),
|
|
196
|
+
(r"(select|values|table|show|explain|fetch|declare|move|listen|unlisten|"
|
|
197
|
+
r"notify|load|discard|checkpoint|prepare|execute|deallocate)\b", None,
|
|
198
|
+
"apply takes writes; reads go through ls, src, cols and grep — and a "
|
|
199
|
+
"SELECT can call a function that writes, so it is refused, not run."),
|
|
200
|
+
(r"copy\b", None,
|
|
201
|
+
"COPY reads or writes files and streams outside the database; refused."),
|
|
202
|
+
(r"(create (unique )?index concurrently|drop index concurrently|"
|
|
203
|
+
r"reindex\b.*\bconcurrently|vacuum|create database|drop database|"
|
|
204
|
+
r"create tablespace|drop tablespace|alter system|"
|
|
205
|
+
r"(create|alter|drop) subscription)\b", None, NONTX),
|
|
206
|
+
(r"create (or replace )?(temp|temporary)\b", None,
|
|
207
|
+
"A temporary object dies with this connection, which ends after the "
|
|
208
|
+
"command: nothing would be left to verify or undo."),
|
|
209
|
+
(r"create (or replace )?(function|procedure)\b", "source", None),
|
|
210
|
+
(r"create (or replace )?(recursive )?view\b", "source", None),
|
|
211
|
+
(r"comment on (table|column)\b", "comments", None),
|
|
212
|
+
(r"comment on\b", None,
|
|
213
|
+
"Only TABLE and COLUMN comments are snapshotted and restorable; other "
|
|
214
|
+
"comments are refused."),
|
|
215
|
+
(r"(insert|update|delete|merge)\b", "data_dml", None),
|
|
216
|
+
(r"(grant|revoke)\b|alter default privileges\b", "grants", None),
|
|
217
|
+
(r"(set|reset) (session |local )?(role|session authorization)\b", None,
|
|
218
|
+
"SET ROLE and SET SESSION AUTHORIZATION change who the session is — and "
|
|
219
|
+
"so who verifies the write. Refused."),
|
|
220
|
+
(r"(set|reset)\b", "session", None),
|
|
221
|
+
(r"(create|alter|drop|truncate|refresh materialized view|reindex|cluster|"
|
|
222
|
+
r"security label|import foreign schema)\b", "structural", None),
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def classify(text):
|
|
227
|
+
"""(group, None) or (None, why it is refused)."""
|
|
228
|
+
lexed = lex(text)
|
|
229
|
+
if lexed.unterminated:
|
|
230
|
+
return None, ("The file ends inside a quote, a comment or a $$ body, so "
|
|
231
|
+
"it cannot be read safely — refused.")
|
|
232
|
+
if lexed.meta:
|
|
233
|
+
return None, ("psql meta-commands (\\i, \\set, \\gexec ...) are refused: "
|
|
234
|
+
"pythia-pg runs SQL, not psql scripts.")
|
|
235
|
+
stmts = split_statements(text)
|
|
236
|
+
if not stmts:
|
|
237
|
+
return None, "The file holds no statement."
|
|
238
|
+
if len(stmts) > 1:
|
|
239
|
+
return None, (f"{len(stmts)} statements in one file — apply takes one "
|
|
240
|
+
"statement per file, so each is previewed, approved and "
|
|
241
|
+
"journaled on its own.")
|
|
242
|
+
code = _code(stmts[0])
|
|
243
|
+
for pattern, group, why in RULES:
|
|
244
|
+
if re.match(pattern, code):
|
|
245
|
+
return group, why
|
|
246
|
+
return None, CANNOT
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def prepare_statement(text, group):
|
|
250
|
+
return split_statements(text)[0]
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
_IDENT = re.compile(r'"((?:[^"]|"")+)"|([A-Za-z_][A-Za-z0-9_$]*)')
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _qualified_name(text, pos):
|
|
257
|
+
"""The parts of a dotted name starting at pos, folded as PostgreSQL folds
|
|
258
|
+
them: unquoted to lower case, quoted exactly."""
|
|
259
|
+
parts = []
|
|
260
|
+
while True:
|
|
261
|
+
while pos < len(text) and text[pos].isspace():
|
|
262
|
+
pos += 1
|
|
263
|
+
m = _IDENT.match(text, pos)
|
|
264
|
+
if not m:
|
|
265
|
+
break
|
|
266
|
+
parts.append(m.group(1).replace('""', '"') if m.group(1) is not None
|
|
267
|
+
else m.group(2).lower())
|
|
268
|
+
pos = m.end()
|
|
269
|
+
while pos < len(text) and text[pos].isspace():
|
|
270
|
+
pos += 1
|
|
271
|
+
if pos < len(text) and text[pos] == ".":
|
|
272
|
+
pos += 1
|
|
273
|
+
continue
|
|
274
|
+
break
|
|
275
|
+
return parts
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
_SOURCE_HEAD = re.compile(r"\s*create\s+(?:or\s+replace\s+)?"
|
|
279
|
+
r"(function|procedure|(?:recursive\s+)?view)\s+", re.I)
|
|
280
|
+
_COMMENT_HEAD = re.compile(r"\s*comment\s+on\s+(table|column)\s+", re.I)
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def parse_object(stmt, group):
|
|
284
|
+
# comments blanked, quoted names kept: a blanked "Name" would read as space
|
|
285
|
+
masked = lex(stmt).uncommented
|
|
286
|
+
if group == "source":
|
|
287
|
+
m = _SOURCE_HEAD.match(masked)
|
|
288
|
+
if not m:
|
|
289
|
+
return None
|
|
290
|
+
parts = _qualified_name(stmt, m.end())
|
|
291
|
+
kind = "VIEW" if "view" in m.group(1).lower() else m.group(1).upper()
|
|
292
|
+
return {"type": kind, "name": parts[-1],
|
|
293
|
+
"schema": parts[-2] if len(parts) > 1 else None}
|
|
294
|
+
if group == "comments":
|
|
295
|
+
m = _COMMENT_HEAD.match(masked)
|
|
296
|
+
if not m:
|
|
297
|
+
return None
|
|
298
|
+
parts = _qualified_name(stmt, m.end())
|
|
299
|
+
if m.group(1).lower() == "table":
|
|
300
|
+
return {"type": "COMMENT", "kind": "TABLE", "name": parts[-1],
|
|
301
|
+
"table": parts[-1], "column": None,
|
|
302
|
+
"schema": parts[-2] if len(parts) > 1 else None}
|
|
303
|
+
return {"type": "COMMENT", "kind": "COLUMN",
|
|
304
|
+
"name": f"{parts[-2]}.{parts[-1]}", "table": parts[-2],
|
|
305
|
+
"column": parts[-1],
|
|
306
|
+
"schema": parts[-3] if len(parts) > 2 else None}
|
|
307
|
+
return None
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
REVALIDATE = ("Refused: pythia-pg cannot show you the rows this statement would "
|
|
311
|
+
"touch — {}.\nUPDATE and DELETE are approved on their row set, so "
|
|
312
|
+
"a set nobody can see cannot be approved. Write it as one "
|
|
313
|
+
"statement over one table with a plain WHERE, or run it by hand "
|
|
314
|
+
"with a DBA watching.")
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def dml_probe(stmt):
|
|
318
|
+
"""(target, predicate or None) of the rows an UPDATE or DELETE touches;
|
|
319
|
+
None for INSERT, which has no rows beforehand. What it cannot read as
|
|
320
|
+
one predicate over one table it refuses: a parser that guesses
|
|
321
|
+
generously is one that lets a row set through unseen."""
|
|
322
|
+
lexed = lex(stmt)
|
|
323
|
+
low = lexed.masked.lower()
|
|
324
|
+
stmt = lexed.uncommented # same positions; a comment never reaches the probe
|
|
325
|
+
verb = re.match(r"\s*(insert|update|delete|merge)\b", low).group(1)
|
|
326
|
+
if verb == "insert":
|
|
327
|
+
return None
|
|
328
|
+
if verb == "merge":
|
|
329
|
+
sys.exit("Refused: a MERGE cannot be revalidated — the rows it touches "
|
|
330
|
+
"are decided by a join against its source, so pythia-pg cannot "
|
|
331
|
+
"show you the set you would be approving.\nExpress it as an "
|
|
332
|
+
"UPDATE and/or a DELETE, or run it by hand with a DBA watching.")
|
|
333
|
+
if re.search(r"\bselect\b", low):
|
|
334
|
+
sys.exit(REVALIDATE.format("it contains a subquery, so its row set is "
|
|
335
|
+
"not one predicate over one table"))
|
|
336
|
+
if re.search(r"\bcurrent\s+of\b", low):
|
|
337
|
+
sys.exit(REVALIDATE.format("WHERE CURRENT OF names a cursor's row, not "
|
|
338
|
+
"a predicate"))
|
|
339
|
+
wheres = [w.start() for w in re.finditer(r"\bwhere\b", low)]
|
|
340
|
+
if len(wheres) > 1:
|
|
341
|
+
sys.exit(REVALIDATE.format("it has more than one WHERE"))
|
|
342
|
+
returning = re.search(r"\breturning\b", low)
|
|
343
|
+
stop = returning.start() if returning else len(low)
|
|
344
|
+
if verb == "update":
|
|
345
|
+
start = re.match(r"\s*update\s+(only\s+)?", low).end()
|
|
346
|
+
end = start + re.search(r"\bset\b", low[start:]).start()
|
|
347
|
+
if re.search(r"\bfrom\b", low[end:stop]):
|
|
348
|
+
sys.exit(REVALIDATE.format("UPDATE ... FROM joins another table, so "
|
|
349
|
+
"its row set is not one predicate over "
|
|
350
|
+
"one table"))
|
|
351
|
+
else:
|
|
352
|
+
start = re.match(r"\s*delete\s+from\s+(only\s+)?", low).end()
|
|
353
|
+
if re.search(r"\busing\b", low[start:stop]):
|
|
354
|
+
sys.exit(REVALIDATE.format("DELETE ... USING joins another table, so "
|
|
355
|
+
"its row set is not one predicate over "
|
|
356
|
+
"one table"))
|
|
357
|
+
end = wheres[0] if wheres else stop
|
|
358
|
+
if wheres and wheres[0] < end:
|
|
359
|
+
sys.exit(REVALIDATE.format("its WHERE comes before the table it reads"))
|
|
360
|
+
target = stmt[start:end].strip()
|
|
361
|
+
pred = stmt[wheres[0] + 5:stop].strip() if wheres else None
|
|
362
|
+
return target, pred
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def destructive_reason(stmt, group):
|
|
366
|
+
"""Why a statement destroys something no journal brings back, or None. A
|
|
367
|
+
guess in the safe direction is fine: a false alarm costs one question."""
|
|
368
|
+
m = _code(stmt)
|
|
369
|
+
if group == "structural":
|
|
370
|
+
if m.startswith("drop "):
|
|
371
|
+
return ("DROP removes the object and everything in it"
|
|
372
|
+
+ (" — CASCADE also drops everything that depends on it"
|
|
373
|
+
if " cascade" in m else ""))
|
|
374
|
+
if m.startswith("truncate"):
|
|
375
|
+
return "TRUNCATE deletes every row, and nothing in the journal brings them back"
|
|
376
|
+
if re.match(r"(create|alter|drop) (role|user|group)\b", m) or " owner to " in m:
|
|
377
|
+
return "account, role and ownership changes reach every session that uses them"
|
|
378
|
+
if re.match(r"alter sequence\b.*\brestart\b", m):
|
|
379
|
+
return "restarting a sequence can hand out keys already in use"
|
|
380
|
+
if re.search(r"\brename\b", m):
|
|
381
|
+
return "renaming breaks every caller that still uses the old name"
|
|
382
|
+
if m.startswith("alter ") and re.search(r"\bdrop\b", m):
|
|
383
|
+
return "ALTER ... DROP removes a column or constraint, and a column's data with it"
|
|
384
|
+
if re.search(r"\balter (column )?\S+ (set data )?type\b", m):
|
|
385
|
+
return ("changing a column's type rewrites the table and can lose what "
|
|
386
|
+
"the new type cannot hold")
|
|
387
|
+
if m.startswith("create or replace "):
|
|
388
|
+
return ("CREATE OR REPLACE overwrites an existing definition, and this "
|
|
389
|
+
"kind of object has no snapshot")
|
|
390
|
+
if group == "data_dml":
|
|
391
|
+
if re.match(r"(delete|update)\b", m) and not re.search(r"\bwhere\b", m):
|
|
392
|
+
return "no WHERE clause — it touches every row in the table"
|
|
393
|
+
if re.match(r"(delete|update)\b", m) and re.search(
|
|
394
|
+
r"\bwhere \(?\s*(1 ?= ?1|0 ?= ?0|true)\b", m):
|
|
395
|
+
return "its WHERE clause is always true — it touches every row"
|
|
396
|
+
if group == "grants":
|
|
397
|
+
if m.startswith("revoke "):
|
|
398
|
+
return "REVOKE can break other users' and programs' access"
|
|
399
|
+
if re.search(r"\bto public\b", m):
|
|
400
|
+
return "GRANT ... TO PUBLIC gives every role in the database access"
|
|
401
|
+
if re.search(r"\bwith (grant|admin) option\b", m):
|
|
402
|
+
return "a grant option lets the grantee pass the privilege on"
|
|
403
|
+
if re.search(r"\ball (tables|sequences|functions|routines) in schema\b", m):
|
|
404
|
+
return "ON ALL ... IN SCHEMA reaches every object there at once"
|
|
405
|
+
return None
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def quote_ident(name):
|
|
409
|
+
if re.fullmatch(r"[a-z_][a-z0-9_$]*", name):
|
|
410
|
+
return name
|
|
411
|
+
return '"' + name.replace('"', '""') + '"'
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def restore_statement(group, ident, before, outcome):
|
|
415
|
+
"""The statement that puts it back: the snapshot for a replaced object, a
|
|
416
|
+
DROP for a new one, the old comment (or NULL) for a comment."""
|
|
417
|
+
if group == "source":
|
|
418
|
+
if before.strip():
|
|
419
|
+
text = before.rstrip()
|
|
420
|
+
return text if text.endswith(";") else text + ";"
|
|
421
|
+
return f"DROP {ident['type']} {outcome['drop']};"
|
|
422
|
+
if group == "comments":
|
|
423
|
+
schema = ident.get("schema") or outcome.get("schema")
|
|
424
|
+
target = ".".join(quote_ident(p) for p in
|
|
425
|
+
(schema, ident["table"], ident["column"]) if p)
|
|
426
|
+
literal = "NULL" if not before else "'" + before.replace("'", "''") + "'"
|
|
427
|
+
return f"COMMENT ON {ident['kind']} {target} IS {literal};"
|
|
428
|
+
return ""
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
# --- asking the database around a write --------------------------------------------
|
|
432
|
+
|
|
433
|
+
def _run_text(conn, sql):
|
|
434
|
+
"""Run statement text exactly as written: no parameters, so the driver
|
|
435
|
+
reads no % in it (a RAISE format, a LIKE pattern)."""
|
|
436
|
+
with conn.cursor() as cur:
|
|
437
|
+
cur.execute(sql)
|
|
438
|
+
if cur.description is None:
|
|
439
|
+
return [], [], cur.rowcount
|
|
440
|
+
return [d[0] for d in cur.description], cur.fetchall(), cur.rowcount
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
def measure_row_set(conn, target, pred):
|
|
444
|
+
"""Count, fingerprint and sample the rows an UPDATE or DELETE will touch.
|
|
445
|
+
The fingerprint is the rows' ctids: an update between preview and
|
|
446
|
+
confirm gives a row a new one, so it catches a row that changed as well
|
|
447
|
+
as one that joined or left. The text comes from the very file about to
|
|
448
|
+
run, so it opens no door that statement does not open already."""
|
|
449
|
+
where = f"where {pred}" if pred else ""
|
|
450
|
+
|
|
451
|
+
def fill(name):
|
|
452
|
+
return (load_query(QUERY_DIR, name).replace("/*TARGET*/", target)
|
|
453
|
+
.replace("/*WHERE*/", where))
|
|
454
|
+
import psycopg
|
|
455
|
+
try:
|
|
456
|
+
_, agg, _ = _run_text(conn, fill("row-set-count.sql"))
|
|
457
|
+
cols, rows, _ = _run_text(conn, fill("row-set-sample.sql"))
|
|
458
|
+
except psycopg.Error as e:
|
|
459
|
+
sys.exit("Refused: pythia-pg could not measure the rows this statement "
|
|
460
|
+
f"would touch.\n {target} — {_error_text(e)}\nRevalidation is "
|
|
461
|
+
"mandatory for UPDATE and DELETE, so a set that cannot be "
|
|
462
|
+
"measured cannot be approved.")
|
|
463
|
+
n, fingerprint = agg[0]
|
|
464
|
+
return {"target": target, "predicate": pred or "", "count": int(n or 0),
|
|
465
|
+
"hash": fingerprint, "columns": [str(c).upper() for c in cols],
|
|
466
|
+
"sample": [[cell(v) for v in r] for r in rows[:SAMPLE_ROWS]],
|
|
467
|
+
"truncated": len(rows) > SAMPLE_ROWS}
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _code_objects(conn, schema, ident):
|
|
471
|
+
kinds = ("VIEW",) if ident["type"] == "VIEW" else ("FUNCTION", "PROCEDURE")
|
|
472
|
+
return {unit: (kind, qualified, definition)
|
|
473
|
+
for unit, kind, qualified, definition in
|
|
474
|
+
q(conn, "code-objects.sql", s=schema, n=ident["name"])[1]
|
|
475
|
+
if kind in kinds}
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def snapshot(conn, schema, group, ident):
|
|
479
|
+
"""Everything the statement could replace, as it stands: every overload
|
|
480
|
+
of a function (which one is replaced is the rehearsal's to say), a view,
|
|
481
|
+
or a comment."""
|
|
482
|
+
if group == "source":
|
|
483
|
+
return {unit: d for unit, (_, _, d) in _code_objects(conn, schema, ident).items()}
|
|
484
|
+
if group == "comments":
|
|
485
|
+
rows = q(conn, "comment-current.sql", s=ident.get("schema") or schema,
|
|
486
|
+
t=ident["table"], c=ident["column"])[1]
|
|
487
|
+
return {ident["name"]: rows[0][0] or ""} if rows else {}
|
|
488
|
+
return {}
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _error_text(e):
|
|
492
|
+
diag = getattr(e, "diag", None)
|
|
493
|
+
first = (getattr(diag, "message_primary", None) or str(e)).strip().splitlines()[0]
|
|
494
|
+
detail = getattr(diag, "message_detail", None)
|
|
495
|
+
state = getattr(e, "sqlstate", None)
|
|
496
|
+
if state == "57014":
|
|
497
|
+
return (f"{first} — a rehearsal is bounded at {STATEMENT_TIMEOUT}, so a "
|
|
498
|
+
"long rewrite is not run in a preview: run it by hand, with a "
|
|
499
|
+
"DBA watching")
|
|
500
|
+
if state == "55P03":
|
|
501
|
+
return (f"{first} — another session holds a lock it needs; the "
|
|
502
|
+
f"rehearsal waits at most {LOCK_TIMEOUT}")
|
|
503
|
+
return first + (f" ({detail})" if detail else "")
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _code_findings(conn, schema):
|
|
507
|
+
"""(errors as {(name, type, message)}, blind spot) across the schema —
|
|
508
|
+
the before and after of a rehearsal, the way pythia compares INVALID."""
|
|
509
|
+
rows, blind = read.compile_errors(conn, schema, None)
|
|
510
|
+
return {(str(r[0]), str(r[1]), str(r[5])) for r in rows
|
|
511
|
+
if str(r[4]).lower() == "error"}, blind
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def rehearse(conn, schema, group, stmt, ident, snap):
|
|
515
|
+
"""Run the statement in the open transaction and verify it. Core rolls
|
|
516
|
+
back after a preview and commits an approved confirm."""
|
|
517
|
+
import psycopg
|
|
518
|
+
q(conn, "rehearsal-limits.sql", st=STATEMENT_TIMEOUT, lt=LOCK_TIMEOUT)
|
|
519
|
+
checked = group in ("source", "structural")
|
|
520
|
+
found_before, _ = _code_findings(conn, schema) if checked else (set(), None)
|
|
521
|
+
try:
|
|
522
|
+
_, _, rowcount = _run_text(conn, stmt)
|
|
523
|
+
except psycopg.Error as e:
|
|
524
|
+
return {"error": _error_text(e)}
|
|
525
|
+
out = {"error": None, "object": None, "type": None, "after": stmt,
|
|
526
|
+
"rows": None, "broken": [], "own_errors": [], "verified": True,
|
|
527
|
+
"blind": None, "notes": [], "schema": schema}
|
|
528
|
+
if group == "source":
|
|
529
|
+
after = _code_objects(conn, schema, ident)
|
|
530
|
+
created = [u for u in after if u not in snap]
|
|
531
|
+
changed = [u for u in after if u in snap and after[u][2] != snap[u]]
|
|
532
|
+
obj = (created + changed + list(after))[0] if after else ident["name"]
|
|
533
|
+
if obj in after:
|
|
534
|
+
out.update(object=obj, type=after[obj][0], after=after[obj][2],
|
|
535
|
+
drop=after[obj][1])
|
|
536
|
+
if created and snap:
|
|
537
|
+
others = ", ".join(snap)
|
|
538
|
+
out["notes"].append(
|
|
539
|
+
f"{obj} is a new overload: {others} "
|
|
540
|
+
f"{'stays' if len(snap) == 1 else 'stay'} callable — CREATE OR "
|
|
541
|
+
"REPLACE replaces only the same argument types")
|
|
542
|
+
if len(created) + len(changed) > 1:
|
|
543
|
+
out["notes"].append("the statement changed more than one object: "
|
|
544
|
+
+ ", ".join(created + changed))
|
|
545
|
+
elif group == "comments":
|
|
546
|
+
now = snapshot(conn, schema, group, ident)
|
|
547
|
+
out.update(object=ident["name"], type="COMMENT",
|
|
548
|
+
after=now.get(ident["name"], ""))
|
|
549
|
+
elif group == "data_dml":
|
|
550
|
+
out["rows"] = rowcount
|
|
551
|
+
out["notes"].append("a rehearsal runs the statement's triggers and "
|
|
552
|
+
"advances any sequence it draws from: rolling back "
|
|
553
|
+
"undoes the rows, not nextval")
|
|
554
|
+
elif group == "grants":
|
|
555
|
+
out.update(verified=False, blind="who can still do what after a GRANT "
|
|
556
|
+
"or REVOKE is not something a rehearsal can check")
|
|
557
|
+
if checked:
|
|
558
|
+
found_after, blind = _code_findings(conn, schema)
|
|
559
|
+
own = sorted(f for f in found_after if f[0] == out["object"])
|
|
560
|
+
out["own_errors"] = own
|
|
561
|
+
out["broken"] = sorted(f for f in found_after - found_before if f not in own)
|
|
562
|
+
if blind:
|
|
563
|
+
out.update(verified=False, blind=blind)
|
|
564
|
+
return out
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def privilege_warning(conn, schema):
|
|
568
|
+
rows = q(conn, "session-power.sql", s=schema)[1]
|
|
569
|
+
if not rows:
|
|
570
|
+
return None
|
|
571
|
+
superuser, createrole, owns, account = rows[0]
|
|
572
|
+
if superuser:
|
|
573
|
+
return (f"! Connected as a superuser ({account}): this account can do "
|
|
574
|
+
"anything on the server. A role that owns only what it changes "
|
|
575
|
+
"is safer.")
|
|
576
|
+
if createrole:
|
|
577
|
+
return (f"! {account} can create roles: far more power than changing "
|
|
578
|
+
"this schema needs.")
|
|
579
|
+
if owns:
|
|
580
|
+
return (f"! Connected as the owner of schema {schema} ({account}): the "
|
|
581
|
+
"account can drop anything in it. A least-privilege account is "
|
|
582
|
+
"safer.")
|
|
583
|
+
return None
|