pg-logical-repl 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. pg_logical_repl/__init__.py +3 -0
  2. pg_logical_repl/__main__.py +6 -0
  3. pg_logical_repl/capabilities.py +136 -0
  4. pg_logical_repl/catalog.py +320 -0
  5. pg_logical_repl/cli.py +277 -0
  6. pg_logical_repl/component.py +66 -0
  7. pg_logical_repl/config.py +356 -0
  8. pg_logical_repl/coverage.py +117 -0
  9. pg_logical_repl/cutover.py +239 -0
  10. pg_logical_repl/db.py +131 -0
  11. pg_logical_repl/ddl.py +352 -0
  12. pg_logical_repl/ddl_queue.py +461 -0
  13. pg_logical_repl/emergency.py +296 -0
  14. pg_logical_repl/errors.py +44 -0
  15. pg_logical_repl/health.py +194 -0
  16. pg_logical_repl/integrity.py +185 -0
  17. pg_logical_repl/logging_config.py +45 -0
  18. pg_logical_repl/monitoring.py +277 -0
  19. pg_logical_repl/plans.py +167 -0
  20. pg_logical_repl/preflight.py +290 -0
  21. pg_logical_repl/process.py +150 -0
  22. pg_logical_repl/recovery.py +188 -0
  23. pg_logical_repl/resync.py +267 -0
  24. pg_logical_repl/roles.py +123 -0
  25. pg_logical_repl/schema.py +89 -0
  26. pg_logical_repl/security.py +117 -0
  27. pg_logical_repl/service.py +773 -0
  28. pg_logical_repl/sql/base.sql +67 -0
  29. pg_logical_repl/sql/capture.sql +215 -0
  30. pg_logical_repl/sql/ddl_support.sql +109 -0
  31. pg_logical_repl/sql/freeze.sql +117 -0
  32. pg_logical_repl/sql/replay.sql +48 -0
  33. pg_logical_repl/state.py +160 -0
  34. pg_logical_repl/validation.py +353 -0
  35. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/IMPLEMENTATION_STATUS.md +207 -0
  36. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/README.md +406 -0
  37. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/DDL_IMPLEMENTATION.md +170 -0
  38. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/RELEASING.md +92 -0
  39. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/TESTING.md +142 -0
  40. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/examples/local.yaml +55 -0
  41. pg_logical_repl-0.2.0.dist-info/METADATA +425 -0
  42. pg_logical_repl-0.2.0.dist-info/RECORD +45 -0
  43. pg_logical_repl-0.2.0.dist-info/WHEEL +5 -0
  44. pg_logical_repl-0.2.0.dist-info/entry_points.txt +2 -0
  45. pg_logical_repl-0.2.0.dist-info/top_level.txt +1 -0
pg_logical_repl/ddl.py ADDED
@@ -0,0 +1,352 @@
1
+ """DDL planning and source execution. Native apply replays the protected journal."""
2
+
3
+ import hashlib
4
+ import json
5
+ from pathlib import Path
6
+
7
+ from pglast import ast, parse_sql, parser
8
+ from pglast.enums import AlterTableType, DropBehavior, ObjectType
9
+
10
+ from . import ddl_queue
11
+ from .errors import PreconditionError
12
+ from .security import qualified as qn
13
+
14
+
15
+ def walk(node):
16
+ if isinstance(node, ast.Node):
17
+ yield node
18
+ for name in node:
19
+ yield from walk(getattr(node, name))
20
+ elif isinstance(node, (tuple, list)):
21
+ for child in node:
22
+ yield from walk(child)
23
+
24
+
25
+ def scope(schema):
26
+ if not schema or schema in {"lc", "information_schema"} or schema.startswith("pg_"):
27
+ raise PreconditionError("DDL_SCOPE", "Use fully qualified application objects")
28
+
29
+
30
+ def classify(statement):
31
+ try:
32
+ parsed = parse_sql(statement)
33
+ except parser.ParseError as exc:
34
+ raise PreconditionError("DDL_PARSE", str(exc)) from None
35
+ if len(parsed) != 1:
36
+ raise PreconditionError("DDL_MULTIPLE", "Exactly one top-level DDL command is required")
37
+ node = parsed[0].stmt
38
+ allowed = {
39
+ "AlterTableStmt",
40
+ "RenameStmt",
41
+ "CreateStmt",
42
+ "CreateTableAsStmt",
43
+ "IndexStmt",
44
+ "DropStmt",
45
+ "ViewStmt",
46
+ "CreateFunctionStmt",
47
+ "AlterFunctionStmt",
48
+ "CreateTrigStmt",
49
+ "CreateSeqStmt",
50
+ "AlterSeqStmt",
51
+ "CreateSchemaStmt",
52
+ "AlterOwnerStmt",
53
+ "AlterObjectSchemaStmt",
54
+ "CommentStmt",
55
+ "GrantStmt",
56
+ "AlterDefaultPrivilegesStmt",
57
+ "CreateEnumStmt",
58
+ "AlterEnumStmt",
59
+ "CreateDomainStmt",
60
+ "AlterDomainStmt",
61
+ }
62
+ if type(node).__name__ not in allowed:
63
+ raise PreconditionError("DDL_NOT_QUALIFIED", "This DDL class is not qualified for replay")
64
+ classification = "in_stream"
65
+ relation = getattr(node, "relation", None)
66
+ if isinstance(node, ast.CreateTableAsStmt):
67
+ relation = node.into.rel
68
+ if isinstance(node, ast.CreateSchemaStmt):
69
+ scope(node.schemaname)
70
+ if node.schemaElts:
71
+ raise PreconditionError("DDL_MULTIPLE", "CREATE SCHEMA must not contain nested commands")
72
+ if isinstance(node, ast.CreateFunctionStmt):
73
+ if len(node.funcname) != 2:
74
+ raise PreconditionError("DDL_SCOPE", "Fully qualify the function name")
75
+ scope(node.funcname[0].sval)
76
+ for part in walk(node):
77
+ if isinstance(part, ast.RangeVar):
78
+ # References in view bodies may use the recorded caller search_path.
79
+ if part is relation or part.schemaname:
80
+ scope(part.schemaname)
81
+ if part.relpersistence != "p":
82
+ raise PreconditionError("DDL_PERSISTENCE", "Only logged application DDL is supported")
83
+ if getattr(part, "behavior", None) == DropBehavior.DROP_CASCADE:
84
+ raise PreconditionError(
85
+ "DDL_CASCADE", "CASCADE requires a separately reviewed dependency migration"
86
+ )
87
+ if isinstance(node, ast.AlterTableStmt):
88
+ if len(node.cmds) != 1:
89
+ raise PreconditionError("DDL_MULTIPLE", "Use one ALTER TABLE action per command")
90
+ cmd = node.cmds[0]
91
+ supported = {
92
+ "AT_AddColumn",
93
+ "AT_DropColumn",
94
+ "AT_ColumnDefault",
95
+ "AT_SetNotNull",
96
+ "AT_DropNotNull",
97
+ "AT_AlterColumnType",
98
+ "AT_SetStorage",
99
+ "AT_SetStatistics",
100
+ "AT_SetCompression",
101
+ "AT_AddConstraint",
102
+ "AT_DropConstraint",
103
+ "AT_ValidateConstraint",
104
+ "AT_ReplicaIdentity",
105
+ "AT_AttachPartition",
106
+ "AT_DetachPartition",
107
+ "AT_ChangeOwner",
108
+ "AT_SetRelOptions",
109
+ "AT_ResetRelOptions",
110
+ "AT_EnableTrig",
111
+ "AT_DisableTrig",
112
+ "AT_EnableTrigUser",
113
+ "AT_DisableTrigUser",
114
+ }
115
+ if cmd.subtype.name not in supported:
116
+ raise PreconditionError("DDL_NOT_QUALIFIED", "ALTER action is outside the DDL matrix")
117
+ if cmd.subtype in {AlterTableType.AT_AddColumn, AlterTableType.AT_AlterColumnType}:
118
+ if any(isinstance(x, (ast.FuncCall, ast.SQLValueFunction)) for x in walk(cmd.def_)):
119
+ raise PreconditionError(
120
+ "DDL_NONDETERMINISTIC",
121
+ "Function calls in ADD/USING require a separate deterministic migration",
122
+ )
123
+ if cmd.subtype in {AlterTableType.AT_ValidateConstraint, AlterTableType.AT_AddConstraint}:
124
+ classification = "in_stream_and_queue"
125
+ if cmd.subtype in {AlterTableType.AT_AttachPartition, AlterTableType.AT_DetachPartition}:
126
+ if cmd.def_.concurrent:
127
+ raise PreconditionError("DDL_NOT_QUALIFIED", "Concurrent partition detach is not supported")
128
+ classification = "in_stream_and_queue"
129
+ if isinstance(node, (ast.CreateStmt, ast.CreateTableAsStmt)):
130
+ classification = "in_stream_and_queue"
131
+ if isinstance(node, ast.IndexStmt):
132
+ if not node.idxname or node.if_not_exists:
133
+ raise PreconditionError("DDL_INDEX_NAME", "Use an explicit index name without IF NOT EXISTS")
134
+ classification = "out_of_stream"
135
+ if isinstance(node, ast.DropStmt):
136
+ allowed_drop = {
137
+ "OBJECT_TABLE",
138
+ "OBJECT_INDEX",
139
+ "OBJECT_VIEW",
140
+ "OBJECT_MATVIEW",
141
+ "OBJECT_SEQUENCE",
142
+ "OBJECT_FUNCTION",
143
+ "OBJECT_PROCEDURE",
144
+ "OBJECT_TRIGGER",
145
+ "OBJECT_SCHEMA",
146
+ "OBJECT_TYPE",
147
+ "OBJECT_DOMAIN",
148
+ }
149
+ if node.removeType.name not in allowed_drop or len(node.objects) != 1:
150
+ raise PreconditionError("DDL_NOT_QUALIFIED", "Drop one application object with RESTRICT")
151
+ obj = node.objects[0]
152
+ names = obj.objname if isinstance(obj, ast.ObjectWithArgs) else obj
153
+ if len(names) < (1 if node.removeType == ObjectType.OBJECT_SCHEMA else 2):
154
+ raise PreconditionError("DDL_SCOPE", "Fully qualify the object being dropped")
155
+ scope(names[0].sval)
156
+ if node.removeType == ObjectType.OBJECT_INDEX:
157
+ classification = "out_of_stream"
158
+ if node.removeType == ObjectType.OBJECT_TABLE:
159
+ classification = "in_stream_and_queue"
160
+ if isinstance(node, ast.AlterSeqStmt) and any(x.defname == "restart" for x in node.options):
161
+ raise PreconditionError("DDL_SEQUENCE_RESTART", "Sequence values are synchronized at cutover")
162
+ return dict(
163
+ classification=classification,
164
+ table=qn(relation.schemaname, relation.relname) if relation else None,
165
+ statement=statement,
166
+ statement_sha256=hashlib.sha256(statement.encode()).hexdigest(),
167
+ source_autocommit=bool(getattr(node, "concurrent", False)),
168
+ )
169
+
170
+
171
+ async def execute(service, path, plan_hash, sql_file):
172
+ statement = Path(sql_file).read_text()
173
+ classification = classify(statement)
174
+ async with service.mutation("ddl-exec", path, plan_hash, states={"REPLICATING"}) as (sc, tc, plan):
175
+ await service.owned(sc)
176
+ await service.owned(tc)
177
+ if service.config.ddl_mode == "frozen":
178
+ raise PreconditionError("DDL_FROZEN", "DDL mode is frozen")
179
+ if plan.get("extra", {}).get("statement_sha256") != classification["statement_sha256"]:
180
+ raise PreconditionError("DDL_PLAN_MISMATCH", "Plan must bind the exact SQL file")
181
+ await ddl_queue.process(service, sc, tc)
182
+ archived = await sc.fetchval(
183
+ "SELECT details FROM lc.operations WHERE name=$1", "ddl_archive:" + plan_hash
184
+ )
185
+ if archived:
186
+ receipt = json.loads(archived)
187
+ if receipt["statement_sha256"] != classification["statement_sha256"]:
188
+ raise PreconditionError("DDL_PLAN_MISMATCH", "Archived operation differs from the SQL file")
189
+ state = service.store.read()
190
+ state["active_plan"] = None
191
+ service.store.save(state)
192
+ return dict(receipt, applied=True, archived=True)
193
+ ddl_id = await sc.fetchval("SELECT id FROM lc.ddl_log WHERE operation_key=$1", plan_hash)
194
+ if not ddl_id:
195
+ await sc.fetchval("SELECT set_config('lc.managed','on',false)")
196
+ await sc.fetchval("SELECT set_config('lc.operation_key',$1,false)", plan_hash)
197
+ try:
198
+ if classification["source_autocommit"]:
199
+ await concurrent_source(service, sc, statement, plan_hash)
200
+ else:
201
+ async with sc.transaction():
202
+ await sc.execute(statement)
203
+ finally:
204
+ await sc.fetchval("SELECT set_config('lc.managed','',false)")
205
+ await sc.fetchval("SELECT set_config('lc.operation_key','',false)")
206
+ ddl_id = await sc.fetchval("SELECT id FROM lc.ddl_log WHERE operation_key=$1", plan_hash)
207
+ if not ddl_id:
208
+ raise PreconditionError(
209
+ "DDL_NOT_CAPTURED", "Source command has no transactional capture receipt"
210
+ )
211
+ await sc.execute(
212
+ "UPDATE lc.operations SET details=details||'{\"complete\":true}'::jsonb WHERE name=$1",
213
+ "ddl_concurrent:" + plan_hash,
214
+ )
215
+ await service.barrier(sc, tc, "ddl")
216
+ # Drain any queue created by the last transaction before reporting success.
217
+ await service.drain_ddl(sc, tc)
218
+ if not await tc.fetchval("SELECT EXISTS(SELECT FROM lc.ddl_applied WHERE id=$1)", ddl_id):
219
+ raise PreconditionError("DDL_NOT_APPLIED", "DDL journal row has no replay receipt")
220
+ s = service.store.read()
221
+ s["active_plan"] = None
222
+ service.store.save(s)
223
+ return dict(id=ddl_id, applied=True, statement_sha256=classification["statement_sha256"])
224
+
225
+
226
+ def index_signature(statement):
227
+ """Normalize PG's index definition and a planned CREATE INDEX for recovery."""
228
+ from pglast.stream import RawStream
229
+
230
+ node = parse_sql(statement)[0].stmt
231
+ if not isinstance(node, ast.IndexStmt):
232
+ raise PreconditionError("DDL_INDEX_SIGNATURE", "Expected a CREATE INDEX definition")
233
+ node.concurrent = False
234
+ node.if_not_exists = False
235
+ return RawStream()(node)
236
+
237
+
238
+ async def concurrent_source(service, sc, statement, plan_hash):
239
+ """Reconcile a concurrent source operation across PostgreSQL's internal commits."""
240
+ import json
241
+
242
+ node = parse_sql(statement)[0].stmt
243
+ creating = isinstance(node, ast.IndexStmt)
244
+ if creating:
245
+ schema, name = node.relation.schemaname, node.idxname
246
+ else:
247
+ schema, name = [x.sval for x in node.objects[0]]
248
+ qualified = qn(schema, name)
249
+ operation = "ddl_concurrent:" + plan_hash
250
+ receipt = await sc.fetchval("SELECT details FROM lc.operations WHERE name=$1", operation)
251
+ index = await sc.fetchrow(
252
+ """SELECT c.oid::int,i.indisvalid,pg_get_indexdef(c.oid) AS definition
253
+ FROM pg_class c JOIN pg_index i ON i.indexrelid=c.oid WHERE c.oid=to_regclass($1)""",
254
+ qualified,
255
+ )
256
+ if not receipt:
257
+ if creating and index:
258
+ raise PreconditionError("DDL_INDEX_EXISTS", "Concurrent CREATE requires an absent source index")
259
+ if not creating and not index and not node.missing_ok:
260
+ raise PreconditionError("DDL_INDEX_MISSING", "Concurrent DROP requires the planned source index")
261
+ intent = dict(
262
+ plan_hash=plan_hash,
263
+ schema=schema,
264
+ name=name,
265
+ kind="create_index" if creating else "drop_index",
266
+ oid=index["oid"] if index else None,
267
+ definition=index["definition"] if index else None,
268
+ complete=False,
269
+ )
270
+ await sc.execute(
271
+ "INSERT INTO lc.operations(name,details) VALUES($1,$2::jsonb)", operation, json.dumps(intent)
272
+ )
273
+ else:
274
+ intent = json.loads(receipt)
275
+ if index and intent.get("oid") and index["oid"] != intent["oid"]:
276
+ raise PreconditionError("DDL_INDEX_OWNERSHIP", "Source index OID changed after the intent")
277
+ if creating and index:
278
+ if index_signature(index["definition"]) != index_signature(statement):
279
+ raise PreconditionError(
280
+ "DDL_INDEX_DRIFT", "Source index definition differs from the approved CREATE"
281
+ )
282
+ if not index["indisvalid"]:
283
+ # The failed build belongs to the durable creation intent.
284
+ await sc.fetchval("SELECT set_config('lc.managed','',false)")
285
+ await sc.execute("DROP INDEX CONCURRENTLY " + qualified)
286
+ await sc.fetchval("SELECT set_config('lc.managed','on',false)")
287
+ index = None
288
+ already_done = bool(creating and index and index["indisvalid"]) or (not creating and not index)
289
+ if not already_done:
290
+ try:
291
+ await sc.execute(statement)
292
+ finally:
293
+ oid = await sc.fetchval("SELECT to_regclass($1)::oid::int", qualified)
294
+ if oid:
295
+ await sc.execute(
296
+ "UPDATE lc.operations SET details=details||jsonb_build_object('oid',$2::int) WHERE name=$1",
297
+ operation,
298
+ oid,
299
+ )
300
+ if not await sc.fetchval("SELECT EXISTS(SELECT FROM lc.ddl_log WHERE operation_key=$1)", plan_hash):
301
+ # A concurrent operation can commit before its event-trigger transaction.
302
+ # Verify the catalog then append the missing journal exactly once.
303
+ if creating:
304
+ definition = await sc.fetchval("SELECT pg_get_indexdef(to_regclass($1))", qualified)
305
+ valid = await sc.fetchval(
306
+ "SELECT indisvalid FROM pg_index WHERE indexrelid=to_regclass($1)", qualified
307
+ )
308
+ if not valid or index_signature(definition) != index_signature(statement):
309
+ raise PreconditionError(
310
+ "DDL_INDEX_POSTCONDITION", "Source concurrent index is not valid/matching"
311
+ )
312
+ import re
313
+
314
+ target_statement = re.sub(
315
+ r"^CREATE (UNIQUE )?INDEX ", r"CREATE \1INDEX CONCURRENTLY ", definition
316
+ )
317
+ jobs = [
318
+ dict(
319
+ kind="create_index",
320
+ statement=target_statement,
321
+ schema=schema,
322
+ name=name,
323
+ definition=definition,
324
+ )
325
+ ]
326
+ else:
327
+ if await sc.fetchval("SELECT to_regclass($1) IS NOT NULL", qualified):
328
+ raise PreconditionError("DDL_INDEX_POSTCONDITION", "Source concurrent DROP is incomplete")
329
+ jobs = (
330
+ [
331
+ dict(
332
+ kind="drop_index",
333
+ statement=statement,
334
+ schema=schema,
335
+ name=name,
336
+ definition=intent["definition"],
337
+ )
338
+ ]
339
+ if intent.get("definition")
340
+ else []
341
+ )
342
+ await sc.fetchval("SELECT set_config('lc.caller_search_path',current_setting('search_path'),false)")
343
+ await sc.fetchval(
344
+ "SELECT lc.record_ddl($1,$2,$3::jsonb,$4::jsonb)",
345
+ statement,
346
+ "CREATE INDEX" if creating else "DROP INDEX",
347
+ "[]",
348
+ json.dumps(jobs),
349
+ )
350
+ await sc.execute(
351
+ "UPDATE lc.operations SET details=details||'{\"complete\":true}'::jsonb WHERE name=$1", operation
352
+ )