pg-logical-repl 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. pg_logical_repl/__init__.py +3 -0
  2. pg_logical_repl/__main__.py +6 -0
  3. pg_logical_repl/capabilities.py +136 -0
  4. pg_logical_repl/catalog.py +320 -0
  5. pg_logical_repl/cli.py +277 -0
  6. pg_logical_repl/component.py +66 -0
  7. pg_logical_repl/config.py +356 -0
  8. pg_logical_repl/coverage.py +117 -0
  9. pg_logical_repl/cutover.py +239 -0
  10. pg_logical_repl/db.py +131 -0
  11. pg_logical_repl/ddl.py +352 -0
  12. pg_logical_repl/ddl_queue.py +461 -0
  13. pg_logical_repl/emergency.py +296 -0
  14. pg_logical_repl/errors.py +44 -0
  15. pg_logical_repl/health.py +194 -0
  16. pg_logical_repl/integrity.py +185 -0
  17. pg_logical_repl/logging_config.py +45 -0
  18. pg_logical_repl/monitoring.py +277 -0
  19. pg_logical_repl/plans.py +167 -0
  20. pg_logical_repl/preflight.py +290 -0
  21. pg_logical_repl/process.py +150 -0
  22. pg_logical_repl/recovery.py +188 -0
  23. pg_logical_repl/resync.py +267 -0
  24. pg_logical_repl/roles.py +123 -0
  25. pg_logical_repl/schema.py +89 -0
  26. pg_logical_repl/security.py +117 -0
  27. pg_logical_repl/service.py +773 -0
  28. pg_logical_repl/sql/base.sql +67 -0
  29. pg_logical_repl/sql/capture.sql +215 -0
  30. pg_logical_repl/sql/ddl_support.sql +109 -0
  31. pg_logical_repl/sql/freeze.sql +117 -0
  32. pg_logical_repl/sql/replay.sql +48 -0
  33. pg_logical_repl/state.py +160 -0
  34. pg_logical_repl/validation.py +353 -0
  35. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/IMPLEMENTATION_STATUS.md +207 -0
  36. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/README.md +406 -0
  37. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/DDL_IMPLEMENTATION.md +170 -0
  38. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/RELEASING.md +92 -0
  39. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/TESTING.md +142 -0
  40. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/examples/local.yaml +55 -0
  41. pg_logical_repl-0.2.0.dist-info/METADATA +425 -0
  42. pg_logical_repl-0.2.0.dist-info/RECORD +45 -0
  43. pg_logical_repl-0.2.0.dist-info/WHEEL +5 -0
  44. pg_logical_repl-0.2.0.dist-info/entry_points.txt +2 -0
  45. pg_logical_repl-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,239 @@
1
+ """Two-phase detachment with a required external write fence."""
2
+
3
+ import asyncio
4
+ import json
5
+ import re
6
+ from contextlib import asynccontextmanager
7
+ from datetime import UTC, datetime
8
+ from pathlib import Path
9
+
10
+ from . import integrity
11
+ from .catalog import fingerprint, inventory
12
+ from .ddl_queue import require_ready
13
+ from .errors import PreconditionError
14
+ from .health import require_healthy
15
+ from .process import execute, pg_environment, tool
16
+ from .security import qualified as qn
17
+ from .state import write_private
18
+ from .validation import large_objects, sequences, verify
19
+
20
+
21
+ def fence(service, path):
22
+ try:
23
+ data = json.loads(Path(path).read_text())
24
+ age = (datetime.now(UTC) - datetime.fromisoformat(data["created_at"])).total_seconds()
25
+ valid = (
26
+ data["migration_id"] == service.config.migration_id
27
+ and data["source_identity"] == service.store.read()["source_identity"]
28
+ and 0 <= age <= service.config.get("cutover", "max_validation_age_seconds", 21600)
29
+ and bool(data["responsible"])
30
+ and bool(data["external_fence"])
31
+ )
32
+ except (OSError, ValueError, KeyError, TypeError):
33
+ valid = False
34
+ if not valid:
35
+ raise PreconditionError(
36
+ "WRITE_FENCE_INVALID", "A fresh external write-fence record matching source identity is required"
37
+ )
38
+ return data
39
+
40
+
41
+ @asynccontextmanager
42
+ async def locked_source(service, sc):
43
+ prepared = await sc.fetchval("SELECT count(*) FROM pg_prepared_xacts WHERE database=current_database()")
44
+ writers = await sc.fetchval("""SELECT count(*) FROM pg_stat_activity WHERE datname=current_database()
45
+ AND pid<>pg_backend_pid() AND backend_type='client backend' AND xact_start IS NOT NULL
46
+ AND application_name NOT LIKE 'pg-logical-repl/%' """)
47
+ if prepared or writers:
48
+ raise PreconditionError("WRITERS_NOT_DRAINED", "Prepared or open application transactions remain")
49
+ async with sc.transaction():
50
+ si = await inventory(sc)
51
+ if service.config.get("cutover", "share_locks", True):
52
+ for r in si["relations"]:
53
+ if r["kind"] in ("r", "p") and not r["extension"]:
54
+ await sc.execute("LOCK TABLE ONLY " + qn(r["schema"], r["name"]) + " IN SHARE MODE")
55
+ yield
56
+
57
+
58
+ async def deferred_copy(service, sc, tc):
59
+ si = await inventory(sc)
60
+ major = si["identity"]["major"]
61
+ unlogged = [
62
+ r for r in si["relations"] if r["kind"] == "r" and r["persistence"] == "u" and not r["extension"]
63
+ ]
64
+ # Include all LO metadata/ACL and unlogged data in one archive. Logged rows remain native tablesync's job.
65
+ archive = service.store.directory / "deferred.dump"
66
+ args = [
67
+ tool(service.config, major, "pg_dump"),
68
+ "--format=custom",
69
+ "--data-only",
70
+ "-b",
71
+ "--file=" + str(archive),
72
+ ]
73
+ if unlogged:
74
+ args += ["--table=" + qn(r["schema"], r["name"]) for r in unlogged]
75
+ else:
76
+ args += ["--exclude-table-data=*.*"]
77
+ with pg_environment(service.source, service.store.directory) as env:
78
+ await execute(args, env=env, redactor=service.redactor)
79
+ archive.chmod(0o600)
80
+ toc = await execute(
81
+ [tool(service.config, major, "pg_restore"), "--list", str(archive)], redactor=service.redactor
82
+ )
83
+ keep = [
84
+ line
85
+ for line in toc.splitlines()
86
+ if re.search(r"; \d+ \d+ (TABLE DATA|BLOB|BLOBS|BLOB COMMENTS|LARGE OBJECT)", line)
87
+ or re.search(r"; \d+ \d+ (ACL|COMMENT|SECURITY LABEL) - (LARGE OBJECT|BLOB)", line)
88
+ ]
89
+ if not unlogged:
90
+ keep = [line for line in keep if not re.search(r"; \d+ \d+ TABLE DATA ", line)]
91
+ listing = service.store.directory / "deferred.list"
92
+ write_private(listing, "\n".join(keep) + "\n")
93
+ output = service.store.directory / "deferred.sql"
94
+ await execute(
95
+ [tool(service.config, major, "pg_restore"), "--file=-", "--use-list=" + str(listing), str(archive)],
96
+ stdout_path=output,
97
+ redactor=service.redactor,
98
+ )
99
+ cleanup = []
100
+ if unlogged:
101
+ cleanup.append("TRUNCATE " + ",".join(qn(r["schema"], r["name"]) for r in unlogged) + ";")
102
+ # Empty target was verified before bootstrap; its LO set is migration-owned until detachment.
103
+ cleanup.append("SELECT pg_catalog.lo_unlink(oid) FROM pg_catalog.pg_largeobject_metadata;")
104
+ before = service.store.directory / "deferred-before.sql"
105
+ write_private(before, "SET LOCAL session_replication_role=replica;\n" + "\n".join(cleanup))
106
+ with pg_environment(service.target, service.store.directory) as env:
107
+ await execute(
108
+ [
109
+ tool(service.config, major, "psql"),
110
+ "-X",
111
+ "-w",
112
+ "--quiet",
113
+ "--single-transaction",
114
+ "-v",
115
+ "ON_ERROR_STOP=1",
116
+ "--file",
117
+ str(before),
118
+ "--file",
119
+ str(output),
120
+ ],
121
+ env=env,
122
+ redactor=service.redactor,
123
+ )
124
+ lo_source, lo_target = await large_objects(sc), await large_objects(tc)
125
+ if lo_source != lo_target:
126
+ raise PreconditionError("LARGE_OBJECT_MISMATCH", "Large object content, OID or privileges differ")
127
+ return dict(unlogged_tables=len(unlogged), large_objects=len(lo_source))
128
+
129
+
130
+ async def refresh_matviews(service, sc, tc):
131
+ rows = await sc.fetch("""SELECT c.oid::int,n.nspname AS schema,c.relname AS name,c.relispopulated
132
+ FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE c.relkind='m'
133
+ AND n.nspname !~ '^pg_' AND n.nspname<>'lc' ORDER BY 2,3""")
134
+ dependencies = await sc.fetch("""SELECT DISTINCT r.ev_class::int AS view,d.refobjid::int AS dependency
135
+ FROM pg_rewrite r JOIN pg_depend d ON d.classid='pg_rewrite'::regclass AND d.objid=r.oid
136
+ JOIN pg_class c ON c.oid=d.refobjid WHERE c.relkind='m' AND d.refobjid<>r.ev_class""")
137
+ remaining = {r["oid"]: r for r in rows}
138
+ done = []
139
+ while remaining:
140
+ ready = [
141
+ r
142
+ for oid, r in remaining.items()
143
+ if not any(d["view"] == oid and d["dependency"] in remaining for d in dependencies)
144
+ ]
145
+ if not ready:
146
+ raise PreconditionError("MATVIEW_DEPENDENCY_CYCLE", "Cannot determine refresh order")
147
+ for r in ready:
148
+ if r["relispopulated"]:
149
+ await tc.execute("REFRESH MATERIALIZED VIEW " + qn(r["schema"], r["name"]))
150
+ done.append(qn(r["schema"], r["name"]))
151
+ remaining.pop(r["oid"])
152
+ return done
153
+
154
+
155
+ async def finalize(service, path, plan_hash, write_fence):
156
+ async with service.mutation(
157
+ "cutover-finalize", path, plan_hash, states={"REPLICATING", "FINALIZING"}
158
+ ) as (sc, tc, plan):
159
+ await service.owned(sc)
160
+ await service.owned(tc)
161
+ await require_healthy(service, sc, tc)
162
+ await require_ready(service, tc)
163
+ await service.barrier(sc, tc)
164
+ await integrity.require(service, sc, tc)
165
+ evidence = fence(service, write_fence)
166
+ await service.state("FINALIZING", sc, tc, write_fence=evidence)
167
+ async with locked_source(service, sc):
168
+ async with service.source.connect() as marker:
169
+ barrier = await service.barrier(marker, tc, "cutover")
170
+ seq_before = await sequences(sc, tc)
171
+ await asyncio.sleep(2)
172
+ await deferred_copy(service, sc, tc)
173
+ seq = await sequences(sc, tc, apply=True)
174
+ matviews = await refresh_matviews(service, sc, tc)
175
+ report = await verify(service, sc, tc, include_deferred=True)
176
+ await integrity.require(service, sc, tc)
177
+ seq_after = await sequences(sc, tc)
178
+ if any(a["source"] != b["source"] for a, b in zip(seq_before, seq_after)):
179
+ raise PreconditionError(
180
+ "SEQUENCE_WRITER", "Source sequences changed under the external fence"
181
+ )
182
+ if not report["passed"] or not all(x["passed"] for x in seq_after):
183
+ raise PreconditionError("FINAL_VALIDATION_FAILED", "Source and target do not match")
184
+ report.update(
185
+ authoritative=True,
186
+ write_fence=evidence,
187
+ barrier=barrier,
188
+ sequences=seq,
189
+ matviews_refreshed=matviews,
190
+ )
191
+ report["large_objects"] = await large_objects(sc)
192
+ report["target_schema"] = fingerprint(await inventory(tc))
193
+ report["interventions"] = service.store.read().get("interventions", [])
194
+ if service.config.get("cutover", "analyze", "never") == "all":
195
+ for row in (await inventory(tc))["relations"]:
196
+ if row["kind"] in ("r", "p", "m") and not row["extension"]:
197
+ await tc.execute("ANALYZE " + qn(row["schema"], row["name"]))
198
+ report["analyze"] = service.config.get("cutover", "analyze", "never")
199
+ write_private(service.store.directory / "reports/cutover.json", json.dumps(report, indent=2))
200
+ await service.state("CUTOVER_READY", sc, tc, active_plan=None)
201
+ return report
202
+
203
+
204
+ async def commit(service, path, plan_hash):
205
+ async with service.mutation(
206
+ "cutover-commit", path, plan_hash, states={"CUTOVER_READY", "COMMITTING"}
207
+ ) as (sc, tc, plan):
208
+ state = service.store.read()
209
+ if state["state"] == "CUTOVER_READY":
210
+ await service.owned(sc)
211
+ await service.owned(tc)
212
+ await require_healthy(service, sc, tc)
213
+ await require_ready(service, tc)
214
+ saved = service.store.directory / "write-fence.json"
215
+ write_private(saved, json.dumps(state["write_fence"]))
216
+ fence(service, saved)
217
+ async with locked_source(service, sc):
218
+ async with service.source.connect() as marker:
219
+ await service.barrier(marker, tc, "pre-detach")
220
+ report = await verify(service, sc, tc, include_deferred=True)
221
+ await integrity.require(service, sc, tc)
222
+ baseline = json.loads((service.store.directory / "reports/cutover.json").read_text())
223
+ if [(r["table"], r["source"]) for r in report["tables"]] != [
224
+ (r["table"], r["source"]) for r in baseline["tables"]
225
+ ]:
226
+ raise PreconditionError("FENCE_VIOLATED", "Source table contents changed after finalize")
227
+ if not report["passed"] or not all(x["passed"] for x in await sequences(sc, tc)):
228
+ raise PreconditionError("CUTOVER_CHANGED", "Data or sequences changed after finalize")
229
+ if await large_objects(sc) != baseline["large_objects"] or await large_objects(
230
+ sc
231
+ ) != await large_objects(tc):
232
+ raise PreconditionError("CUTOVER_CHANGED", "Large objects changed after finalize")
233
+ await service.state("COMMITTING", tc=tc)
234
+ # Source lock transaction cannot contain DROP SUBSCRIPTION (which is on target).
235
+ await service.cleanup(sc, tc)
236
+ else:
237
+ await service.cleanup(sc, tc)
238
+ await service.state("DETACHED", active_plan=None)
239
+ return await service.status()
pg_logical_repl/db.py ADDED
@@ -0,0 +1,131 @@
1
+ """SQL-only asyncpg connections and libpq configuration for native subscriptions."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import ssl
6
+ from contextlib import asynccontextmanager
7
+
8
+ import asyncpg
9
+
10
+ from .errors import ConnectorError
11
+ from .security import password
12
+
13
+
14
+ class Database:
15
+ def __init__(self, config, side, redactor):
16
+ self.config, self.side, self.redactor = config, side, redactor
17
+ self.settings = config.endpoint(side)
18
+
19
+ def tls(self):
20
+ e = self.settings
21
+ mode = e.get("sslmode", "verify-full")
22
+ if mode == "disable":
23
+ return False
24
+ if mode == "require":
25
+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
26
+ ctx.check_hostname = False
27
+ ctx.verify_mode = ssl.CERT_NONE
28
+ else:
29
+ ca = str(self.config.path.parent / e["sslrootcert"]) if e.get("sslrootcert") else None
30
+ ctx = ssl.create_default_context(cafile=ca)
31
+ ctx.check_hostname = mode == "verify-full"
32
+ if e.get("sslcert"):
33
+ ctx.load_cert_chain(
34
+ str(self.config.path.parent / e["sslcert"]), str(self.config.path.parent / e["sslkey"])
35
+ )
36
+ return ctx
37
+
38
+ @asynccontextmanager
39
+ async def connect(self, *, readonly=False, timeout=None):
40
+ e = self.settings
41
+ pw = password(e, self.config.path.parent, self.redactor)
42
+ settings = {
43
+ "application_name": "pg-logical-repl/" + self.config.migration_id,
44
+ "statement_timeout": str(int(1000 * self.config.get("limits", "query_timeout_seconds", 60))),
45
+ "lock_timeout": str(int(1000 * self.config.get("limits", "lock_timeout_seconds", 5))),
46
+ }
47
+ if readonly:
48
+ settings["default_transaction_read_only"] = "on"
49
+ try:
50
+ c = await asyncpg.connect(
51
+ host=e["host"].split(","),
52
+ port=e["port"],
53
+ database=e["database"],
54
+ user=e["user"],
55
+ password=pw,
56
+ ssl=self.tls(),
57
+ timeout=timeout
58
+ or e.get("connect_timeout_seconds", self.config.get("limits", "connect_timeout_seconds", 10)),
59
+ server_settings=settings,
60
+ target_session_attrs="primary",
61
+ )
62
+ except Exception as exc:
63
+ raise ConnectorError(
64
+ "DATABASE_UNAVAILABLE",
65
+ self.redactor.text(exc),
66
+ retryable=True,
67
+ sqlstate=getattr(exc, "sqlstate", None),
68
+ ) from None
69
+ try:
70
+ yield c
71
+ finally:
72
+ await c.close(timeout=5)
73
+
74
+ def libpq(self, *, replication=False):
75
+ e = self.config.endpoint(self.side, "replication" if replication else "admin")
76
+ pw = password(e, self.config.path.parent, self.redactor)
77
+ values = dict(
78
+ host=e["host"],
79
+ port=e["port"],
80
+ dbname=e["database"],
81
+ user=e["user"],
82
+ sslmode=e.get("sslmode", "verify-full"),
83
+ connect_timeout=e.get(
84
+ "connect_timeout_seconds", self.config.get("limits", "connect_timeout_seconds", 10)
85
+ ),
86
+ application_name=self.config.subscription if replication else "pg-logical-repl",
87
+ )
88
+ if e.get("hostaddr"):
89
+ values["hostaddr"] = e["hostaddr"]
90
+ if replication:
91
+ values.update(
92
+ password=pw,
93
+ keepalives=1,
94
+ keepalives_idle=30,
95
+ keepalives_interval=10,
96
+ keepalives_count=6,
97
+ options="-crow_security=off",
98
+ )
99
+ if e.get("sslrootcert_on_target"):
100
+ values["sslrootcert"] = e["sslrootcert_on_target"]
101
+ elif e.get("sslrootcert"):
102
+ values["sslrootcert"] = str(self.config.path.parent / e["sslrootcert"])
103
+ if not replication:
104
+ for key in ("sslcert", "sslkey"):
105
+ if e.get(key):
106
+ values[key] = str(self.config.path.parent / e[key])
107
+ return values, pw
108
+
109
+
110
+ async def identity(c):
111
+ version = int(await c.fetchval("SHOW server_version_num"))
112
+ locale = "datlocale" if version >= 170000 else "daticulocale"
113
+ row = dict(
114
+ await c.fetchrow(
115
+ """SELECT oid::int AS database_oid, datname AS database,
116
+ pg_encoding_to_char(encoding) AS encoding, datcollate, datctype, datlocprovider::text,
117
+ datcollversion, """
118
+ + locale
119
+ + " AS locale FROM pg_database WHERE datname=current_database()"
120
+ )
121
+ )
122
+ try:
123
+ row["system_identifier"] = str(await c.fetchval("SELECT system_identifier FROM pg_control_system()"))
124
+ row["timeline_id"] = await c.fetchval("SELECT timeline_id FROM pg_control_checkpoint()")
125
+ except asyncpg.InsufficientPrivilegeError:
126
+ row["system_identifier"] = None
127
+ row["timeline_id"] = None
128
+ row.update(
129
+ version=version, major=version // 10000, in_recovery=await c.fetchval("SELECT pg_is_in_recovery()")
130
+ )
131
+ return row