pg-logical-repl 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pg_logical_repl/__init__.py +3 -0
- pg_logical_repl/__main__.py +6 -0
- pg_logical_repl/capabilities.py +136 -0
- pg_logical_repl/catalog.py +320 -0
- pg_logical_repl/cli.py +277 -0
- pg_logical_repl/component.py +66 -0
- pg_logical_repl/config.py +356 -0
- pg_logical_repl/coverage.py +117 -0
- pg_logical_repl/cutover.py +239 -0
- pg_logical_repl/db.py +131 -0
- pg_logical_repl/ddl.py +352 -0
- pg_logical_repl/ddl_queue.py +461 -0
- pg_logical_repl/emergency.py +296 -0
- pg_logical_repl/errors.py +44 -0
- pg_logical_repl/health.py +194 -0
- pg_logical_repl/integrity.py +185 -0
- pg_logical_repl/logging_config.py +45 -0
- pg_logical_repl/monitoring.py +277 -0
- pg_logical_repl/plans.py +167 -0
- pg_logical_repl/preflight.py +290 -0
- pg_logical_repl/process.py +150 -0
- pg_logical_repl/recovery.py +188 -0
- pg_logical_repl/resync.py +267 -0
- pg_logical_repl/roles.py +123 -0
- pg_logical_repl/schema.py +89 -0
- pg_logical_repl/security.py +117 -0
- pg_logical_repl/service.py +773 -0
- pg_logical_repl/sql/base.sql +67 -0
- pg_logical_repl/sql/capture.sql +215 -0
- pg_logical_repl/sql/ddl_support.sql +109 -0
- pg_logical_repl/sql/freeze.sql +117 -0
- pg_logical_repl/sql/replay.sql +48 -0
- pg_logical_repl/state.py +160 -0
- pg_logical_repl/validation.py +353 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/IMPLEMENTATION_STATUS.md +207 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/README.md +406 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/DDL_IMPLEMENTATION.md +170 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/RELEASING.md +92 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/TESTING.md +142 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/examples/local.yaml +55 -0
- pg_logical_repl-0.2.0.dist-info/METADATA +425 -0
- pg_logical_repl-0.2.0.dist-info/RECORD +45 -0
- pg_logical_repl-0.2.0.dist-info/WHEEL +5 -0
- pg_logical_repl-0.2.0.dist-info/entry_points.txt +2 -0
- pg_logical_repl-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"""Explicit source-offline detach and separately planned, ownership-checked cleanup."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from datetime import UTC, datetime, timedelta
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from . import __version__, integrity, plans, recovery, validation
|
|
8
|
+
from .catalog import fingerprint, inventory
|
|
9
|
+
from .config import digest
|
|
10
|
+
from .db import identity
|
|
11
|
+
from .errors import ConnectorError, OwnershipError, PreconditionError
|
|
12
|
+
from .security import identifier as qi
|
|
13
|
+
from .security import qualified as qn
|
|
14
|
+
from .state import now, write_private
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def read_plan(path, expected_hash, kind, service):
|
|
18
|
+
try:
|
|
19
|
+
plan = json.loads(Path(path).read_text())
|
|
20
|
+
except (OSError, ValueError):
|
|
21
|
+
raise PreconditionError("PLAN_INVALID", "Cannot read emergency plan") from None
|
|
22
|
+
if (
|
|
23
|
+
digest(plan) != expected_hash
|
|
24
|
+
or plan["plan_kind"] != kind
|
|
25
|
+
or plan["migration_id"] != service.config.migration_id
|
|
26
|
+
):
|
|
27
|
+
raise PreconditionError("PLAN_MISMATCH", "Emergency plan does not match the reviewed operation")
|
|
28
|
+
retry = service.store.read().get("active_plan") == expected_hash
|
|
29
|
+
if plan["config_hash"] != service.config.hash or (
|
|
30
|
+
not retry and datetime.fromisoformat(plan["expires_at"]) < datetime.now(UTC)
|
|
31
|
+
):
|
|
32
|
+
raise PreconditionError("PLAN_EXPIRED", "Configuration changed or plan expired")
|
|
33
|
+
return plan
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def is_offline(path):
|
|
37
|
+
try:
|
|
38
|
+
return json.loads(Path(path).read_text()).get("extra", {}).get("source_unavailable", False)
|
|
39
|
+
except (OSError, ValueError):
|
|
40
|
+
raise PreconditionError("PLAN_INVALID", "Cannot read plan") from None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
async def require_unavailable(service):
|
|
44
|
+
try:
|
|
45
|
+
async with service.source.connect(readonly=True):
|
|
46
|
+
pass
|
|
47
|
+
except ConnectorError as exc:
|
|
48
|
+
if exc.code == "DATABASE_UNAVAILABLE":
|
|
49
|
+
return
|
|
50
|
+
raise
|
|
51
|
+
raise PreconditionError(
|
|
52
|
+
"SOURCE_AVAILABLE", "Use the ordinary online operation while source is accessible"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
async def slot_snapshot(service, sc):
|
|
57
|
+
row = await sc.fetchrow(
|
|
58
|
+
"""SELECT slot_name,slot_type,plugin,database,active,
|
|
59
|
+
restart_lsn::text,confirmed_flush_lsn::text FROM pg_replication_slots WHERE slot_name=$1""",
|
|
60
|
+
service.config.subscription,
|
|
61
|
+
)
|
|
62
|
+
return dict(row) if row else None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def assert_endpoint(service, c, side):
|
|
66
|
+
expected = service.store.read().get(side + "_identity")
|
|
67
|
+
if not expected or await identity(c) != expected:
|
|
68
|
+
raise OwnershipError(side + " identity differs from the migration")
|
|
69
|
+
await service.owned(c)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
async def create_plan(service, kind, *, out=None, extra=None):
|
|
73
|
+
details = dict(extra or {})
|
|
74
|
+
if kind not in {"abort", "cutover-commit", "cleanup"}:
|
|
75
|
+
raise PreconditionError("PLAN_KIND", "Offline mode supports abort and cutover-commit only")
|
|
76
|
+
if not isinstance(details.get("reason"), str) or not details["reason"].strip():
|
|
77
|
+
raise PreconditionError("REASON_REQUIRED", "Emergency operations require a reason")
|
|
78
|
+
state = service.store.read()
|
|
79
|
+
if not state.get("source_identity") or not state.get("target_identity"):
|
|
80
|
+
raise OwnershipError("Emergency cleanup requires recorded endpoint identities")
|
|
81
|
+
if kind == "cleanup":
|
|
82
|
+
if state["state"] not in {"DETACH_CLEANUP_PENDING", "ABORT_CLEANUP_PENDING"}:
|
|
83
|
+
raise PreconditionError("CLEANUP_STATE", "No deferred cleanup is pending")
|
|
84
|
+
details["source_unavailable"] = False
|
|
85
|
+
else:
|
|
86
|
+
await require_unavailable(service)
|
|
87
|
+
details["source_unavailable"] = True
|
|
88
|
+
allowed = (
|
|
89
|
+
{"CUTOVER_READY", "COMMITTING", "DETACH_CLEANUP_PENDING"}
|
|
90
|
+
if kind == "cutover-commit"
|
|
91
|
+
else {
|
|
92
|
+
"PREPARED",
|
|
93
|
+
"STARTING",
|
|
94
|
+
"SYNCING",
|
|
95
|
+
"CATCHING_UP",
|
|
96
|
+
"REPLICATING",
|
|
97
|
+
"BLOCKED",
|
|
98
|
+
"PAUSED",
|
|
99
|
+
"PAUSING",
|
|
100
|
+
"RESUMING",
|
|
101
|
+
"RESYNCING",
|
|
102
|
+
"ABORTING",
|
|
103
|
+
"ABORT_CLEANUP_PENDING",
|
|
104
|
+
}
|
|
105
|
+
)
|
|
106
|
+
if state["state"] not in allowed:
|
|
107
|
+
raise PreconditionError("EMERGENCY_STATE", "Offline operation is not valid in " + state["state"])
|
|
108
|
+
async with service.target.connect(readonly=True) as tc:
|
|
109
|
+
await assert_endpoint(service, tc, "target")
|
|
110
|
+
target = plans.snapshot(await inventory(tc, include_passwords=True), state["ownership_token"])
|
|
111
|
+
if kind == "cutover-commit":
|
|
112
|
+
await verify_final_target(service, tc)
|
|
113
|
+
source = dict(identity=state["source_identity"])
|
|
114
|
+
if kind == "cleanup":
|
|
115
|
+
async with service.source.connect(readonly=True) as sc:
|
|
116
|
+
await assert_endpoint(service, sc, "source")
|
|
117
|
+
source = plans.snapshot(await inventory(sc, include_passwords=True), state["ownership_token"])
|
|
118
|
+
details["slot"] = await slot_snapshot(service, sc)
|
|
119
|
+
if details["slot"] and details["slot"]["active"]:
|
|
120
|
+
raise PreconditionError("SLOT_ACTIVE", "Deferred slot is still active")
|
|
121
|
+
plan = dict(
|
|
122
|
+
schema_version="pg_logical_connector/plan-v2",
|
|
123
|
+
plan_kind=kind,
|
|
124
|
+
migration_id=service.config.migration_id,
|
|
125
|
+
tool_version=__version__,
|
|
126
|
+
config_hash=service.config.hash,
|
|
127
|
+
source=source,
|
|
128
|
+
target=target,
|
|
129
|
+
state=state["state"],
|
|
130
|
+
created_at=now(),
|
|
131
|
+
expires_at=(datetime.now(UTC) + timedelta(hours=1)).isoformat(),
|
|
132
|
+
extra=details,
|
|
133
|
+
actions=["Detach only the recorded target subscription; retain source cleanup obligations"]
|
|
134
|
+
if kind != "cleanup"
|
|
135
|
+
else [
|
|
136
|
+
"Recheck both identities, ownership receipts and the reviewed inactive slot; remove migration resources"
|
|
137
|
+
],
|
|
138
|
+
source_revalidated=kind == "cleanup",
|
|
139
|
+
)
|
|
140
|
+
value = digest(plan)
|
|
141
|
+
dest = Path(out) if out else service.store.directory / "plans" / f"{kind}-{value[:16]}.json"
|
|
142
|
+
write_private(dest, json.dumps(plan, indent=2))
|
|
143
|
+
write_private(
|
|
144
|
+
dest.with_suffix(".md"), f"# {kind}\n\nHash: `{value}`\n\n" + "\n".join(plan["actions"]) + "\n"
|
|
145
|
+
)
|
|
146
|
+
return dict(plan=plan, plan_hash=value, path=str(dest))
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
async def verify_final_target(service, tc):
|
|
150
|
+
state = service.store.read()
|
|
151
|
+
try:
|
|
152
|
+
report = json.loads((service.store.directory / "reports/cutover.json").read_text())
|
|
153
|
+
evidence = state["write_fence"]
|
|
154
|
+
age = (datetime.now(UTC) - datetime.fromisoformat(evidence["created_at"])).total_seconds()
|
|
155
|
+
if not report["authoritative"] or not 0 <= age <= service.config.get(
|
|
156
|
+
"cutover", "max_validation_age_seconds", 21600
|
|
157
|
+
):
|
|
158
|
+
raise ValueError
|
|
159
|
+
expected_schema = report["target_schema"]
|
|
160
|
+
except (OSError, KeyError, TypeError, ValueError):
|
|
161
|
+
raise PreconditionError(
|
|
162
|
+
"FINAL_EVIDENCE_REQUIRED", "Offline detach requires fresh finalized data and write-fence evidence"
|
|
163
|
+
) from None
|
|
164
|
+
inv = await inventory(tc, include_passwords=True)
|
|
165
|
+
if fingerprint(inv) != expected_schema:
|
|
166
|
+
raise PreconditionError("TARGET_CHANGED", "Target schema changed after finalize")
|
|
167
|
+
executor = "lc_ddl_executor_" + service.config.migration_id
|
|
168
|
+
if (
|
|
169
|
+
integrity.role_view(inv, state["ownership_token"], executor) != state["role_baseline"]["target"]
|
|
170
|
+
or await integrity.resources(tc, executor) != state["resource_baseline"]["target"]
|
|
171
|
+
):
|
|
172
|
+
raise PreconditionError("TARGET_CHANGED", "Target roles or installed resources changed")
|
|
173
|
+
relations = {qn(r["schema"], r["name"]): r for r in inv["relations"]}
|
|
174
|
+
for row in report["tables"]:
|
|
175
|
+
relation = relations[row["table"]]
|
|
176
|
+
async with validation.snapshot(tc):
|
|
177
|
+
actual = await validation.digest_range(
|
|
178
|
+
tc, row["table"], [c["attname"] for c in relation["columns"]], [], None, None, report["level"]
|
|
179
|
+
)
|
|
180
|
+
if actual != row["target"]:
|
|
181
|
+
raise PreconditionError("TARGET_CHANGED", "Target data changed after finalize")
|
|
182
|
+
for row in report["sequences"]:
|
|
183
|
+
if dict(await tc.fetchrow("SELECT last_value,is_called FROM " + row["sequence"])) != row["source"]:
|
|
184
|
+
raise PreconditionError("TARGET_CHANGED", "Target sequence changed after finalize")
|
|
185
|
+
if await validation.large_objects(tc) != report["large_objects"]:
|
|
186
|
+
raise PreconditionError("TARGET_CHANGED", "Target large objects changed after finalize")
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
async def execute(service, kind, path, plan_hash):
|
|
190
|
+
plan = read_plan(path, plan_hash, kind, service)
|
|
191
|
+
if not plan["extra"].get("source_unavailable"):
|
|
192
|
+
raise PreconditionError("PLAN_MISMATCH", "Explicit offline plan required")
|
|
193
|
+
await require_unavailable(service)
|
|
194
|
+
with service.store.lock():
|
|
195
|
+
state = service.store.read()
|
|
196
|
+
retry = state.get("active_plan") == plan_hash
|
|
197
|
+
async with service.target.connect() as tc:
|
|
198
|
+
await service.controller_lock(None, tc)
|
|
199
|
+
await assert_endpoint(service, tc, "target")
|
|
200
|
+
observed = plans.snapshot(await inventory(tc, include_passwords=True), state["ownership_token"])
|
|
201
|
+
if not retry and (observed != plan["target"] or state["state"] != plan["state"]):
|
|
202
|
+
raise PreconditionError("PLAN_DRIFT", "Target or migration changed after the emergency plan")
|
|
203
|
+
if kind == "cutover-commit":
|
|
204
|
+
await verify_final_target(service, tc)
|
|
205
|
+
sub = await tc.fetchrow(
|
|
206
|
+
"SELECT oid::int,subslotname,subpublications FROM pg_subscription WHERE subname=$1",
|
|
207
|
+
service.config.subscription,
|
|
208
|
+
)
|
|
209
|
+
if sub and (
|
|
210
|
+
sub["oid"] != state.get("subscription_oid")
|
|
211
|
+
or sub["subslotname"] not in {service.config.subscription, None}
|
|
212
|
+
or sub["subpublications"] != [service.config.publication]
|
|
213
|
+
):
|
|
214
|
+
raise OwnershipError("Subscription differs from its creation receipt")
|
|
215
|
+
if sub and sub["subslotname"] is None and not retry:
|
|
216
|
+
raise OwnershipError("Subscription slot was detached outside this operation")
|
|
217
|
+
pending = "DETACH_CLEANUP_PENDING" if kind == "cutover-commit" else "ABORT_CLEANUP_PENDING"
|
|
218
|
+
await service.state(
|
|
219
|
+
pending,
|
|
220
|
+
tc=tc,
|
|
221
|
+
active_plan=plan_hash,
|
|
222
|
+
cleanup_pending=dict(
|
|
223
|
+
outcome="DETACHED" if kind == "cutover-commit" else "ABORTED",
|
|
224
|
+
reason=plan["extra"]["reason"],
|
|
225
|
+
source_revalidated=False,
|
|
226
|
+
),
|
|
227
|
+
)
|
|
228
|
+
if sub:
|
|
229
|
+
await tc.execute("ALTER SUBSCRIPTION " + qi(service.config.subscription) + " DISABLE")
|
|
230
|
+
await recovery.wait_stopped(service, tc)
|
|
231
|
+
await tc.execute(
|
|
232
|
+
"ALTER SUBSCRIPTION " + qi(service.config.subscription) + " SET (slot_name=NONE)"
|
|
233
|
+
)
|
|
234
|
+
await tc.execute("DROP SUBSCRIPTION " + qi(service.config.subscription))
|
|
235
|
+
if kind == "abort" and state.get("resync"):
|
|
236
|
+
from .resync import restore_for_abort
|
|
237
|
+
|
|
238
|
+
await restore_for_abort(service, tc)
|
|
239
|
+
service.store.event(
|
|
240
|
+
"offline_detached", source_cleanup_pending=True, reason=plan["extra"]["reason"]
|
|
241
|
+
)
|
|
242
|
+
return await service.status()
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
async def cleanup(service, path, plan_hash):
|
|
246
|
+
plan = read_plan(path, plan_hash, "cleanup", service)
|
|
247
|
+
with service.store.lock():
|
|
248
|
+
state = service.store.read()
|
|
249
|
+
if state["state"] not in {"DETACH_CLEANUP_PENDING", "ABORT_CLEANUP_PENDING"}:
|
|
250
|
+
raise PreconditionError("CLEANUP_STATE", "No deferred cleanup is pending")
|
|
251
|
+
async with service.source.connect() as sc, service.target.connect() as tc:
|
|
252
|
+
await service.controller_lock(sc, tc)
|
|
253
|
+
retry = state.get("active_plan") == plan_hash
|
|
254
|
+
for side, c in [("source", sc), ("target", tc)]:
|
|
255
|
+
if await identity(c) != plan[side]["identity"]:
|
|
256
|
+
raise OwnershipError(side + " identity changed")
|
|
257
|
+
if not retry:
|
|
258
|
+
await service.owned(c)
|
|
259
|
+
if (
|
|
260
|
+
plans.snapshot(await inventory(c, include_passwords=True), state["ownership_token"])
|
|
261
|
+
!= plan[side]
|
|
262
|
+
):
|
|
263
|
+
raise PreconditionError("PLAN_DRIFT", side + " changed after the cleanup plan")
|
|
264
|
+
await c.fetchval("SELECT set_config('lc.ownership_token',$1,false)", state["ownership_token"])
|
|
265
|
+
if await tc.fetchval(
|
|
266
|
+
"SELECT EXISTS(SELECT FROM pg_subscription WHERE subname=$1)", service.config.subscription
|
|
267
|
+
):
|
|
268
|
+
raise OwnershipError("Subscription must be detached before deferred cleanup")
|
|
269
|
+
slot = await slot_snapshot(service, sc)
|
|
270
|
+
if slot:
|
|
271
|
+
await service.owned(sc)
|
|
272
|
+
receipt = await sc.fetchval("SELECT details FROM lc.operations WHERE name='subscription'")
|
|
273
|
+
if not receipt or json.loads(receipt).get("oid") != state.get("subscription_oid"):
|
|
274
|
+
raise OwnershipError("Missing source receipt for the detached subscription")
|
|
275
|
+
if (
|
|
276
|
+
slot != plan["extra"]["slot"]
|
|
277
|
+
or slot["active"]
|
|
278
|
+
or slot["plugin"] != "pgoutput"
|
|
279
|
+
or slot["database"] != state["source_identity"]["database"]
|
|
280
|
+
):
|
|
281
|
+
raise OwnershipError("Slot differs from the reviewed inactive slot")
|
|
282
|
+
# Never guess ownership of leftover tablesync or unrelated slots.
|
|
283
|
+
other = await sc.fetchval(
|
|
284
|
+
"SELECT EXISTS(SELECT FROM pg_replication_slots WHERE database=current_database() AND slot_name<>$1 AND NOT slot_name=ANY($2::text[]))",
|
|
285
|
+
service.config.subscription,
|
|
286
|
+
state.get("source_slots_before", []),
|
|
287
|
+
)
|
|
288
|
+
if other:
|
|
289
|
+
raise OwnershipError("Additional slots require ownership reconciliation before cleanup")
|
|
290
|
+
state["active_plan"] = plan_hash
|
|
291
|
+
service.store.save(state)
|
|
292
|
+
if slot:
|
|
293
|
+
await sc.fetchval("SELECT pg_drop_replication_slot($1)", service.config.subscription)
|
|
294
|
+
await service.cleanup(sc, tc)
|
|
295
|
+
await service.state(state["cleanup_pending"]["outcome"], active_plan=None, cleanup_pending=None)
|
|
296
|
+
return await service.status()
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Stable domain errors; never expose raw driver/child exceptions."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ConnectorError(Exception):
|
|
5
|
+
def __init__(
|
|
6
|
+
self,
|
|
7
|
+
code: str,
|
|
8
|
+
message: str,
|
|
9
|
+
*,
|
|
10
|
+
exit_code: int = 6,
|
|
11
|
+
retryable: bool = False,
|
|
12
|
+
recovery_hint: str = "",
|
|
13
|
+
sqlstate: str | None = None,
|
|
14
|
+
):
|
|
15
|
+
super().__init__(message)
|
|
16
|
+
self.code = code
|
|
17
|
+
self.exit_code = exit_code
|
|
18
|
+
self.retryable = retryable
|
|
19
|
+
self.recovery_hint = recovery_hint
|
|
20
|
+
self.sqlstate = sqlstate
|
|
21
|
+
|
|
22
|
+
def as_dict(self):
|
|
23
|
+
return dict(
|
|
24
|
+
code=self.code,
|
|
25
|
+
message=str(self),
|
|
26
|
+
retryable=self.retryable,
|
|
27
|
+
recovery_hint=self.recovery_hint,
|
|
28
|
+
sqlstate=self.sqlstate,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ConfigurationError(ConnectorError):
|
|
33
|
+
def __init__(self, message):
|
|
34
|
+
super().__init__("CONFIGURATION_ERROR", message, exit_code=2)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class PreconditionError(ConnectorError):
|
|
38
|
+
def __init__(self, code, message, **kwargs):
|
|
39
|
+
super().__init__(code, message, exit_code=3, **kwargs)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class OwnershipError(ConnectorError):
|
|
43
|
+
def __init__(self, message):
|
|
44
|
+
super().__init__("OWNERSHIP_ERROR", message, exit_code=8)
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
"""Native replication health checks; never repair or discard WAL implicitly."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from datetime import UTC, datetime
|
|
5
|
+
|
|
6
|
+
from .db import identity
|
|
7
|
+
from .errors import PreconditionError
|
|
8
|
+
from .security import conninfo
|
|
9
|
+
from .state import now
|
|
10
|
+
|
|
11
|
+
ERROR_WINDOW_SECONDS = 300
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
async def require_healthy(service, sc, tc):
|
|
15
|
+
result = await observe(service, sc, tc)
|
|
16
|
+
if result["blockers"]:
|
|
17
|
+
raise PreconditionError("HEALTH_BLOCKED", ", ".join(result["blockers"]))
|
|
18
|
+
return result
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
async def observe(service, sc, tc, *, allow_disabled=False, ignore_history=False):
|
|
22
|
+
state = service.store.read()
|
|
23
|
+
blockers, advisories = [], []
|
|
24
|
+
|
|
25
|
+
def require(condition, code):
|
|
26
|
+
if not condition:
|
|
27
|
+
blockers.append(code)
|
|
28
|
+
|
|
29
|
+
for side, c in [("source", sc), ("target", tc)]:
|
|
30
|
+
require(await identity(c) == state[side + "_identity"], side.upper() + "_IDENTITY_CHANGED")
|
|
31
|
+
sub = await tc.fetchrow(
|
|
32
|
+
"""SELECT oid::int,subenabled,subslotname,subpublications,subbinary,
|
|
33
|
+
substream::text,subsynccommit,subconninfo,subdisableonerr,
|
|
34
|
+
to_jsonb(pg_subscription)->>'subrunasowner' AS run_as_owner
|
|
35
|
+
FROM pg_subscription WHERE subname=$1""",
|
|
36
|
+
service.config.subscription,
|
|
37
|
+
)
|
|
38
|
+
require(sub is not None, "SUBSCRIPTION_MISSING")
|
|
39
|
+
errors = None
|
|
40
|
+
if sub:
|
|
41
|
+
require(sub["oid"] == state.get("subscription_oid"), "SUBSCRIPTION_IDENTITY_CHANGED")
|
|
42
|
+
values, _ = service.source.libpq(replication=True)
|
|
43
|
+
require(
|
|
44
|
+
(sub["subenabled"] or allow_disabled)
|
|
45
|
+
and sub["subslotname"] == service.config.subscription
|
|
46
|
+
and sub["subpublications"] == [service.config.publication]
|
|
47
|
+
and not sub["subbinary"]
|
|
48
|
+
# PG15 stores boolean; PG16+ stores a char to include parallel mode.
|
|
49
|
+
and sub["substream"] in {"t", "true"}
|
|
50
|
+
and sub["subsynccommit"] == "on"
|
|
51
|
+
and not sub["subdisableonerr"]
|
|
52
|
+
and (state["source_identity"]["major"] < 16 or sub["run_as_owner"] == "true")
|
|
53
|
+
and sub["subconninfo"] == conninfo(values),
|
|
54
|
+
"SUBSCRIPTION_CONFIGURATION_CHANGED",
|
|
55
|
+
)
|
|
56
|
+
errors = await tc.fetchrow(
|
|
57
|
+
"SELECT apply_error_count,sync_error_count,to_jsonb(s) AS all_stats FROM pg_stat_subscription_stats s WHERE subid=$1",
|
|
58
|
+
sub["oid"],
|
|
59
|
+
)
|
|
60
|
+
pub = await sc.fetchrow(
|
|
61
|
+
"""SELECT oid::int,puballtables,pubinsert,pubupdate,pubdelete,pubtruncate,pubviaroot
|
|
62
|
+
FROM pg_publication WHERE pubname=$1""",
|
|
63
|
+
service.config.publication,
|
|
64
|
+
)
|
|
65
|
+
receipt_text = await sc.fetchval("SELECT details FROM lc.operations WHERE name='publication'")
|
|
66
|
+
receipt = json.loads(receipt_text) if receipt_text else {}
|
|
67
|
+
require(pub is not None, "PUBLICATION_MISSING")
|
|
68
|
+
if pub:
|
|
69
|
+
require(pub["oid"] == receipt.get("oid"), "PUBLICATION_IDENTITY_CHANGED")
|
|
70
|
+
require(
|
|
71
|
+
not pub["puballtables"]
|
|
72
|
+
and not pub["pubviaroot"]
|
|
73
|
+
and all(pub[k] for k in ("pubinsert", "pubupdate", "pubdelete", "pubtruncate")),
|
|
74
|
+
"PUBLICATION_CONFIGURATION_CHANGED",
|
|
75
|
+
)
|
|
76
|
+
tables = [
|
|
77
|
+
[r["schema"], r["name"]]
|
|
78
|
+
for r in await sc.fetch(
|
|
79
|
+
"""SELECT n.nspname AS schema,c.relname AS name FROM pg_publication_rel r
|
|
80
|
+
JOIN pg_class c ON c.oid=r.prrelid JOIN pg_namespace n ON n.oid=c.relnamespace
|
|
81
|
+
WHERE r.prpubid=$1 ORDER BY 1,2""",
|
|
82
|
+
pub["oid"],
|
|
83
|
+
)
|
|
84
|
+
]
|
|
85
|
+
require(tables == receipt.get("tables"), "PUBLICATION_MEMBERSHIP_CHANGED")
|
|
86
|
+
require(
|
|
87
|
+
not await sc.fetchval(
|
|
88
|
+
"SELECT EXISTS(SELECT FROM pg_publication_namespace WHERE pnpubid=$1)", pub["oid"]
|
|
89
|
+
),
|
|
90
|
+
"PUBLICATION_SCHEMAS_CHANGED",
|
|
91
|
+
)
|
|
92
|
+
require(
|
|
93
|
+
not await sc.fetchval(
|
|
94
|
+
"""SELECT EXISTS(SELECT FROM pg_publication_rel
|
|
95
|
+
WHERE prpubid=$1 AND (prattrs IS NOT NULL OR prqual IS NOT NULL))""",
|
|
96
|
+
pub["oid"],
|
|
97
|
+
),
|
|
98
|
+
"PUBLICATION_FILTER_CHANGED",
|
|
99
|
+
)
|
|
100
|
+
slot = await sc.fetchrow(
|
|
101
|
+
"""SELECT slot_name,slot_type,plugin,database,active,wal_status,
|
|
102
|
+
pg_wal_lsn_diff(pg_current_wal_lsn(),restart_lsn)::bigint AS retained_bytes,
|
|
103
|
+
pg_wal_lsn_diff(pg_current_wal_lsn(),confirmed_flush_lsn)::bigint AS lag_bytes,
|
|
104
|
+
confirmed_flush_lsn::text FROM pg_replication_slots WHERE slot_name=$1""",
|
|
105
|
+
service.config.subscription,
|
|
106
|
+
)
|
|
107
|
+
require(slot is not None, "SLOT_MISSING_RESYNC_REQUIRED")
|
|
108
|
+
if slot:
|
|
109
|
+
require(slot["wal_status"] != "lost", "SLOT_LOST_RESYNC_REQUIRED")
|
|
110
|
+
require(
|
|
111
|
+
slot["slot_type"] == "logical"
|
|
112
|
+
and slot["plugin"] == "pgoutput"
|
|
113
|
+
and slot["database"] == state["source_identity"]["database"],
|
|
114
|
+
"SLOT_CONFIGURATION_CHANGED",
|
|
115
|
+
)
|
|
116
|
+
threshold = service.config.get("limits", "alert_slot_retained_bytes", 8 * 1024**3)
|
|
117
|
+
if slot["retained_bytes"] is not None and slot["retained_bytes"] >= threshold:
|
|
118
|
+
advisories.append("SLOT_WAL_RETENTION_HIGH")
|
|
119
|
+
if slot["wal_status"] == "unreserved":
|
|
120
|
+
advisories.append("SLOT_WAL_AT_RISK")
|
|
121
|
+
health = dict(
|
|
122
|
+
checked_at=now(),
|
|
123
|
+
blockers=blockers,
|
|
124
|
+
advisories=advisories,
|
|
125
|
+
errors=dict(errors) if errors else {},
|
|
126
|
+
slot=dict(slot) if slot else None,
|
|
127
|
+
)
|
|
128
|
+
lag = await sc.fetchval(
|
|
129
|
+
"""SELECT max(extract(epoch FROM replay_lag))::float8
|
|
130
|
+
FROM pg_stat_replication WHERE application_name=$1""",
|
|
131
|
+
service.config.subscription,
|
|
132
|
+
)
|
|
133
|
+
health["lag_seconds"] = lag
|
|
134
|
+
if slot and slot["lag_bytes"] == 0:
|
|
135
|
+
health["lag_seconds"] = 0.0
|
|
136
|
+
if health["lag_seconds"] is not None and health["lag_seconds"] > service.config.get(
|
|
137
|
+
"limits", "alert_lag_seconds", 60
|
|
138
|
+
):
|
|
139
|
+
advisories.append("REPLICATION_LAG_HIGH")
|
|
140
|
+
for key, table, predicate, option in [
|
|
141
|
+
("queue_age_seconds", "ddl_queue", "status NOT IN ('done','skipped')", "alert_queue_age_seconds"),
|
|
142
|
+
(
|
|
143
|
+
"refresh_age_seconds",
|
|
144
|
+
"table_sync",
|
|
145
|
+
"state NOT IN ('ready','removed')",
|
|
146
|
+
"alert_refresh_age_seconds",
|
|
147
|
+
),
|
|
148
|
+
]:
|
|
149
|
+
age = await tc.fetchval(
|
|
150
|
+
"SELECT COALESCE(max(extract(epoch FROM clock_timestamp()-created_at)),0)::float8 FROM lc."
|
|
151
|
+
+ table
|
|
152
|
+
+ " WHERE "
|
|
153
|
+
+ predicate
|
|
154
|
+
)
|
|
155
|
+
health[key] = age
|
|
156
|
+
if age > service.config.get("limits", option, 300):
|
|
157
|
+
advisories.append(option.upper())
|
|
158
|
+
previous = {} if ignore_history else state.get("health", {})
|
|
159
|
+
application_sessions = await tc.fetchval("""SELECT count(*) FROM pg_stat_activity a JOIN pg_roles r ON r.oid=a.usesysid
|
|
160
|
+
WHERE a.datname=current_database() AND a.backend_type='client backend' AND NOT r.rolsuper
|
|
161
|
+
AND a.application_name NOT LIKE 'pg-logical-repl/%'""")
|
|
162
|
+
if application_sessions:
|
|
163
|
+
blockers.append("TARGET_APPLICATION_SESSIONS")
|
|
164
|
+
stats = json.loads(health["errors"].pop("all_stats", "{}"))
|
|
165
|
+
health["conflicts"] = {k: v for k, v in stats.items() if k.startswith("confl_")}
|
|
166
|
+
for key in ("confl_update_missing", "confl_delete_missing"):
|
|
167
|
+
if not ignore_history and health["conflicts"].get(key, 0) > previous.get("conflicts", {}).get(key, 0):
|
|
168
|
+
blockers.append(key.upper())
|
|
169
|
+
timestamp = datetime.now(UTC)
|
|
170
|
+
# Historical nonzero counters alone are not a failure. Block only on recurring
|
|
171
|
+
# errors spanning the window; a gap of a full window starts a new observation.
|
|
172
|
+
for counter in ("apply_error_count", "sync_error_count"):
|
|
173
|
+
value = health["errors"].get(counter, 0)
|
|
174
|
+
old_value = previous.get("errors", {}).get(counter, 0)
|
|
175
|
+
tracking = previous.get(counter + "_window")
|
|
176
|
+
if (
|
|
177
|
+
tracking
|
|
178
|
+
and (timestamp - datetime.fromisoformat(tracking["last"])).total_seconds() > ERROR_WINDOW_SECONDS
|
|
179
|
+
):
|
|
180
|
+
tracking = None
|
|
181
|
+
if value < old_value:
|
|
182
|
+
tracking = None
|
|
183
|
+
if value > old_value:
|
|
184
|
+
tracking = tracking or dict(first=health["checked_at"], increments=0)
|
|
185
|
+
tracking.update(last=health["checked_at"], increments=tracking["increments"] + 1)
|
|
186
|
+
if (
|
|
187
|
+
tracking["increments"] > 1
|
|
188
|
+
and (timestamp - datetime.fromisoformat(tracking["first"])).total_seconds()
|
|
189
|
+
>= ERROR_WINDOW_SECONDS
|
|
190
|
+
):
|
|
191
|
+
blockers.append("PERSISTENT_" + counter.upper())
|
|
192
|
+
if tracking:
|
|
193
|
+
health[counter + "_window"] = tracking
|
|
194
|
+
return health
|