pg-logical-repl 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. pg_logical_repl/__init__.py +3 -0
  2. pg_logical_repl/__main__.py +6 -0
  3. pg_logical_repl/capabilities.py +136 -0
  4. pg_logical_repl/catalog.py +320 -0
  5. pg_logical_repl/cli.py +277 -0
  6. pg_logical_repl/component.py +66 -0
  7. pg_logical_repl/config.py +356 -0
  8. pg_logical_repl/coverage.py +117 -0
  9. pg_logical_repl/cutover.py +239 -0
  10. pg_logical_repl/db.py +131 -0
  11. pg_logical_repl/ddl.py +352 -0
  12. pg_logical_repl/ddl_queue.py +461 -0
  13. pg_logical_repl/emergency.py +296 -0
  14. pg_logical_repl/errors.py +44 -0
  15. pg_logical_repl/health.py +194 -0
  16. pg_logical_repl/integrity.py +185 -0
  17. pg_logical_repl/logging_config.py +45 -0
  18. pg_logical_repl/monitoring.py +277 -0
  19. pg_logical_repl/plans.py +167 -0
  20. pg_logical_repl/preflight.py +290 -0
  21. pg_logical_repl/process.py +150 -0
  22. pg_logical_repl/recovery.py +188 -0
  23. pg_logical_repl/resync.py +267 -0
  24. pg_logical_repl/roles.py +123 -0
  25. pg_logical_repl/schema.py +89 -0
  26. pg_logical_repl/security.py +117 -0
  27. pg_logical_repl/service.py +773 -0
  28. pg_logical_repl/sql/base.sql +67 -0
  29. pg_logical_repl/sql/capture.sql +215 -0
  30. pg_logical_repl/sql/ddl_support.sql +109 -0
  31. pg_logical_repl/sql/freeze.sql +117 -0
  32. pg_logical_repl/sql/replay.sql +48 -0
  33. pg_logical_repl/state.py +160 -0
  34. pg_logical_repl/validation.py +353 -0
  35. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/IMPLEMENTATION_STATUS.md +207 -0
  36. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/README.md +406 -0
  37. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/DDL_IMPLEMENTATION.md +170 -0
  38. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/RELEASING.md +92 -0
  39. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/TESTING.md +142 -0
  40. pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/examples/local.yaml +55 -0
  41. pg_logical_repl-0.2.0.dist-info/METADATA +425 -0
  42. pg_logical_repl-0.2.0.dist-info/RECORD +45 -0
  43. pg_logical_repl-0.2.0.dist-info/WHEEL +5 -0
  44. pg_logical_repl-0.2.0.dist-info/entry_points.txt +2 -0
  45. pg_logical_repl-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,296 @@
1
+ """Explicit source-offline detach and separately planned, ownership-checked cleanup."""
2
+
3
+ import json
4
+ from datetime import UTC, datetime, timedelta
5
+ from pathlib import Path
6
+
7
+ from . import __version__, integrity, plans, recovery, validation
8
+ from .catalog import fingerprint, inventory
9
+ from .config import digest
10
+ from .db import identity
11
+ from .errors import ConnectorError, OwnershipError, PreconditionError
12
+ from .security import identifier as qi
13
+ from .security import qualified as qn
14
+ from .state import now, write_private
15
+
16
+
17
+ def read_plan(path, expected_hash, kind, service):
18
+ try:
19
+ plan = json.loads(Path(path).read_text())
20
+ except (OSError, ValueError):
21
+ raise PreconditionError("PLAN_INVALID", "Cannot read emergency plan") from None
22
+ if (
23
+ digest(plan) != expected_hash
24
+ or plan["plan_kind"] != kind
25
+ or plan["migration_id"] != service.config.migration_id
26
+ ):
27
+ raise PreconditionError("PLAN_MISMATCH", "Emergency plan does not match the reviewed operation")
28
+ retry = service.store.read().get("active_plan") == expected_hash
29
+ if plan["config_hash"] != service.config.hash or (
30
+ not retry and datetime.fromisoformat(plan["expires_at"]) < datetime.now(UTC)
31
+ ):
32
+ raise PreconditionError("PLAN_EXPIRED", "Configuration changed or plan expired")
33
+ return plan
34
+
35
+
36
+ def is_offline(path):
37
+ try:
38
+ return json.loads(Path(path).read_text()).get("extra", {}).get("source_unavailable", False)
39
+ except (OSError, ValueError):
40
+ raise PreconditionError("PLAN_INVALID", "Cannot read plan") from None
41
+
42
+
43
+ async def require_unavailable(service):
44
+ try:
45
+ async with service.source.connect(readonly=True):
46
+ pass
47
+ except ConnectorError as exc:
48
+ if exc.code == "DATABASE_UNAVAILABLE":
49
+ return
50
+ raise
51
+ raise PreconditionError(
52
+ "SOURCE_AVAILABLE", "Use the ordinary online operation while source is accessible"
53
+ )
54
+
55
+
56
+ async def slot_snapshot(service, sc):
57
+ row = await sc.fetchrow(
58
+ """SELECT slot_name,slot_type,plugin,database,active,
59
+ restart_lsn::text,confirmed_flush_lsn::text FROM pg_replication_slots WHERE slot_name=$1""",
60
+ service.config.subscription,
61
+ )
62
+ return dict(row) if row else None
63
+
64
+
65
+ async def assert_endpoint(service, c, side):
66
+ expected = service.store.read().get(side + "_identity")
67
+ if not expected or await identity(c) != expected:
68
+ raise OwnershipError(side + " identity differs from the migration")
69
+ await service.owned(c)
70
+
71
+
72
+ async def create_plan(service, kind, *, out=None, extra=None):
73
+ details = dict(extra or {})
74
+ if kind not in {"abort", "cutover-commit", "cleanup"}:
75
+ raise PreconditionError("PLAN_KIND", "Offline mode supports abort and cutover-commit only")
76
+ if not isinstance(details.get("reason"), str) or not details["reason"].strip():
77
+ raise PreconditionError("REASON_REQUIRED", "Emergency operations require a reason")
78
+ state = service.store.read()
79
+ if not state.get("source_identity") or not state.get("target_identity"):
80
+ raise OwnershipError("Emergency cleanup requires recorded endpoint identities")
81
+ if kind == "cleanup":
82
+ if state["state"] not in {"DETACH_CLEANUP_PENDING", "ABORT_CLEANUP_PENDING"}:
83
+ raise PreconditionError("CLEANUP_STATE", "No deferred cleanup is pending")
84
+ details["source_unavailable"] = False
85
+ else:
86
+ await require_unavailable(service)
87
+ details["source_unavailable"] = True
88
+ allowed = (
89
+ {"CUTOVER_READY", "COMMITTING", "DETACH_CLEANUP_PENDING"}
90
+ if kind == "cutover-commit"
91
+ else {
92
+ "PREPARED",
93
+ "STARTING",
94
+ "SYNCING",
95
+ "CATCHING_UP",
96
+ "REPLICATING",
97
+ "BLOCKED",
98
+ "PAUSED",
99
+ "PAUSING",
100
+ "RESUMING",
101
+ "RESYNCING",
102
+ "ABORTING",
103
+ "ABORT_CLEANUP_PENDING",
104
+ }
105
+ )
106
+ if state["state"] not in allowed:
107
+ raise PreconditionError("EMERGENCY_STATE", "Offline operation is not valid in " + state["state"])
108
+ async with service.target.connect(readonly=True) as tc:
109
+ await assert_endpoint(service, tc, "target")
110
+ target = plans.snapshot(await inventory(tc, include_passwords=True), state["ownership_token"])
111
+ if kind == "cutover-commit":
112
+ await verify_final_target(service, tc)
113
+ source = dict(identity=state["source_identity"])
114
+ if kind == "cleanup":
115
+ async with service.source.connect(readonly=True) as sc:
116
+ await assert_endpoint(service, sc, "source")
117
+ source = plans.snapshot(await inventory(sc, include_passwords=True), state["ownership_token"])
118
+ details["slot"] = await slot_snapshot(service, sc)
119
+ if details["slot"] and details["slot"]["active"]:
120
+ raise PreconditionError("SLOT_ACTIVE", "Deferred slot is still active")
121
+ plan = dict(
122
+ schema_version="pg_logical_connector/plan-v2",
123
+ plan_kind=kind,
124
+ migration_id=service.config.migration_id,
125
+ tool_version=__version__,
126
+ config_hash=service.config.hash,
127
+ source=source,
128
+ target=target,
129
+ state=state["state"],
130
+ created_at=now(),
131
+ expires_at=(datetime.now(UTC) + timedelta(hours=1)).isoformat(),
132
+ extra=details,
133
+ actions=["Detach only the recorded target subscription; retain source cleanup obligations"]
134
+ if kind != "cleanup"
135
+ else [
136
+ "Recheck both identities, ownership receipts and the reviewed inactive slot; remove migration resources"
137
+ ],
138
+ source_revalidated=kind == "cleanup",
139
+ )
140
+ value = digest(plan)
141
+ dest = Path(out) if out else service.store.directory / "plans" / f"{kind}-{value[:16]}.json"
142
+ write_private(dest, json.dumps(plan, indent=2))
143
+ write_private(
144
+ dest.with_suffix(".md"), f"# {kind}\n\nHash: `{value}`\n\n" + "\n".join(plan["actions"]) + "\n"
145
+ )
146
+ return dict(plan=plan, plan_hash=value, path=str(dest))
147
+
148
+
149
+ async def verify_final_target(service, tc):
150
+ state = service.store.read()
151
+ try:
152
+ report = json.loads((service.store.directory / "reports/cutover.json").read_text())
153
+ evidence = state["write_fence"]
154
+ age = (datetime.now(UTC) - datetime.fromisoformat(evidence["created_at"])).total_seconds()
155
+ if not report["authoritative"] or not 0 <= age <= service.config.get(
156
+ "cutover", "max_validation_age_seconds", 21600
157
+ ):
158
+ raise ValueError
159
+ expected_schema = report["target_schema"]
160
+ except (OSError, KeyError, TypeError, ValueError):
161
+ raise PreconditionError(
162
+ "FINAL_EVIDENCE_REQUIRED", "Offline detach requires fresh finalized data and write-fence evidence"
163
+ ) from None
164
+ inv = await inventory(tc, include_passwords=True)
165
+ if fingerprint(inv) != expected_schema:
166
+ raise PreconditionError("TARGET_CHANGED", "Target schema changed after finalize")
167
+ executor = "lc_ddl_executor_" + service.config.migration_id
168
+ if (
169
+ integrity.role_view(inv, state["ownership_token"], executor) != state["role_baseline"]["target"]
170
+ or await integrity.resources(tc, executor) != state["resource_baseline"]["target"]
171
+ ):
172
+ raise PreconditionError("TARGET_CHANGED", "Target roles or installed resources changed")
173
+ relations = {qn(r["schema"], r["name"]): r for r in inv["relations"]}
174
+ for row in report["tables"]:
175
+ relation = relations[row["table"]]
176
+ async with validation.snapshot(tc):
177
+ actual = await validation.digest_range(
178
+ tc, row["table"], [c["attname"] for c in relation["columns"]], [], None, None, report["level"]
179
+ )
180
+ if actual != row["target"]:
181
+ raise PreconditionError("TARGET_CHANGED", "Target data changed after finalize")
182
+ for row in report["sequences"]:
183
+ if dict(await tc.fetchrow("SELECT last_value,is_called FROM " + row["sequence"])) != row["source"]:
184
+ raise PreconditionError("TARGET_CHANGED", "Target sequence changed after finalize")
185
+ if await validation.large_objects(tc) != report["large_objects"]:
186
+ raise PreconditionError("TARGET_CHANGED", "Target large objects changed after finalize")
187
+
188
+
189
+ async def execute(service, kind, path, plan_hash):
190
+ plan = read_plan(path, plan_hash, kind, service)
191
+ if not plan["extra"].get("source_unavailable"):
192
+ raise PreconditionError("PLAN_MISMATCH", "Explicit offline plan required")
193
+ await require_unavailable(service)
194
+ with service.store.lock():
195
+ state = service.store.read()
196
+ retry = state.get("active_plan") == plan_hash
197
+ async with service.target.connect() as tc:
198
+ await service.controller_lock(None, tc)
199
+ await assert_endpoint(service, tc, "target")
200
+ observed = plans.snapshot(await inventory(tc, include_passwords=True), state["ownership_token"])
201
+ if not retry and (observed != plan["target"] or state["state"] != plan["state"]):
202
+ raise PreconditionError("PLAN_DRIFT", "Target or migration changed after the emergency plan")
203
+ if kind == "cutover-commit":
204
+ await verify_final_target(service, tc)
205
+ sub = await tc.fetchrow(
206
+ "SELECT oid::int,subslotname,subpublications FROM pg_subscription WHERE subname=$1",
207
+ service.config.subscription,
208
+ )
209
+ if sub and (
210
+ sub["oid"] != state.get("subscription_oid")
211
+ or sub["subslotname"] not in {service.config.subscription, None}
212
+ or sub["subpublications"] != [service.config.publication]
213
+ ):
214
+ raise OwnershipError("Subscription differs from its creation receipt")
215
+ if sub and sub["subslotname"] is None and not retry:
216
+ raise OwnershipError("Subscription slot was detached outside this operation")
217
+ pending = "DETACH_CLEANUP_PENDING" if kind == "cutover-commit" else "ABORT_CLEANUP_PENDING"
218
+ await service.state(
219
+ pending,
220
+ tc=tc,
221
+ active_plan=plan_hash,
222
+ cleanup_pending=dict(
223
+ outcome="DETACHED" if kind == "cutover-commit" else "ABORTED",
224
+ reason=plan["extra"]["reason"],
225
+ source_revalidated=False,
226
+ ),
227
+ )
228
+ if sub:
229
+ await tc.execute("ALTER SUBSCRIPTION " + qi(service.config.subscription) + " DISABLE")
230
+ await recovery.wait_stopped(service, tc)
231
+ await tc.execute(
232
+ "ALTER SUBSCRIPTION " + qi(service.config.subscription) + " SET (slot_name=NONE)"
233
+ )
234
+ await tc.execute("DROP SUBSCRIPTION " + qi(service.config.subscription))
235
+ if kind == "abort" and state.get("resync"):
236
+ from .resync import restore_for_abort
237
+
238
+ await restore_for_abort(service, tc)
239
+ service.store.event(
240
+ "offline_detached", source_cleanup_pending=True, reason=plan["extra"]["reason"]
241
+ )
242
+ return await service.status()
243
+
244
+
245
+ async def cleanup(service, path, plan_hash):
246
+ plan = read_plan(path, plan_hash, "cleanup", service)
247
+ with service.store.lock():
248
+ state = service.store.read()
249
+ if state["state"] not in {"DETACH_CLEANUP_PENDING", "ABORT_CLEANUP_PENDING"}:
250
+ raise PreconditionError("CLEANUP_STATE", "No deferred cleanup is pending")
251
+ async with service.source.connect() as sc, service.target.connect() as tc:
252
+ await service.controller_lock(sc, tc)
253
+ retry = state.get("active_plan") == plan_hash
254
+ for side, c in [("source", sc), ("target", tc)]:
255
+ if await identity(c) != plan[side]["identity"]:
256
+ raise OwnershipError(side + " identity changed")
257
+ if not retry:
258
+ await service.owned(c)
259
+ if (
260
+ plans.snapshot(await inventory(c, include_passwords=True), state["ownership_token"])
261
+ != plan[side]
262
+ ):
263
+ raise PreconditionError("PLAN_DRIFT", side + " changed after the cleanup plan")
264
+ await c.fetchval("SELECT set_config('lc.ownership_token',$1,false)", state["ownership_token"])
265
+ if await tc.fetchval(
266
+ "SELECT EXISTS(SELECT FROM pg_subscription WHERE subname=$1)", service.config.subscription
267
+ ):
268
+ raise OwnershipError("Subscription must be detached before deferred cleanup")
269
+ slot = await slot_snapshot(service, sc)
270
+ if slot:
271
+ await service.owned(sc)
272
+ receipt = await sc.fetchval("SELECT details FROM lc.operations WHERE name='subscription'")
273
+ if not receipt or json.loads(receipt).get("oid") != state.get("subscription_oid"):
274
+ raise OwnershipError("Missing source receipt for the detached subscription")
275
+ if (
276
+ slot != plan["extra"]["slot"]
277
+ or slot["active"]
278
+ or slot["plugin"] != "pgoutput"
279
+ or slot["database"] != state["source_identity"]["database"]
280
+ ):
281
+ raise OwnershipError("Slot differs from the reviewed inactive slot")
282
+ # Never guess ownership of leftover tablesync or unrelated slots.
283
+ other = await sc.fetchval(
284
+ "SELECT EXISTS(SELECT FROM pg_replication_slots WHERE database=current_database() AND slot_name<>$1 AND NOT slot_name=ANY($2::text[]))",
285
+ service.config.subscription,
286
+ state.get("source_slots_before", []),
287
+ )
288
+ if other:
289
+ raise OwnershipError("Additional slots require ownership reconciliation before cleanup")
290
+ state["active_plan"] = plan_hash
291
+ service.store.save(state)
292
+ if slot:
293
+ await sc.fetchval("SELECT pg_drop_replication_slot($1)", service.config.subscription)
294
+ await service.cleanup(sc, tc)
295
+ await service.state(state["cleanup_pending"]["outcome"], active_plan=None, cleanup_pending=None)
296
+ return await service.status()
@@ -0,0 +1,44 @@
1
+ """Stable domain errors; never expose raw driver/child exceptions."""
2
+
3
+
4
+ class ConnectorError(Exception):
5
+ def __init__(
6
+ self,
7
+ code: str,
8
+ message: str,
9
+ *,
10
+ exit_code: int = 6,
11
+ retryable: bool = False,
12
+ recovery_hint: str = "",
13
+ sqlstate: str | None = None,
14
+ ):
15
+ super().__init__(message)
16
+ self.code = code
17
+ self.exit_code = exit_code
18
+ self.retryable = retryable
19
+ self.recovery_hint = recovery_hint
20
+ self.sqlstate = sqlstate
21
+
22
+ def as_dict(self):
23
+ return dict(
24
+ code=self.code,
25
+ message=str(self),
26
+ retryable=self.retryable,
27
+ recovery_hint=self.recovery_hint,
28
+ sqlstate=self.sqlstate,
29
+ )
30
+
31
+
32
+ class ConfigurationError(ConnectorError):
33
+ def __init__(self, message):
34
+ super().__init__("CONFIGURATION_ERROR", message, exit_code=2)
35
+
36
+
37
+ class PreconditionError(ConnectorError):
38
+ def __init__(self, code, message, **kwargs):
39
+ super().__init__(code, message, exit_code=3, **kwargs)
40
+
41
+
42
+ class OwnershipError(ConnectorError):
43
+ def __init__(self, message):
44
+ super().__init__("OWNERSHIP_ERROR", message, exit_code=8)
@@ -0,0 +1,194 @@
1
+ """Native replication health checks; never repair or discard WAL implicitly."""
2
+
3
+ import json
4
+ from datetime import UTC, datetime
5
+
6
+ from .db import identity
7
+ from .errors import PreconditionError
8
+ from .security import conninfo
9
+ from .state import now
10
+
11
+ ERROR_WINDOW_SECONDS = 300
12
+
13
+
14
+ async def require_healthy(service, sc, tc):
15
+ result = await observe(service, sc, tc)
16
+ if result["blockers"]:
17
+ raise PreconditionError("HEALTH_BLOCKED", ", ".join(result["blockers"]))
18
+ return result
19
+
20
+
21
+ async def observe(service, sc, tc, *, allow_disabled=False, ignore_history=False):
22
+ state = service.store.read()
23
+ blockers, advisories = [], []
24
+
25
+ def require(condition, code):
26
+ if not condition:
27
+ blockers.append(code)
28
+
29
+ for side, c in [("source", sc), ("target", tc)]:
30
+ require(await identity(c) == state[side + "_identity"], side.upper() + "_IDENTITY_CHANGED")
31
+ sub = await tc.fetchrow(
32
+ """SELECT oid::int,subenabled,subslotname,subpublications,subbinary,
33
+ substream::text,subsynccommit,subconninfo,subdisableonerr,
34
+ to_jsonb(pg_subscription)->>'subrunasowner' AS run_as_owner
35
+ FROM pg_subscription WHERE subname=$1""",
36
+ service.config.subscription,
37
+ )
38
+ require(sub is not None, "SUBSCRIPTION_MISSING")
39
+ errors = None
40
+ if sub:
41
+ require(sub["oid"] == state.get("subscription_oid"), "SUBSCRIPTION_IDENTITY_CHANGED")
42
+ values, _ = service.source.libpq(replication=True)
43
+ require(
44
+ (sub["subenabled"] or allow_disabled)
45
+ and sub["subslotname"] == service.config.subscription
46
+ and sub["subpublications"] == [service.config.publication]
47
+ and not sub["subbinary"]
48
+ # PG15 stores boolean; PG16+ stores a char to include parallel mode.
49
+ and sub["substream"] in {"t", "true"}
50
+ and sub["subsynccommit"] == "on"
51
+ and not sub["subdisableonerr"]
52
+ and (state["source_identity"]["major"] < 16 or sub["run_as_owner"] == "true")
53
+ and sub["subconninfo"] == conninfo(values),
54
+ "SUBSCRIPTION_CONFIGURATION_CHANGED",
55
+ )
56
+ errors = await tc.fetchrow(
57
+ "SELECT apply_error_count,sync_error_count,to_jsonb(s) AS all_stats FROM pg_stat_subscription_stats s WHERE subid=$1",
58
+ sub["oid"],
59
+ )
60
+ pub = await sc.fetchrow(
61
+ """SELECT oid::int,puballtables,pubinsert,pubupdate,pubdelete,pubtruncate,pubviaroot
62
+ FROM pg_publication WHERE pubname=$1""",
63
+ service.config.publication,
64
+ )
65
+ receipt_text = await sc.fetchval("SELECT details FROM lc.operations WHERE name='publication'")
66
+ receipt = json.loads(receipt_text) if receipt_text else {}
67
+ require(pub is not None, "PUBLICATION_MISSING")
68
+ if pub:
69
+ require(pub["oid"] == receipt.get("oid"), "PUBLICATION_IDENTITY_CHANGED")
70
+ require(
71
+ not pub["puballtables"]
72
+ and not pub["pubviaroot"]
73
+ and all(pub[k] for k in ("pubinsert", "pubupdate", "pubdelete", "pubtruncate")),
74
+ "PUBLICATION_CONFIGURATION_CHANGED",
75
+ )
76
+ tables = [
77
+ [r["schema"], r["name"]]
78
+ for r in await sc.fetch(
79
+ """SELECT n.nspname AS schema,c.relname AS name FROM pg_publication_rel r
80
+ JOIN pg_class c ON c.oid=r.prrelid JOIN pg_namespace n ON n.oid=c.relnamespace
81
+ WHERE r.prpubid=$1 ORDER BY 1,2""",
82
+ pub["oid"],
83
+ )
84
+ ]
85
+ require(tables == receipt.get("tables"), "PUBLICATION_MEMBERSHIP_CHANGED")
86
+ require(
87
+ not await sc.fetchval(
88
+ "SELECT EXISTS(SELECT FROM pg_publication_namespace WHERE pnpubid=$1)", pub["oid"]
89
+ ),
90
+ "PUBLICATION_SCHEMAS_CHANGED",
91
+ )
92
+ require(
93
+ not await sc.fetchval(
94
+ """SELECT EXISTS(SELECT FROM pg_publication_rel
95
+ WHERE prpubid=$1 AND (prattrs IS NOT NULL OR prqual IS NOT NULL))""",
96
+ pub["oid"],
97
+ ),
98
+ "PUBLICATION_FILTER_CHANGED",
99
+ )
100
+ slot = await sc.fetchrow(
101
+ """SELECT slot_name,slot_type,plugin,database,active,wal_status,
102
+ pg_wal_lsn_diff(pg_current_wal_lsn(),restart_lsn)::bigint AS retained_bytes,
103
+ pg_wal_lsn_diff(pg_current_wal_lsn(),confirmed_flush_lsn)::bigint AS lag_bytes,
104
+ confirmed_flush_lsn::text FROM pg_replication_slots WHERE slot_name=$1""",
105
+ service.config.subscription,
106
+ )
107
+ require(slot is not None, "SLOT_MISSING_RESYNC_REQUIRED")
108
+ if slot:
109
+ require(slot["wal_status"] != "lost", "SLOT_LOST_RESYNC_REQUIRED")
110
+ require(
111
+ slot["slot_type"] == "logical"
112
+ and slot["plugin"] == "pgoutput"
113
+ and slot["database"] == state["source_identity"]["database"],
114
+ "SLOT_CONFIGURATION_CHANGED",
115
+ )
116
+ threshold = service.config.get("limits", "alert_slot_retained_bytes", 8 * 1024**3)
117
+ if slot["retained_bytes"] is not None and slot["retained_bytes"] >= threshold:
118
+ advisories.append("SLOT_WAL_RETENTION_HIGH")
119
+ if slot["wal_status"] == "unreserved":
120
+ advisories.append("SLOT_WAL_AT_RISK")
121
+ health = dict(
122
+ checked_at=now(),
123
+ blockers=blockers,
124
+ advisories=advisories,
125
+ errors=dict(errors) if errors else {},
126
+ slot=dict(slot) if slot else None,
127
+ )
128
+ lag = await sc.fetchval(
129
+ """SELECT max(extract(epoch FROM replay_lag))::float8
130
+ FROM pg_stat_replication WHERE application_name=$1""",
131
+ service.config.subscription,
132
+ )
133
+ health["lag_seconds"] = lag
134
+ if slot and slot["lag_bytes"] == 0:
135
+ health["lag_seconds"] = 0.0
136
+ if health["lag_seconds"] is not None and health["lag_seconds"] > service.config.get(
137
+ "limits", "alert_lag_seconds", 60
138
+ ):
139
+ advisories.append("REPLICATION_LAG_HIGH")
140
+ for key, table, predicate, option in [
141
+ ("queue_age_seconds", "ddl_queue", "status NOT IN ('done','skipped')", "alert_queue_age_seconds"),
142
+ (
143
+ "refresh_age_seconds",
144
+ "table_sync",
145
+ "state NOT IN ('ready','removed')",
146
+ "alert_refresh_age_seconds",
147
+ ),
148
+ ]:
149
+ age = await tc.fetchval(
150
+ "SELECT COALESCE(max(extract(epoch FROM clock_timestamp()-created_at)),0)::float8 FROM lc."
151
+ + table
152
+ + " WHERE "
153
+ + predicate
154
+ )
155
+ health[key] = age
156
+ if age > service.config.get("limits", option, 300):
157
+ advisories.append(option.upper())
158
+ previous = {} if ignore_history else state.get("health", {})
159
+ application_sessions = await tc.fetchval("""SELECT count(*) FROM pg_stat_activity a JOIN pg_roles r ON r.oid=a.usesysid
160
+ WHERE a.datname=current_database() AND a.backend_type='client backend' AND NOT r.rolsuper
161
+ AND a.application_name NOT LIKE 'pg-logical-repl/%'""")
162
+ if application_sessions:
163
+ blockers.append("TARGET_APPLICATION_SESSIONS")
164
+ stats = json.loads(health["errors"].pop("all_stats", "{}"))
165
+ health["conflicts"] = {k: v for k, v in stats.items() if k.startswith("confl_")}
166
+ for key in ("confl_update_missing", "confl_delete_missing"):
167
+ if not ignore_history and health["conflicts"].get(key, 0) > previous.get("conflicts", {}).get(key, 0):
168
+ blockers.append(key.upper())
169
+ timestamp = datetime.now(UTC)
170
+ # Historical nonzero counters alone are not a failure. Block only on recurring
171
+ # errors spanning the window; a gap of a full window starts a new observation.
172
+ for counter in ("apply_error_count", "sync_error_count"):
173
+ value = health["errors"].get(counter, 0)
174
+ old_value = previous.get("errors", {}).get(counter, 0)
175
+ tracking = previous.get(counter + "_window")
176
+ if (
177
+ tracking
178
+ and (timestamp - datetime.fromisoformat(tracking["last"])).total_seconds() > ERROR_WINDOW_SECONDS
179
+ ):
180
+ tracking = None
181
+ if value < old_value:
182
+ tracking = None
183
+ if value > old_value:
184
+ tracking = tracking or dict(first=health["checked_at"], increments=0)
185
+ tracking.update(last=health["checked_at"], increments=tracking["increments"] + 1)
186
+ if (
187
+ tracking["increments"] > 1
188
+ and (timestamp - datetime.fromisoformat(tracking["first"])).total_seconds()
189
+ >= ERROR_WINDOW_SECONDS
190
+ ):
191
+ blockers.append("PERSISTENT_" + counter.upper())
192
+ if tracking:
193
+ health[counter + "_window"] = tracking
194
+ return health