pg-logical-repl 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pg_logical_repl/__init__.py +3 -0
- pg_logical_repl/__main__.py +6 -0
- pg_logical_repl/capabilities.py +136 -0
- pg_logical_repl/catalog.py +320 -0
- pg_logical_repl/cli.py +277 -0
- pg_logical_repl/component.py +66 -0
- pg_logical_repl/config.py +356 -0
- pg_logical_repl/coverage.py +117 -0
- pg_logical_repl/cutover.py +239 -0
- pg_logical_repl/db.py +131 -0
- pg_logical_repl/ddl.py +352 -0
- pg_logical_repl/ddl_queue.py +461 -0
- pg_logical_repl/emergency.py +296 -0
- pg_logical_repl/errors.py +44 -0
- pg_logical_repl/health.py +194 -0
- pg_logical_repl/integrity.py +185 -0
- pg_logical_repl/logging_config.py +45 -0
- pg_logical_repl/monitoring.py +277 -0
- pg_logical_repl/plans.py +167 -0
- pg_logical_repl/preflight.py +290 -0
- pg_logical_repl/process.py +150 -0
- pg_logical_repl/recovery.py +188 -0
- pg_logical_repl/resync.py +267 -0
- pg_logical_repl/roles.py +123 -0
- pg_logical_repl/schema.py +89 -0
- pg_logical_repl/security.py +117 -0
- pg_logical_repl/service.py +773 -0
- pg_logical_repl/sql/base.sql +67 -0
- pg_logical_repl/sql/capture.sql +215 -0
- pg_logical_repl/sql/ddl_support.sql +109 -0
- pg_logical_repl/sql/freeze.sql +117 -0
- pg_logical_repl/sql/replay.sql +48 -0
- pg_logical_repl/state.py +160 -0
- pg_logical_repl/validation.py +353 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/IMPLEMENTATION_STATUS.md +207 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/README.md +406 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/DDL_IMPLEMENTATION.md +170 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/RELEASING.md +92 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/docs/TESTING.md +142 -0
- pg_logical_repl-0.2.0.data/data/share/doc/pg-logical-repl/examples/local.yaml +55 -0
- pg_logical_repl-0.2.0.dist-info/METADATA +425 -0
- pg_logical_repl-0.2.0.dist-info/RECORD +45 -0
- pg_logical_repl-0.2.0.dist-info/WHEEL +5 -0
- pg_logical_repl-0.2.0.dist-info/entry_points.txt +2 -0
- pg_logical_repl-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
"""Strict, versioned YAML configuration with secret references only."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import hashlib
|
|
7
|
+
import re
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import rfc8785
|
|
12
|
+
import yaml
|
|
13
|
+
|
|
14
|
+
from .errors import ConfigurationError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class UniqueLoader(yaml.SafeLoader):
|
|
18
|
+
pass
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
# YAML 1.2 boolean spelling: replication settings on/off must stay strings.
|
|
22
|
+
UniqueLoader.yaml_implicit_resolvers = copy.deepcopy(yaml.SafeLoader.yaml_implicit_resolvers)
|
|
23
|
+
for first, resolvers in UniqueLoader.yaml_implicit_resolvers.items():
|
|
24
|
+
UniqueLoader.yaml_implicit_resolvers[first] = [
|
|
25
|
+
(tag, rx) for tag, rx in resolvers if tag != "tag:yaml.org,2002:bool"
|
|
26
|
+
]
|
|
27
|
+
UniqueLoader.add_implicit_resolver(
|
|
28
|
+
"tag:yaml.org,2002:bool", re.compile(r"^(true|false|True|False|TRUE|FALSE)$"), list("tTfF")
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def mapping(loader, node, deep=False):
|
|
33
|
+
result = {}
|
|
34
|
+
for key_node, value_node in node.value:
|
|
35
|
+
key = loader.construct_object(key_node, deep=deep)
|
|
36
|
+
if not isinstance(key, str):
|
|
37
|
+
raise ConfigurationError("Configuration keys must be strings")
|
|
38
|
+
if key in result:
|
|
39
|
+
raise ConfigurationError(f"Duplicate YAML key: {key}")
|
|
40
|
+
result[key] = loader.construct_object(value_node, deep=deep)
|
|
41
|
+
return result
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
UniqueLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, mapping)
|
|
45
|
+
|
|
46
|
+
ENDPOINT_KEYS = set(
|
|
47
|
+
"host hostaddr port database user password_env password_file passfile sslmode sslrootcert sslcert sslkey connect_timeout_seconds sslrootcert_on_target create".split()
|
|
48
|
+
)
|
|
49
|
+
SECTIONS = {
|
|
50
|
+
"source": {"admin", "replication"},
|
|
51
|
+
"target": {"admin"},
|
|
52
|
+
"client_tools": {"bin_dirs"},
|
|
53
|
+
"publication": {"mode", "include_schemas", "extension_tables"},
|
|
54
|
+
"subscription": {"streaming", "binary", "failover", "disable_on_error"},
|
|
55
|
+
"ddl": {
|
|
56
|
+
"mode",
|
|
57
|
+
"new_tables_without_pk",
|
|
58
|
+
"rewrite_max_table_bytes",
|
|
59
|
+
"set_not_null_max_table_bytes",
|
|
60
|
+
"drop_cascade",
|
|
61
|
+
"unknown",
|
|
62
|
+
"contact",
|
|
63
|
+
},
|
|
64
|
+
"policies": set(
|
|
65
|
+
"roles_scope roles_passwords existing_roles extensions replica_identity materialized_views unlogged_tables tablespaces target_access".split()
|
|
66
|
+
),
|
|
67
|
+
"limits": set(
|
|
68
|
+
"lock_timeout_seconds query_timeout_seconds connect_timeout_seconds catchup_lag_seconds catchup_stable_seconds alert_lag_seconds alert_slot_retained_bytes alert_refresh_age_seconds alert_queue_age_seconds operation_timeout_seconds".split()
|
|
69
|
+
),
|
|
70
|
+
"capacity": {"mode", "evidence_file", "max_age_seconds"},
|
|
71
|
+
"sequences": {"presync_interval_seconds"},
|
|
72
|
+
"validation": {
|
|
73
|
+
"online",
|
|
74
|
+
"chunk_rows",
|
|
75
|
+
"max_parallel",
|
|
76
|
+
"pause_between_chunks_ms",
|
|
77
|
+
"max_retries",
|
|
78
|
+
"cutover_level",
|
|
79
|
+
},
|
|
80
|
+
"cutover": {"share_locks", "analyze", "max_validation_age_seconds"},
|
|
81
|
+
"daemon": {
|
|
82
|
+
"poll_interval_seconds",
|
|
83
|
+
"drift_interval_seconds",
|
|
84
|
+
"max_parallel_index_builds",
|
|
85
|
+
"queue_max_attempts",
|
|
86
|
+
"metrics_textfile",
|
|
87
|
+
},
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def canonical(value) -> bytes:
|
|
92
|
+
try:
|
|
93
|
+
return rfc8785.dumps(value)
|
|
94
|
+
except (ValueError, TypeError) as exc:
|
|
95
|
+
raise ConfigurationError("Value is not canonical JSON: " + str(exc)) from None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def digest(value) -> str:
|
|
99
|
+
return hashlib.sha256(canonical(value)).hexdigest()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def keys(value, allowed, location):
|
|
103
|
+
if not isinstance(value, dict):
|
|
104
|
+
raise ConfigurationError(f"{location} must be a mapping")
|
|
105
|
+
unknown = set(value) - allowed
|
|
106
|
+
if unknown:
|
|
107
|
+
raise ConfigurationError(f"Unknown fields in {location}: {', '.join(sorted(unknown))}")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@dataclass(frozen=True)
|
|
111
|
+
class Config:
|
|
112
|
+
path: Path
|
|
113
|
+
raw: dict
|
|
114
|
+
|
|
115
|
+
@property
|
|
116
|
+
def migration_id(self):
|
|
117
|
+
return self.raw["migration_id"]
|
|
118
|
+
|
|
119
|
+
@property
|
|
120
|
+
def state_dir(self):
|
|
121
|
+
return (self.path.parent / self.raw["state_dir"]).absolute()
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def hash(self):
|
|
125
|
+
return digest(self.raw)
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def control_hash(self):
|
|
129
|
+
# Scheduling and validation limits may be tuned between daemon ticks.
|
|
130
|
+
# Connection, publication and server-installed DDL policies may not.
|
|
131
|
+
value = {
|
|
132
|
+
k: v
|
|
133
|
+
for k, v in self.raw.items()
|
|
134
|
+
if k not in {"daemon", "limits", "validation", "sequences", "cutover"}
|
|
135
|
+
}
|
|
136
|
+
value["server_lock_timeout"] = self.get("limits", "lock_timeout_seconds", 5)
|
|
137
|
+
return digest(value)
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def ddl_mode(self):
|
|
141
|
+
return self.get("ddl", "mode", "frozen")
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def publication(self):
|
|
145
|
+
return "lc_pub_" + self.migration_id
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def subscription(self):
|
|
149
|
+
return "lc_sub_" + self.migration_id
|
|
150
|
+
|
|
151
|
+
def get(self, section, key, default=None):
|
|
152
|
+
return self.raw.get(section, {}).get(key, default)
|
|
153
|
+
|
|
154
|
+
def endpoint(self, side: str, purpose: str = "admin") -> dict:
|
|
155
|
+
base = dict(self.raw[side]["admin"])
|
|
156
|
+
if purpose == "replication":
|
|
157
|
+
base.update(self.raw[side].get("replication", {}))
|
|
158
|
+
if any(
|
|
159
|
+
k in self.raw[side].get("replication", {})
|
|
160
|
+
for k in ("password_env", "password_file", "passfile")
|
|
161
|
+
):
|
|
162
|
+
for k in ("password_env", "password_file", "passfile"):
|
|
163
|
+
if k not in self.raw[side]["replication"]:
|
|
164
|
+
base.pop(k, None)
|
|
165
|
+
base.setdefault("port", 5432)
|
|
166
|
+
base.setdefault("sslmode", "verify-full")
|
|
167
|
+
return base
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def load(path: str | Path) -> Config:
|
|
171
|
+
path = Path(path).expanduser().resolve()
|
|
172
|
+
try:
|
|
173
|
+
raw = yaml.load(path.read_text(), Loader=UniqueLoader)
|
|
174
|
+
except (OSError, yaml.YAMLError) as exc:
|
|
175
|
+
raise ConfigurationError(f"Cannot read YAML configuration ({type(exc).__name__})") from None
|
|
176
|
+
keys(raw, set(SECTIONS) | {"schema_version", "migration_id", "mode", "state_dir", "waivers"}, "config")
|
|
177
|
+
if raw.get("schema_version") != "pg_logical_connector/config-v2":
|
|
178
|
+
raise ConfigurationError("Expected schema_version pg_logical_connector/config-v2")
|
|
179
|
+
if not isinstance(raw.get("migration_id"), str) or not re.fullmatch(
|
|
180
|
+
r"[a-z0-9_]{3,40}", raw["migration_id"]
|
|
181
|
+
):
|
|
182
|
+
raise ConfigurationError("migration_id must match [a-z0-9_]{3,40}")
|
|
183
|
+
if not isinstance(raw.get("state_dir"), str) or not raw["state_dir"]:
|
|
184
|
+
raise ConfigurationError("state_dir is required")
|
|
185
|
+
if raw.get("mode", "migration") not in {"migration", "replica"}:
|
|
186
|
+
raise ConfigurationError("mode must be migration or replica")
|
|
187
|
+
for section, allowed in SECTIONS.items():
|
|
188
|
+
if section in raw:
|
|
189
|
+
keys(raw[section], allowed, section)
|
|
190
|
+
for side in ("source", "target"):
|
|
191
|
+
if not isinstance(raw.get(side, {}).get("admin"), dict):
|
|
192
|
+
raise ConfigurationError(f"{side}.admin is required")
|
|
193
|
+
for purpose, endpoint in raw[side].items():
|
|
194
|
+
keys(endpoint, ENDPOINT_KEYS, side + "." + purpose)
|
|
195
|
+
for key, value in endpoint.items():
|
|
196
|
+
if key == "create":
|
|
197
|
+
if type(value) is not bool:
|
|
198
|
+
raise ConfigurationError("create must be a boolean")
|
|
199
|
+
elif key in {"port", "connect_timeout_seconds"}:
|
|
200
|
+
if type(value) is not int or value <= 0:
|
|
201
|
+
raise ConfigurationError(key + " must be a positive integer")
|
|
202
|
+
elif not isinstance(value, str) or not value or "\x00" in value:
|
|
203
|
+
raise ConfigurationError(side + "." + purpose + "." + key + " must be a non-empty string")
|
|
204
|
+
if purpose == "admin":
|
|
205
|
+
for key in ("host", "database", "user"):
|
|
206
|
+
if not isinstance(endpoint.get(key), str) or not endpoint[key]:
|
|
207
|
+
raise ConfigurationError(f"{side}.admin.{key} is required")
|
|
208
|
+
if "port" in endpoint and (
|
|
209
|
+
type(endpoint["port"]) is not int or not 1 <= endpoint["port"] <= 65535
|
|
210
|
+
):
|
|
211
|
+
raise ConfigurationError("port must be an integer from 1 to 65535")
|
|
212
|
+
if endpoint.get("sslmode", "verify-full") not in {
|
|
213
|
+
"verify-full",
|
|
214
|
+
"verify-ca",
|
|
215
|
+
"require",
|
|
216
|
+
"disable",
|
|
217
|
+
}:
|
|
218
|
+
raise ConfigurationError("Unsupported sslmode")
|
|
219
|
+
if sum(bool(endpoint.get(k)) for k in ("password_env", "password_file", "passfile")) > 1:
|
|
220
|
+
raise ConfigurationError("Only one credential reference is allowed per endpoint")
|
|
221
|
+
enums = {
|
|
222
|
+
("ddl", "mode"): {"frozen", "captured", "managed_only"},
|
|
223
|
+
("publication", "mode"): {"explicit_tables", "all_tables"},
|
|
224
|
+
("policies", "roles_scope"): {"all", "database_related"},
|
|
225
|
+
("policies", "roles_passwords"): {"required", "waiver"},
|
|
226
|
+
("policies", "replica_identity"): {"fail", "full_if_needed"},
|
|
227
|
+
("policies", "existing_roles"): {"fail_on_difference", "reconcile_existing"},
|
|
228
|
+
("validation", "cutover_level"): {"counts", "checksums", "full"},
|
|
229
|
+
}
|
|
230
|
+
for (section, key), allowed in enums.items():
|
|
231
|
+
if key in raw.get(section, {}) and raw[section][key] not in allowed:
|
|
232
|
+
raise ConfigurationError(f"Unsupported {section}.{key}")
|
|
233
|
+
types = {
|
|
234
|
+
"limits": {key: int for key in SECTIONS["limits"]},
|
|
235
|
+
"capacity": {"mode": str, "evidence_file": str, "max_age_seconds": int},
|
|
236
|
+
"sequences": {"presync_interval_seconds": int},
|
|
237
|
+
"validation": {
|
|
238
|
+
"online": bool,
|
|
239
|
+
"chunk_rows": int,
|
|
240
|
+
"max_parallel": int,
|
|
241
|
+
"pause_between_chunks_ms": int,
|
|
242
|
+
"max_retries": int,
|
|
243
|
+
"cutover_level": str,
|
|
244
|
+
},
|
|
245
|
+
"cutover": {"share_locks": bool, "analyze": str, "max_validation_age_seconds": int},
|
|
246
|
+
"daemon": {key: (str if key == "metrics_textfile" else int) for key in SECTIONS["daemon"]},
|
|
247
|
+
"ddl": {key: (int if key.endswith("_bytes") else str) for key in SECTIONS["ddl"]},
|
|
248
|
+
"policies": {key: str for key in SECTIONS["policies"]},
|
|
249
|
+
"publication": {"mode": str, "include_schemas": list, "extension_tables": str},
|
|
250
|
+
}
|
|
251
|
+
for section, fields in types.items():
|
|
252
|
+
for key, value in raw.get(section, {}).items():
|
|
253
|
+
if type(value) is not fields[key] or (fields[key] is int and value < 0):
|
|
254
|
+
raise ConfigurationError(f"Invalid type or range for {section}.{key}")
|
|
255
|
+
for key in ("chunk_rows", "max_parallel"):
|
|
256
|
+
if key in raw.get("validation", {}) and raw["validation"][key] == 0:
|
|
257
|
+
raise ConfigurationError("validation." + key + " must be positive")
|
|
258
|
+
for key, value in raw.get("subscription", {}).items():
|
|
259
|
+
allowed = {
|
|
260
|
+
"streaming": {"auto", "on", "off", "parallel"},
|
|
261
|
+
"binary": {"auto", True, False},
|
|
262
|
+
"failover": {"auto", True, False},
|
|
263
|
+
"disable_on_error": {True, False},
|
|
264
|
+
}[key]
|
|
265
|
+
if type(value) not in {str, bool} or value not in allowed:
|
|
266
|
+
raise ConfigurationError("Invalid subscription." + key)
|
|
267
|
+
if "client_tools" in raw:
|
|
268
|
+
directories = raw["client_tools"].get("bin_dirs", {})
|
|
269
|
+
keys(directories, {"15", "16", "17", "18"}, "client_tools.bin_dirs")
|
|
270
|
+
if any(not isinstance(v, str) or not v for v in directories.values()):
|
|
271
|
+
raise ConfigurationError("Client tool paths must be strings")
|
|
272
|
+
if any(not isinstance(v, str) or not v for v in raw.get("publication", {}).get("include_schemas", [])):
|
|
273
|
+
raise ConfigurationError("include_schemas must contain non-empty strings")
|
|
274
|
+
if not isinstance(raw.get("waivers", []), list):
|
|
275
|
+
raise ConfigurationError("waivers must be a list")
|
|
276
|
+
for w in raw.get("waivers", []):
|
|
277
|
+
keys(w, {"object", "decision", "reason", "approved_by", "expires_at"}, "waiver")
|
|
278
|
+
if not all(
|
|
279
|
+
isinstance(w.get(k), str) and w[k]
|
|
280
|
+
for k in ("object", "decision", "reason", "approved_by", "expires_at")
|
|
281
|
+
):
|
|
282
|
+
raise ConfigurationError("Waivers require object/decision/reason/approved_by/expires_at")
|
|
283
|
+
ddl = raw.get("ddl", {})
|
|
284
|
+
if ddl.get("new_tables_without_pk", "reject") not in {"reject", "replica_identity_full"}:
|
|
285
|
+
raise ConfigurationError("ddl.new_tables_without_pk must be reject or replica_identity_full")
|
|
286
|
+
for key in ("rewrite_max_table_bytes", "set_not_null_max_table_bytes"):
|
|
287
|
+
if key in ddl and (type(ddl[key]) is not int or ddl[key] < 0):
|
|
288
|
+
raise ConfigurationError("ddl." + key + " must be a non-negative integer")
|
|
289
|
+
for key in ("queue_max_attempts", "max_parallel_index_builds"):
|
|
290
|
+
value = raw.get("daemon", {}).get(key, 1)
|
|
291
|
+
if type(value) is not int or value < 1:
|
|
292
|
+
raise ConfigurationError("daemon." + key + " must be a positive integer")
|
|
293
|
+
validate_profile(raw)
|
|
294
|
+
return Config(path, raw)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
# Accepted configuration must describe executable behavior, including offline validate.
|
|
298
|
+
def validate_profile(raw):
|
|
299
|
+
fixed = {
|
|
300
|
+
("publication", "mode"): "explicit_tables",
|
|
301
|
+
("publication", "include_schemas"): [],
|
|
302
|
+
("policies", "roles_scope"): "all",
|
|
303
|
+
("policies", "roles_passwords"): "required",
|
|
304
|
+
("policies", "existing_roles"): "fail_on_difference",
|
|
305
|
+
("policies", "extensions"): "exact_version",
|
|
306
|
+
("policies", "materialized_views"): "refresh_at_cutover",
|
|
307
|
+
("policies", "unlogged_tables"): "copy_at_cutover",
|
|
308
|
+
("policies", "tablespaces"): "preserve",
|
|
309
|
+
("policies", "target_access"): "closed",
|
|
310
|
+
("capacity", "mode"): "external_measurements",
|
|
311
|
+
("ddl", "drop_cascade"): "reject",
|
|
312
|
+
("ddl", "unknown"): "reject",
|
|
313
|
+
("daemon", "max_parallel_index_builds"): 1,
|
|
314
|
+
}
|
|
315
|
+
for (section, key), value in fixed.items():
|
|
316
|
+
if raw.get(section, {}).get(key, value) != value:
|
|
317
|
+
raise ConfigurationError(f"Implemented profile requires {section}.{key}={value!r}")
|
|
318
|
+
for section, key in [("publication", "extension_tables"), ("ddl", "contact")]:
|
|
319
|
+
if key in raw.get(section, {}):
|
|
320
|
+
raise ConfigurationError(f"{section}.{key} is not implemented; omit it")
|
|
321
|
+
if raw.get("mode", "migration") != "migration":
|
|
322
|
+
raise ConfigurationError("mode=replica is not implemented")
|
|
323
|
+
if raw.get("waivers"):
|
|
324
|
+
raise ConfigurationError("Waivers are not implemented")
|
|
325
|
+
for side in ("source", "target"):
|
|
326
|
+
if "hostaddr" in raw[side]["admin"]:
|
|
327
|
+
raise ConfigurationError("admin.hostaddr is not supported by the asyncpg transport; use host")
|
|
328
|
+
if bool(raw[side]["admin"].get("sslcert")) != bool(raw[side]["admin"].get("sslkey")):
|
|
329
|
+
raise ConfigurationError("Administrative sslcert and sslkey must be supplied together")
|
|
330
|
+
if "sslrootcert_on_target" in raw["target"]["admin"]:
|
|
331
|
+
raise ConfigurationError("target.admin requires sslrootcert, not sslrootcert_on_target")
|
|
332
|
+
if "sslrootcert" in raw["source"].get("replication", {}):
|
|
333
|
+
raise ConfigurationError("source.replication requires sslrootcert_on_target, not sslrootcert")
|
|
334
|
+
if any(k in raw["source"].get("replication", {}) for k in ("sslcert", "sslkey")):
|
|
335
|
+
raise ConfigurationError("Replication client certificates on target are not implemented")
|
|
336
|
+
for side in ("source", "target"):
|
|
337
|
+
for endpoint in raw[side].values():
|
|
338
|
+
if endpoint.get("create"):
|
|
339
|
+
raise ConfigurationError("Automatic account creation is not implemented")
|
|
340
|
+
for key, allowed in {
|
|
341
|
+
"streaming": ("auto", "on"),
|
|
342
|
+
"binary": ("auto", False),
|
|
343
|
+
"failover": ("auto", False),
|
|
344
|
+
"disable_on_error": (False,),
|
|
345
|
+
}.items():
|
|
346
|
+
if key in raw.get("subscription", {}) and raw["subscription"][key] not in allowed:
|
|
347
|
+
raise ConfigurationError(f"subscription.{key} is outside the implemented profile")
|
|
348
|
+
if raw.get("cutover", {}).get("analyze", "never") not in {"never", "all"}:
|
|
349
|
+
raise ConfigurationError("cutover.analyze must be never or all")
|
|
350
|
+
for section, key in [
|
|
351
|
+
("daemon", "poll_interval_seconds"),
|
|
352
|
+
("limits", "operation_timeout_seconds"),
|
|
353
|
+
("limits", "connect_timeout_seconds"),
|
|
354
|
+
]:
|
|
355
|
+
if raw.get(section, {}).get(key, 1) <= 0:
|
|
356
|
+
raise ConfigurationError(f"{section}.{key} must be positive")
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Conservative catalog coverage for the qualified local profile.
|
|
2
|
+
|
|
3
|
+
Return object identities, never foreign-server options, mappings or label values:
|
|
4
|
+
those may contain credentials. Extension members are handled by extension policy.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
async def unsupported_objects(c):
|
|
9
|
+
# All identifiers below are internal constants, never user-provided SQL.
|
|
10
|
+
catalogs = {
|
|
11
|
+
"pg_operator": ("oprnamespace", "true"),
|
|
12
|
+
"pg_opclass": ("opcnamespace", "true"),
|
|
13
|
+
"pg_opfamily": ("opfnamespace", "true"),
|
|
14
|
+
"pg_collation": ("collnamespace", "true"),
|
|
15
|
+
"pg_conversion": ("connamespace", "true"),
|
|
16
|
+
"pg_ts_config": ("cfgnamespace", "true"),
|
|
17
|
+
"pg_ts_dict": ("dictnamespace", "true"),
|
|
18
|
+
"pg_ts_parser": ("prsnamespace", "true"),
|
|
19
|
+
"pg_ts_template": ("tmplnamespace", "true"),
|
|
20
|
+
"pg_statistic_ext": ("stxnamespace", "true"),
|
|
21
|
+
"pg_proc": ("pronamespace", "o.prokind IN ('a','w')"),
|
|
22
|
+
"pg_type": (
|
|
23
|
+
"typnamespace",
|
|
24
|
+
"o.typelem=0 AND o.typtype IN ('b','r','m','c') AND "
|
|
25
|
+
"(o.typrelid=0 OR EXISTS(SELECT FROM pg_class cr WHERE cr.oid=o.typrelid AND cr.relkind='c'))",
|
|
26
|
+
),
|
|
27
|
+
"pg_foreign_data_wrapper": (None, "true"),
|
|
28
|
+
"pg_foreign_server": (None, "true"),
|
|
29
|
+
"pg_user_mapping": (None, "true"),
|
|
30
|
+
"pg_transform": (None, "true"),
|
|
31
|
+
# User casts have non-bootstrap OIDs, including casts between built-in types.
|
|
32
|
+
"pg_cast": (None, "o.oid>=16384"),
|
|
33
|
+
"pg_language": (None, "o.lanname NOT IN ('internal','c','sql','plpgsql')"),
|
|
34
|
+
}
|
|
35
|
+
queries = []
|
|
36
|
+
for catalog, (namespace, predicate) in catalogs.items():
|
|
37
|
+
query = (
|
|
38
|
+
f"SELECT '{catalog}' AS category, "
|
|
39
|
+
f"(pg_identify_object('{catalog}'::regclass,o.oid,0)).identity AS identity "
|
|
40
|
+
f"FROM {catalog} o "
|
|
41
|
+
)
|
|
42
|
+
if namespace:
|
|
43
|
+
query += f"JOIN pg_namespace n ON n.oid=o.{namespace} "
|
|
44
|
+
predicate += " AND n.nspname !~ '^pg_' AND n.nspname NOT IN ('information_schema','lc')"
|
|
45
|
+
query += (
|
|
46
|
+
f"WHERE ({predicate}) AND NOT EXISTS(SELECT FROM pg_depend d "
|
|
47
|
+
f"WHERE d.classid='{catalog}'::regclass AND d.objid=o.oid AND d.deptype='e')"
|
|
48
|
+
)
|
|
49
|
+
queries.append(query)
|
|
50
|
+
queries.extend(
|
|
51
|
+
[
|
|
52
|
+
"""SELECT 'row_security', format('%I.%I',n.nspname,r.relname)
|
|
53
|
+
FROM pg_class r JOIN pg_namespace n ON n.oid=r.relnamespace
|
|
54
|
+
WHERE (r.relrowsecurity OR r.relforcerowsecurity) AND n.nspname !~ '^pg_'
|
|
55
|
+
AND n.nspname NOT IN ('lc','information_schema')""",
|
|
56
|
+
"""SELECT 'pg_policy',(pg_identify_object('pg_policy'::regclass,p.oid,0)).identity
|
|
57
|
+
FROM pg_policy p JOIN pg_class r ON r.oid=p.polrelid
|
|
58
|
+
JOIN pg_namespace n ON n.oid=r.relnamespace
|
|
59
|
+
WHERE n.nspname !~ '^pg_' AND n.nspname NOT IN ('lc','information_schema')""",
|
|
60
|
+
"""SELECT 'pg_rewrite',(pg_identify_object('pg_rewrite'::regclass,p.oid,0)).identity
|
|
61
|
+
FROM pg_rewrite p JOIN pg_class r ON r.oid=p.ev_class
|
|
62
|
+
JOIN pg_namespace n ON n.oid=r.relnamespace
|
|
63
|
+
WHERE p.rulename<>'_RETURN' AND n.nspname !~ '^pg_'
|
|
64
|
+
AND n.nspname NOT IN ('lc','information_schema')""",
|
|
65
|
+
"""SELECT 'inheritance',format('%I.%I',n.nspname,r.relname)
|
|
66
|
+
FROM pg_class r JOIN pg_namespace n ON n.oid=r.relnamespace
|
|
67
|
+
WHERE NOT r.relispartition AND EXISTS(SELECT FROM pg_inherits i WHERE i.inhrelid=r.oid)
|
|
68
|
+
AND n.nspname !~ '^pg_' AND n.nspname NOT IN ('lc','information_schema')""",
|
|
69
|
+
"""SELECT 'table_access_method',format('%I.%I',n.nspname,r.relname)
|
|
70
|
+
FROM pg_class r JOIN pg_namespace n ON n.oid=r.relnamespace JOIN pg_am a ON a.oid=r.relam
|
|
71
|
+
WHERE r.relkind IN ('r','m') AND a.amname<>'heap' AND n.nspname !~ '^pg_'
|
|
72
|
+
AND n.nspname NOT IN ('lc','information_schema')""",
|
|
73
|
+
"""SELECT 'tablespace',format('%I.%I',n.nspname,r.relname)
|
|
74
|
+
FROM pg_class r JOIN pg_namespace n ON n.oid=r.relnamespace
|
|
75
|
+
WHERE r.reltablespace<>0 AND n.nspname !~ '^pg_'
|
|
76
|
+
AND n.nspname NOT IN ('lc','information_schema')""",
|
|
77
|
+
"""SELECT 'database_tablespace',d.datname FROM pg_database d
|
|
78
|
+
JOIN pg_tablespace t ON t.oid=d.dattablespace
|
|
79
|
+
WHERE d.datname=current_database() AND t.spcname<>'pg_default'""",
|
|
80
|
+
"""SELECT 'security_label',(pg_identify_object(classoid,objoid,objsubid)).identity
|
|
81
|
+
FROM pg_seclabel""",
|
|
82
|
+
"""SELECT 'shared_security_label',(pg_identify_object(classoid,objoid,0)).identity
|
|
83
|
+
FROM pg_shseclabel WHERE classoid<>'pg_database'::regclass
|
|
84
|
+
OR objoid=(SELECT oid FROM pg_database WHERE datname=current_database())""",
|
|
85
|
+
]
|
|
86
|
+
)
|
|
87
|
+
return [dict(r) for r in await c.fetch(" UNION ALL ".join(queries) + " ORDER BY 1,2")]
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def manifest(inv):
|
|
91
|
+
objects = []
|
|
92
|
+
for r in inv["relations"]:
|
|
93
|
+
status = "schema_only"
|
|
94
|
+
if r["extension"]:
|
|
95
|
+
status = "extension_managed"
|
|
96
|
+
elif r["kind"] in {"m", "S"} or r["persistence"] == "u":
|
|
97
|
+
status = "deferred_cutover"
|
|
98
|
+
elif r["kind"] in {"r", "p"}:
|
|
99
|
+
status = "replicated"
|
|
100
|
+
elif r["kind"] == "f":
|
|
101
|
+
status = "blocked"
|
|
102
|
+
objects.append(dict(category="relation", identity=[r["schema"], r["name"]], status=status))
|
|
103
|
+
for role in inv["roles"]:
|
|
104
|
+
objects.append(dict(category="role", identity=role["rolname"], status="schema_only"))
|
|
105
|
+
for category in ("schemas", "functions", "types", "extensions"):
|
|
106
|
+
for r in inv[category]:
|
|
107
|
+
objects.append(
|
|
108
|
+
dict(category=category, identity=[r.get("schema"), r["name"]], status="schema_only")
|
|
109
|
+
)
|
|
110
|
+
for r in inv["unsupported_objects"]:
|
|
111
|
+
objects.append(dict(r, status="blocked", reason="OBJECT_PROFILE_UNSUPPORTED"))
|
|
112
|
+
return dict(
|
|
113
|
+
schema_version="pg_logical_connector/coverage-alpha",
|
|
114
|
+
exhaustive=False,
|
|
115
|
+
objects=objects,
|
|
116
|
+
note="Qualified local profile; full SPEC object-by-object coverage is not yet implemented",
|
|
117
|
+
)
|