dbkit-sql 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbkit/__init__.py +77 -0
- dbkit/_async/__init__.py +17 -0
- dbkit/_async/_compat.py +111 -0
- dbkit/_async/connection.py +328 -0
- dbkit/_async/database.py +884 -0
- dbkit/_async/engine.py +232 -0
- dbkit/_async/executor.py +329 -0
- dbkit/_async/health.py +53 -0
- dbkit/_async/resilience.py +148 -0
- dbkit/_async/streaming.py +137 -0
- dbkit/_async/transaction.py +264 -0
- dbkit/_core/__init__.py +1 -0
- dbkit/_core/bulk.py +91 -0
- dbkit/_core/circuit.py +130 -0
- dbkit/_core/config.py +613 -0
- dbkit/_core/errors/__init__.py +74 -0
- dbkit/_core/errors/base.py +344 -0
- dbkit/_core/errors/classifier.py +138 -0
- dbkit/_core/errors/redaction.py +96 -0
- dbkit/_core/errors/sqlstate.py +74 -0
- dbkit/_core/idempotency_lint.py +57 -0
- dbkit/_core/keys.py +29 -0
- dbkit/_core/policies.py +105 -0
- dbkit/_core/query.py +142 -0
- dbkit/_core/result.py +109 -0
- dbkit/_core/routing.py +248 -0
- dbkit/_pool.py +202 -0
- dbkit/_sync/__init__.py +20 -0
- dbkit/_sync/_compat.py +82 -0
- dbkit/_sync/connection.py +331 -0
- dbkit/_sync/database.py +887 -0
- dbkit/_sync/engine.py +235 -0
- dbkit/_sync/executor.py +332 -0
- dbkit/_sync/health.py +56 -0
- dbkit/_sync/resilience.py +151 -0
- dbkit/_sync/streaming.py +140 -0
- dbkit/_sync/transaction.py +267 -0
- dbkit/cli/__init__.py +1 -0
- dbkit/cli/main.py +256 -0
- dbkit/errors.py +6 -0
- dbkit/integrations/__init__.py +21 -0
- dbkit/integrations/batch.py +81 -0
- dbkit/integrations/inbox.py +153 -0
- dbkit/observability/__init__.py +21 -0
- dbkit/observability/logging.py +154 -0
- dbkit/observability/metrics.py +110 -0
- dbkit/observability/otel_metrics.py +69 -0
- dbkit/observability/prometheus.py +78 -0
- dbkit/observability/tracing.py +128 -0
- dbkit/postgres/__init__.py +1 -0
- dbkit/postgres/copy.py +77 -0
- dbkit/postgres/pipeline.py +48 -0
- dbkit/postgres/unnest.py +73 -0
- dbkit/testing/__init__.py +1 -0
- dbkit_sql-0.1.0.dist-info/METADATA +555 -0
- dbkit_sql-0.1.0.dist-info/RECORD +59 -0
- dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
- dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
- dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
dbkit/_core/config.py
ADDED
|
@@ -0,0 +1,613 @@
|
|
|
1
|
+
"""Configuration model, loaders, validation, and the connection-budget calculator (§30, §10.3).
|
|
2
|
+
|
|
3
|
+
Config is plain frozen dataclasses so there is no hard pydantic dependency. It can be built
|
|
4
|
+
from a dict, a YAML file (``[yaml]`` extra), or environment variables, with ``${VAR}`` /
|
|
5
|
+
``${VAR:-default}`` expansion. :meth:`DbkitConfig.redacted` produces a secret-free copy for
|
|
6
|
+
logging and diagnostics (§29, §30).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from dataclasses import dataclass, field, replace
|
|
15
|
+
from typing import Any
|
|
16
|
+
from urllib.parse import parse_qs, urlsplit
|
|
17
|
+
|
|
18
|
+
from .errors import DatabaseConfigurationError
|
|
19
|
+
from .errors.redaction import redact_dsn
|
|
20
|
+
|
|
21
|
+
_ENV_RE = re.compile(r"\$\{(?P<name>[A-Za-z_][A-Za-z0-9_]*)(?::-(?P<default>[^}]*))?\}")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def expand_env(value: str, environ: Mapping[str, str] | None = None) -> str:
|
|
25
|
+
"""Expand ``${VAR}`` and ``${VAR:-default}`` against the environment."""
|
|
26
|
+
env = environ if environ is not None else os.environ
|
|
27
|
+
|
|
28
|
+
def _sub(m: re.Match[str]) -> str:
|
|
29
|
+
name = m.group("name")
|
|
30
|
+
if name in env:
|
|
31
|
+
return env[name]
|
|
32
|
+
default = m.group("default")
|
|
33
|
+
if default is not None:
|
|
34
|
+
return default
|
|
35
|
+
raise DatabaseConfigurationError(
|
|
36
|
+
f"environment variable {name!r} referenced in config is not set"
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
return _ENV_RE.sub(_sub, value)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _expand_tree(obj: Any, environ: Mapping[str, str] | None) -> Any:
|
|
43
|
+
if isinstance(obj, str):
|
|
44
|
+
return expand_env(obj, environ)
|
|
45
|
+
if isinstance(obj, Mapping):
|
|
46
|
+
return {k: _expand_tree(v, environ) for k, v in obj.items()}
|
|
47
|
+
if isinstance(obj, (list, tuple)):
|
|
48
|
+
return [_expand_tree(v, environ) for v in obj]
|
|
49
|
+
return obj
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True, slots=True)
|
|
53
|
+
class PoolConfig:
|
|
54
|
+
"""Connection pool settings, forwarded to SQLAlchemy's engine (§10.1-10.2)."""
|
|
55
|
+
|
|
56
|
+
size: int = 10
|
|
57
|
+
max_overflow: int = 5
|
|
58
|
+
timeout_seconds: float = 5.0
|
|
59
|
+
recycle_seconds: int = 1800
|
|
60
|
+
pre_ping: bool = True
|
|
61
|
+
use_lifo: bool = True
|
|
62
|
+
reset_on_return: str = "rollback" # "rollback" | "commit" | None
|
|
63
|
+
connect_timeout_seconds: float = 10.0
|
|
64
|
+
# Warn when a connection is held longer than this (leak detection, §10.5).
|
|
65
|
+
long_hold_warning_seconds: float = 2.0
|
|
66
|
+
# Use NullPool (no pooling) for external-pooler deployments like PgBouncer.
|
|
67
|
+
disable_pooling: bool = False
|
|
68
|
+
# Disable server-side prepared-statement autoprep (psycopg's prepare_threshold,
|
|
69
|
+
# asyncpg's statement_cache_size). Required correctness fix under PgBouncer's
|
|
70
|
+
# *transaction* pooling mode: a client connection may land on a different physical
|
|
71
|
+
# backend each transaction, so a server-side prepared statement id from a prior
|
|
72
|
+
# transaction may not exist there. Does not affect PgBouncer session pooling. This
|
|
73
|
+
# toolkit already scopes every session setting with SET LOCAL / per-transaction
|
|
74
|
+
# (never bare session-level SET), so no other change is needed for PgBouncer compatibility.
|
|
75
|
+
pgbouncer_compatible: bool = False
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def max_connections(self) -> int:
|
|
79
|
+
"""Ceiling of concurrent connections this pool can open."""
|
|
80
|
+
if self.disable_pooling:
|
|
81
|
+
return 0 # unbounded / delegated to external pooler; excluded from budget math
|
|
82
|
+
return self.size + self.max_overflow
|
|
83
|
+
|
|
84
|
+
def validate(self) -> None:
|
|
85
|
+
"""Raise :class:`DatabaseConfigurationError` if any field is out of range."""
|
|
86
|
+
if self.size < 0:
|
|
87
|
+
raise DatabaseConfigurationError("pool.size must be >= 0")
|
|
88
|
+
if self.max_overflow < 0:
|
|
89
|
+
raise DatabaseConfigurationError("pool.max_overflow must be >= 0")
|
|
90
|
+
if self.timeout_seconds <= 0:
|
|
91
|
+
raise DatabaseConfigurationError("pool.timeout_seconds must be > 0")
|
|
92
|
+
if self.reset_on_return not in ("rollback", "commit", None, "none"):
|
|
93
|
+
raise DatabaseConfigurationError(
|
|
94
|
+
"pool.reset_on_return must be one of rollback|commit|none"
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass(frozen=True, slots=True)
|
|
99
|
+
class RetryConfig:
|
|
100
|
+
"""Retry policy (§14). Writes are not retried unless explicitly enabled *and* idempotent.
|
|
101
|
+
|
|
102
|
+
``retry_writes`` and ``Query.idempotent`` are both trust-based flags, not verified
|
|
103
|
+
properties — enabling them means "I have checked this SQL is safe to run twice," not
|
|
104
|
+
"dbkit made this SQL safe to run twice." See ``dbkit query-list`` for a best-effort static
|
|
105
|
+
check that flags writes marked idempotent with no visible safety guard in their SQL text.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
attempts: int = 2
|
|
109
|
+
initial_delay_ms: float = 20.0
|
|
110
|
+
maximum_delay_ms: float = 250.0
|
|
111
|
+
multiplier: float = 2.0
|
|
112
|
+
jitter: str = "full" # "full" | "none"
|
|
113
|
+
#: Hard ceiling on total time spent retrying (backoff delays, not attempt execution time),
|
|
114
|
+
#: enforced by ``run_with_retries`` independent of ``attempts``/the caller's own ``deadline``
|
|
115
|
+
#: — a call raises once this budget is exhausted even if attempts remain.
|
|
116
|
+
maximum_total_ms: float = 750.0
|
|
117
|
+
retry_reads: bool = True
|
|
118
|
+
retry_writes: bool = False
|
|
119
|
+
|
|
120
|
+
def validate(self) -> None:
|
|
121
|
+
"""Raise :class:`DatabaseConfigurationError` if any field is out of range."""
|
|
122
|
+
if self.attempts < 1:
|
|
123
|
+
raise DatabaseConfigurationError("retry.attempts must be >= 1")
|
|
124
|
+
if self.jitter not in ("full", "none"):
|
|
125
|
+
raise DatabaseConfigurationError("retry.jitter must be 'full' or 'none'")
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass(frozen=True, slots=True)
|
|
129
|
+
class ConcurrencyConfig:
|
|
130
|
+
"""Optional per-target concurrency limits, independent of pool size (§17)."""
|
|
131
|
+
|
|
132
|
+
database: int | None = None
|
|
133
|
+
reads: int | None = None
|
|
134
|
+
writes: int | None = None
|
|
135
|
+
bulk_writes: int | None = None
|
|
136
|
+
#: A separate cap for queries explicitly marked ``Query(expensive=True)`` — acquired *in
|
|
137
|
+
#: addition to* the reads/writes tier, not instead of it, so a small number of known-heavy
|
|
138
|
+
#: queries (a big aggregation, an ad-hoc report) can be bounded independently of ordinary
|
|
139
|
+
#: traffic without needing their own database/shard/role target.
|
|
140
|
+
expensive_queries: int | None = None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass(frozen=True, slots=True)
|
|
144
|
+
class BulkConfig:
|
|
145
|
+
"""Batch-sizing defaults for bulk operations (§19.1)."""
|
|
146
|
+
|
|
147
|
+
default_batch_rows: int = 1000
|
|
148
|
+
max_batch_rows: int = 10000
|
|
149
|
+
max_payload_bytes: int | None = None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@dataclass(frozen=True, slots=True)
|
|
153
|
+
class CircuitBreakerConfig:
|
|
154
|
+
"""Per db+shard+role circuit breaker settings (§16)."""
|
|
155
|
+
|
|
156
|
+
enabled: bool = False
|
|
157
|
+
failure_threshold: int = 10
|
|
158
|
+
window_seconds: float = 30.0
|
|
159
|
+
open_seconds: float = 10.0
|
|
160
|
+
half_open_max_calls: int = 2
|
|
161
|
+
|
|
162
|
+
def validate(self) -> None:
|
|
163
|
+
"""Raise :class:`DatabaseConfigurationError` if any field is out of range."""
|
|
164
|
+
if self.failure_threshold < 1:
|
|
165
|
+
raise DatabaseConfigurationError("circuit_breaker.failure_threshold must be >= 1")
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@dataclass(frozen=True, slots=True)
|
|
169
|
+
class ObservabilityConfig:
|
|
170
|
+
"""Logging/metrics/tracing toggles (§25)."""
|
|
171
|
+
|
|
172
|
+
metrics: bool = True
|
|
173
|
+
tracing: bool = False
|
|
174
|
+
log_parameters: bool = False # never log bound params in production (§25.3)
|
|
175
|
+
slow_query_ms: float = 500.0
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@dataclass(frozen=True, slots=True)
|
|
179
|
+
class ConnectionBudgetConfig:
|
|
180
|
+
"""A cap on how many connections this process may open across all its engines (§10.3)."""
|
|
181
|
+
|
|
182
|
+
maximum_per_process: int | None = None
|
|
183
|
+
enforce_at_startup: bool = False
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@dataclass(frozen=True, slots=True)
|
|
187
|
+
class Defaults:
|
|
188
|
+
"""Process-wide defaults, overridable per-target/per-call."""
|
|
189
|
+
|
|
190
|
+
driver: str = "psycopg"
|
|
191
|
+
query_timeout_seconds: float = 2.0
|
|
192
|
+
transaction_timeout_seconds: float = 5.0
|
|
193
|
+
#: Warn when an explicit transaction is held open longer than this (§10.5, §16).
|
|
194
|
+
long_transaction_warning_seconds: float = 5.0
|
|
195
|
+
pool: PoolConfig = field(default_factory=PoolConfig)
|
|
196
|
+
retry: RetryConfig = field(default_factory=RetryConfig)
|
|
197
|
+
circuit_breaker: CircuitBreakerConfig = field(default_factory=CircuitBreakerConfig)
|
|
198
|
+
bulk: BulkConfig = field(default_factory=BulkConfig)
|
|
199
|
+
observability: ObservabilityConfig = field(default_factory=ObservabilityConfig)
|
|
200
|
+
|
|
201
|
+
def validate(self) -> None:
|
|
202
|
+
"""Raise :class:`DatabaseConfigurationError` if any nested config is invalid."""
|
|
203
|
+
self.pool.validate()
|
|
204
|
+
self.retry.validate()
|
|
205
|
+
self.circuit_breaker.validate()
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@dataclass(frozen=True, slots=True)
|
|
209
|
+
class TargetConfig:
|
|
210
|
+
"""A single physical database endpoint (primary or one replica)."""
|
|
211
|
+
|
|
212
|
+
url: str
|
|
213
|
+
name: str = "primary"
|
|
214
|
+
required: bool = True
|
|
215
|
+
weight: int = 1
|
|
216
|
+
pool: PoolConfig | None = None # falls back to defaults.pool
|
|
217
|
+
|
|
218
|
+
def resolved_pool(self, defaults: Defaults) -> PoolConfig:
|
|
219
|
+
"""This target's own pool config, or ``defaults.pool`` if it doesn't override one."""
|
|
220
|
+
return self.pool or defaults.pool
|
|
221
|
+
|
|
222
|
+
@property
|
|
223
|
+
def driver(self) -> str:
|
|
224
|
+
"""Driver name parsed from the URL, e.g. ``psycopg`` or ``asyncpg``."""
|
|
225
|
+
scheme = urlsplit(self.url).scheme
|
|
226
|
+
return scheme.split("+", 1)[1] if "+" in scheme else scheme
|
|
227
|
+
|
|
228
|
+
@property
|
|
229
|
+
def dialect(self) -> str:
|
|
230
|
+
"""SQL dialect parsed from the URL, e.g. ``postgresql``."""
|
|
231
|
+
return urlsplit(self.url).scheme.split("+", 1)[0]
|
|
232
|
+
|
|
233
|
+
@property
|
|
234
|
+
def has_explicit_tls_setting(self) -> bool:
|
|
235
|
+
"""Whether the URL's query string sets an explicit ``sslmode``/``ssl`` parameter.
|
|
236
|
+
|
|
237
|
+
dbkit does not enforce TLS itself (that's the driver/infrastructure's job) — this only
|
|
238
|
+
answers whether the DSN is explicit about its intent, for the ``dbkit check`` warning.
|
|
239
|
+
"""
|
|
240
|
+
query = parse_qs(urlsplit(self.url).query)
|
|
241
|
+
return "sslmode" in query or "ssl" in query
|
|
242
|
+
|
|
243
|
+
def validate(self) -> None:
|
|
244
|
+
"""Raise :class:`DatabaseConfigurationError` if the URL is missing or malformed."""
|
|
245
|
+
if not self.url:
|
|
246
|
+
raise DatabaseConfigurationError(f"target {self.name!r} has an empty url")
|
|
247
|
+
if not urlsplit(self.url).scheme:
|
|
248
|
+
raise DatabaseConfigurationError(
|
|
249
|
+
f"target {self.name!r} url is missing a dialect scheme (e.g. postgresql+psycopg://)"
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
@dataclass(frozen=True, slots=True)
|
|
254
|
+
class DatabaseConfig:
|
|
255
|
+
"""A logically named database: one primary and zero or more read replicas (§21)."""
|
|
256
|
+
|
|
257
|
+
primary: TargetConfig
|
|
258
|
+
replicas: tuple[TargetConfig, ...] = ()
|
|
259
|
+
concurrency: ConcurrencyConfig = field(default_factory=ConcurrencyConfig)
|
|
260
|
+
connection_budget: ConnectionBudgetConfig = field(default_factory=ConnectionBudgetConfig)
|
|
261
|
+
|
|
262
|
+
def validate(self) -> None:
|
|
263
|
+
"""Raise :class:`DatabaseConfigurationError` if the primary or any replica is invalid."""
|
|
264
|
+
self.primary.validate()
|
|
265
|
+
for r in self.replicas:
|
|
266
|
+
r.validate()
|
|
267
|
+
|
|
268
|
+
def all_targets(self) -> tuple[TargetConfig, ...]:
|
|
269
|
+
"""The primary followed by every replica, in configured order."""
|
|
270
|
+
return (self.primary, *self.replicas)
|
|
271
|
+
|
|
272
|
+
def max_connections(self, defaults: Defaults) -> int:
|
|
273
|
+
"""This database's own connection ceiling: sum of primary + every replica's pool (§10.3)."""
|
|
274
|
+
return sum(t.resolved_pool(defaults).max_connections for t in self.all_targets())
|
|
275
|
+
|
|
276
|
+
def enforce_connection_budget(self, defaults: Defaults, *, database_name: str) -> None:
|
|
277
|
+
"""Fail startup if this database alone exceeds its own configured budget (§10.3)."""
|
|
278
|
+
limit = self.connection_budget.maximum_per_process
|
|
279
|
+
if limit is None or not self.connection_budget.enforce_at_startup:
|
|
280
|
+
return
|
|
281
|
+
actual = self.max_connections(defaults)
|
|
282
|
+
if actual > limit:
|
|
283
|
+
raise DatabaseConfigurationError(
|
|
284
|
+
f"connection budget exceeded for database {database_name!r}: configured "
|
|
285
|
+
f"pools allow up to {actual} connections but the budget is {limit}"
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@dataclass(frozen=True, slots=True)
|
|
290
|
+
class DbkitConfig:
|
|
291
|
+
"""Root configuration object (§30)."""
|
|
292
|
+
|
|
293
|
+
databases: Mapping[str, DatabaseConfig]
|
|
294
|
+
environment: str = "default"
|
|
295
|
+
defaults: Defaults = field(default_factory=Defaults)
|
|
296
|
+
connection_budget: ConnectionBudgetConfig = field(default_factory=ConnectionBudgetConfig)
|
|
297
|
+
#: Process-wide cap on live engines (across all databases/shards/tenants), for dynamic
|
|
298
|
+
#: per-tenant deployments where the number of distinct tenants may be unbounded (§22.4).
|
|
299
|
+
max_engines: int | None = None
|
|
300
|
+
#: When True and ``max_engines`` is reached, evict (dispose) the least-recently-used
|
|
301
|
+
#: engine instead of failing. Default False: exceeding the cap is a configuration error.
|
|
302
|
+
evict_lru_engines: bool = False
|
|
303
|
+
|
|
304
|
+
# -- construction ------------------------------------------------------------- #
|
|
305
|
+
|
|
306
|
+
@classmethod
|
|
307
|
+
def from_dict(
|
|
308
|
+
cls,
|
|
309
|
+
data: Mapping[str, Any],
|
|
310
|
+
*,
|
|
311
|
+
environ: Mapping[str, str] | None = None,
|
|
312
|
+
expand: bool = True,
|
|
313
|
+
) -> DbkitConfig:
|
|
314
|
+
"""Build and validate a config from a plain dict (e.g. parsed YAML/JSON).
|
|
315
|
+
|
|
316
|
+
``${VAR}``/``${VAR:-default}`` placeholders in string values are expanded against
|
|
317
|
+
``environ`` (default ``os.environ``) unless ``expand=False``.
|
|
318
|
+
"""
|
|
319
|
+
raw = _expand_tree(dict(data), environ) if expand else dict(data)
|
|
320
|
+
try:
|
|
321
|
+
config = _build_config(raw)
|
|
322
|
+
except DatabaseConfigurationError:
|
|
323
|
+
raise
|
|
324
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
325
|
+
raise DatabaseConfigurationError(f"invalid configuration: {exc}") from exc
|
|
326
|
+
config.validate()
|
|
327
|
+
return config
|
|
328
|
+
|
|
329
|
+
@classmethod
|
|
330
|
+
def from_yaml(cls, path: str, *, environ: Mapping[str, str] | None = None) -> DbkitConfig:
|
|
331
|
+
"""Build and validate a config from a YAML file (requires ``dbkit[yaml]``).
|
|
332
|
+
|
|
333
|
+
A top-level ``dbkit:`` wrapper key is unwrapped automatically, so a dbkit config can
|
|
334
|
+
be embedded inside a larger application config file.
|
|
335
|
+
"""
|
|
336
|
+
try:
|
|
337
|
+
import yaml
|
|
338
|
+
except ImportError as exc: # pragma: no cover
|
|
339
|
+
raise DatabaseConfigurationError(
|
|
340
|
+
"from_yaml requires PyYAML — install dbkit[yaml]"
|
|
341
|
+
) from exc
|
|
342
|
+
with open(path, encoding="utf-8") as fh:
|
|
343
|
+
data = yaml.safe_load(fh)
|
|
344
|
+
if not isinstance(data, Mapping):
|
|
345
|
+
raise DatabaseConfigurationError(f"{path} did not contain a mapping")
|
|
346
|
+
# Allow a top-level 'dbkit:' wrapper key for embedding in larger config files.
|
|
347
|
+
if "dbkit" in data and isinstance(data["dbkit"], Mapping):
|
|
348
|
+
data = data["dbkit"]
|
|
349
|
+
return cls.from_dict(data, environ=environ)
|
|
350
|
+
|
|
351
|
+
# -- validation & budget ------------------------------------------------------ #
|
|
352
|
+
|
|
353
|
+
def validate(self) -> None:
|
|
354
|
+
"""Raise :class:`DatabaseConfigurationError` if any database or the process-wide
|
|
355
|
+
connection budget is invalid."""
|
|
356
|
+
if not self.databases:
|
|
357
|
+
raise DatabaseConfigurationError("configuration defines no databases")
|
|
358
|
+
self.defaults.validate()
|
|
359
|
+
for name, db in self.databases.items():
|
|
360
|
+
if not name:
|
|
361
|
+
raise DatabaseConfigurationError("database name must be non-empty")
|
|
362
|
+
db.validate()
|
|
363
|
+
db.enforce_connection_budget(self.defaults, database_name=name)
|
|
364
|
+
self.enforce_connection_budget()
|
|
365
|
+
|
|
366
|
+
def max_connections_per_process(self) -> int:
|
|
367
|
+
"""Sum of every target's pool ceiling — the worst-case connections this process opens."""
|
|
368
|
+
total = 0
|
|
369
|
+
for db in self.databases.values():
|
|
370
|
+
for target in db.all_targets():
|
|
371
|
+
total += target.resolved_pool(self.defaults).max_connections
|
|
372
|
+
return total
|
|
373
|
+
|
|
374
|
+
def connection_budget_report(self, replicas: int = 1) -> dict[str, int]:
|
|
375
|
+
"""Cluster-wide connection projection (§10.3): ``per_process * replicas``."""
|
|
376
|
+
per_process = self.max_connections_per_process()
|
|
377
|
+
return {
|
|
378
|
+
"per_process": per_process,
|
|
379
|
+
"app_replicas": replicas,
|
|
380
|
+
"cluster_total": per_process * max(replicas, 1),
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
def enforce_connection_budget(self) -> None:
|
|
384
|
+
"""Fail startup if a configured per-process budget is exceeded (§10.3)."""
|
|
385
|
+
limit = self.connection_budget.maximum_per_process
|
|
386
|
+
if limit is None or not self.connection_budget.enforce_at_startup:
|
|
387
|
+
return
|
|
388
|
+
actual = self.max_connections_per_process()
|
|
389
|
+
if actual > limit:
|
|
390
|
+
raise DatabaseConfigurationError(
|
|
391
|
+
f"connection budget exceeded: configured pools allow up to {actual} "
|
|
392
|
+
f"connections per process but the budget is {limit}"
|
|
393
|
+
)
|
|
394
|
+
|
|
395
|
+
# -- operational warnings (informational only — never raise) ------------------ #
|
|
396
|
+
|
|
397
|
+
def budget_enforcement_warnings(self) -> list[str]:
|
|
398
|
+
"""Non-fatal warnings (§10.3) about unenforced connection budgets.
|
|
399
|
+
|
|
400
|
+
Always empty when ``environment == "development"`` — a local/dev process silently
|
|
401
|
+
exceeding a budget is not the scenario this guards against. Outside development, an
|
|
402
|
+
unconfigured or unenforced budget means a config change (e.g. a new shard) can silently
|
|
403
|
+
push a fleet's total connections past PostgreSQL's ``max_connections`` with nothing
|
|
404
|
+
failing until the database itself starts rejecting connections under load.
|
|
405
|
+
"""
|
|
406
|
+
if self.environment == "development":
|
|
407
|
+
return []
|
|
408
|
+
warnings: list[str] = []
|
|
409
|
+
if self.connection_budget.maximum_per_process is None:
|
|
410
|
+
warnings.append(
|
|
411
|
+
f"no process-wide connection budget configured (environment={self.environment!r}) "
|
|
412
|
+
"— set connection_budget.maximum_per_process and enforce_at_startup=true, or "
|
|
413
|
+
"run `dbkit connection-budget` to size one"
|
|
414
|
+
)
|
|
415
|
+
elif not self.connection_budget.enforce_at_startup:
|
|
416
|
+
warnings.append(
|
|
417
|
+
"connection_budget.maximum_per_process is set but enforce_at_startup=false — "
|
|
418
|
+
"the budget is reported, not enforced"
|
|
419
|
+
)
|
|
420
|
+
for name, db in self.databases.items():
|
|
421
|
+
if db.connection_budget.maximum_per_process is not None and (
|
|
422
|
+
not db.connection_budget.enforce_at_startup
|
|
423
|
+
):
|
|
424
|
+
warnings.append(
|
|
425
|
+
f"database {name!r}: connection budget configured but enforce_at_startup=false"
|
|
426
|
+
)
|
|
427
|
+
return warnings
|
|
428
|
+
|
|
429
|
+
def tls_warnings(self) -> list[str]:
|
|
430
|
+
"""Non-fatal warnings (§29) about DSNs with no explicit TLS posture.
|
|
431
|
+
|
|
432
|
+
dbkit delegates TLS entirely to the driver/DSN and never enforces it — this only flags,
|
|
433
|
+
outside development, a target whose URL doesn't even state its intent (no ``sslmode``/
|
|
434
|
+
``ssl`` query parameter), which is a cheap, common misconfiguration to catch early.
|
|
435
|
+
"""
|
|
436
|
+
if self.environment == "development":
|
|
437
|
+
return []
|
|
438
|
+
warnings = []
|
|
439
|
+
for name, db in self.databases.items():
|
|
440
|
+
for target in db.all_targets():
|
|
441
|
+
if not target.has_explicit_tls_setting:
|
|
442
|
+
warnings.append(
|
|
443
|
+
f"{name}.{target.name}: DSN has no explicit sslmode/ssl parameter "
|
|
444
|
+
f"(environment={self.environment!r})"
|
|
445
|
+
)
|
|
446
|
+
return warnings
|
|
447
|
+
|
|
448
|
+
# -- redaction ---------------------------------------------------------------- #
|
|
449
|
+
|
|
450
|
+
def redacted(self) -> DbkitConfig:
|
|
451
|
+
"""Return a copy with all target URLs password-redacted (§30)."""
|
|
452
|
+
return replace(
|
|
453
|
+
self,
|
|
454
|
+
databases={
|
|
455
|
+
name: replace(
|
|
456
|
+
db,
|
|
457
|
+
primary=replace(db.primary, url=redact_dsn(db.primary.url)),
|
|
458
|
+
replicas=tuple(replace(r, url=redact_dsn(r.url)) for r in db.replicas),
|
|
459
|
+
)
|
|
460
|
+
for name, db in self.databases.items()
|
|
461
|
+
},
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
# --- dict -> dataclass builders ---------------------------------------------------- #
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _pool(data: Mapping[str, Any] | None) -> PoolConfig | None:
|
|
469
|
+
if data is None:
|
|
470
|
+
return None
|
|
471
|
+
return PoolConfig(
|
|
472
|
+
size=int(data.get("size", 10)),
|
|
473
|
+
max_overflow=int(data.get("max_overflow", 5)),
|
|
474
|
+
timeout_seconds=float(data.get("timeout_seconds", 5.0)),
|
|
475
|
+
recycle_seconds=int(data.get("recycle_seconds", 1800)),
|
|
476
|
+
pre_ping=bool(data.get("pre_ping", True)),
|
|
477
|
+
use_lifo=bool(data.get("use_lifo", True)),
|
|
478
|
+
reset_on_return=data.get("reset_on_return", "rollback"),
|
|
479
|
+
connect_timeout_seconds=float(data.get("connect_timeout_seconds", 10.0)),
|
|
480
|
+
long_hold_warning_seconds=float(data.get("long_hold_warning_seconds", 2.0)),
|
|
481
|
+
disable_pooling=bool(data.get("disable_pooling", False)),
|
|
482
|
+
pgbouncer_compatible=bool(data.get("pgbouncer_compatible", False)),
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _retry(data: Mapping[str, Any] | None) -> RetryConfig:
|
|
487
|
+
if data is None:
|
|
488
|
+
return RetryConfig()
|
|
489
|
+
return RetryConfig(
|
|
490
|
+
attempts=int(data.get("attempts", 2)),
|
|
491
|
+
initial_delay_ms=float(data.get("initial_delay_ms", 20.0)),
|
|
492
|
+
maximum_delay_ms=float(data.get("maximum_delay_ms", 250.0)),
|
|
493
|
+
multiplier=float(data.get("multiplier", 2.0)),
|
|
494
|
+
jitter=data.get("jitter", "full"),
|
|
495
|
+
maximum_total_ms=float(data.get("maximum_total_ms", 750.0)),
|
|
496
|
+
retry_reads=bool(data.get("retry_reads", True)),
|
|
497
|
+
retry_writes=bool(data.get("retry_writes", False)),
|
|
498
|
+
)
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def _bulk(data: Mapping[str, Any] | None) -> BulkConfig:
|
|
502
|
+
if data is None:
|
|
503
|
+
return BulkConfig()
|
|
504
|
+
payload = data.get("max_payload_bytes")
|
|
505
|
+
return BulkConfig(
|
|
506
|
+
default_batch_rows=int(data.get("default_batch_rows", 1000)),
|
|
507
|
+
max_batch_rows=int(data.get("max_batch_rows", 10000)),
|
|
508
|
+
max_payload_bytes=int(payload) if payload is not None else None,
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _circuit_breaker(data: Mapping[str, Any] | None) -> CircuitBreakerConfig:
|
|
513
|
+
if data is None:
|
|
514
|
+
return CircuitBreakerConfig()
|
|
515
|
+
return CircuitBreakerConfig(
|
|
516
|
+
enabled=bool(data.get("enabled", False)),
|
|
517
|
+
failure_threshold=int(data.get("failure_threshold", 10)),
|
|
518
|
+
window_seconds=float(data.get("window_seconds", 30.0)),
|
|
519
|
+
open_seconds=float(data.get("open_seconds", 10.0)),
|
|
520
|
+
half_open_max_calls=int(data.get("half_open_max_calls", 2)),
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def _observability(data: Mapping[str, Any] | None) -> ObservabilityConfig:
|
|
525
|
+
if data is None:
|
|
526
|
+
return ObservabilityConfig()
|
|
527
|
+
return ObservabilityConfig(
|
|
528
|
+
metrics=bool(data.get("metrics", True)),
|
|
529
|
+
tracing=bool(data.get("tracing", False)),
|
|
530
|
+
log_parameters=bool(data.get("log_parameters", False)),
|
|
531
|
+
slow_query_ms=float(data.get("slow_query_ms", 500.0)),
|
|
532
|
+
)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _budget(data: Mapping[str, Any] | None) -> ConnectionBudgetConfig:
|
|
536
|
+
if data is None:
|
|
537
|
+
return ConnectionBudgetConfig()
|
|
538
|
+
limit = data.get("maximum_per_process", data.get("maximum_application_share"))
|
|
539
|
+
return ConnectionBudgetConfig(
|
|
540
|
+
maximum_per_process=int(limit) if limit is not None else None,
|
|
541
|
+
enforce_at_startup=bool(data.get("enforce_at_startup", False)),
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
def _concurrency(data: Mapping[str, Any] | None) -> ConcurrencyConfig:
|
|
546
|
+
if data is None:
|
|
547
|
+
return ConcurrencyConfig()
|
|
548
|
+
return ConcurrencyConfig(
|
|
549
|
+
database=data.get("database"),
|
|
550
|
+
reads=data.get("reads"),
|
|
551
|
+
writes=data.get("writes"),
|
|
552
|
+
bulk_writes=data.get("bulk", data.get("bulk_writes")),
|
|
553
|
+
expensive_queries=data.get("expensive_queries"),
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def _defaults(data: Mapping[str, Any] | None) -> Defaults:
|
|
558
|
+
if data is None:
|
|
559
|
+
return Defaults()
|
|
560
|
+
return Defaults(
|
|
561
|
+
driver=data.get("driver", "psycopg"),
|
|
562
|
+
query_timeout_seconds=float(data.get("query_timeout_seconds", 2.0)),
|
|
563
|
+
transaction_timeout_seconds=float(data.get("transaction_timeout_seconds", 5.0)),
|
|
564
|
+
long_transaction_warning_seconds=float(data.get("long_transaction_warning_seconds", 5.0)),
|
|
565
|
+
pool=_pool(data.get("pool")) or PoolConfig(),
|
|
566
|
+
retry=_retry(data.get("retry")),
|
|
567
|
+
circuit_breaker=_circuit_breaker(data.get("circuit_breaker")),
|
|
568
|
+
bulk=_bulk(data.get("bulk")),
|
|
569
|
+
observability=_observability(data.get("observability")),
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def _target(data: Mapping[str, Any], *, name: str) -> TargetConfig:
|
|
574
|
+
if "url" not in data:
|
|
575
|
+
raise DatabaseConfigurationError(f"target {name!r} is missing 'url'")
|
|
576
|
+
return TargetConfig(
|
|
577
|
+
url=data["url"],
|
|
578
|
+
name=data.get("name", name),
|
|
579
|
+
required=bool(data.get("required", True)),
|
|
580
|
+
weight=int(data.get("weight", 1)),
|
|
581
|
+
pool=_pool(data.get("pool")),
|
|
582
|
+
)
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def _database(name: str, data: Mapping[str, Any]) -> DatabaseConfig:
|
|
586
|
+
if "primary" not in data:
|
|
587
|
+
raise DatabaseConfigurationError(f"database {name!r} is missing a 'primary' target")
|
|
588
|
+
replicas_raw = data.get("replicas", [])
|
|
589
|
+
replicas = tuple(
|
|
590
|
+
_target(r, name=r.get("name", f"replica-{i}")) for i, r in enumerate(replicas_raw)
|
|
591
|
+
)
|
|
592
|
+
return DatabaseConfig(
|
|
593
|
+
primary=_target(data["primary"], name="primary"),
|
|
594
|
+
replicas=replicas,
|
|
595
|
+
concurrency=_concurrency(data.get("concurrency")),
|
|
596
|
+
connection_budget=_budget(data.get("connection_budget")),
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _build_config(raw: Mapping[str, Any]) -> DbkitConfig:
|
|
601
|
+
databases_raw = raw.get("databases")
|
|
602
|
+
if not databases_raw:
|
|
603
|
+
raise DatabaseConfigurationError("configuration must define a 'databases' mapping")
|
|
604
|
+
databases = {name: _database(name, cfg) for name, cfg in databases_raw.items()}
|
|
605
|
+
max_engines = raw.get("max_engines")
|
|
606
|
+
return DbkitConfig(
|
|
607
|
+
databases=databases,
|
|
608
|
+
environment=raw.get("environment", "default"),
|
|
609
|
+
defaults=_defaults(raw.get("defaults")),
|
|
610
|
+
connection_budget=_budget(raw.get("connection_budget")),
|
|
611
|
+
max_engines=int(max_engines) if max_engines is not None else None,
|
|
612
|
+
evict_lru_engines=bool(raw.get("evict_lru_engines", False)),
|
|
613
|
+
)
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""dbkit error model (§13). Import everything from ``dbkit.errors``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .base import (
|
|
6
|
+
DatabaseCancellationError,
|
|
7
|
+
DatabaseCheckViolationError,
|
|
8
|
+
DatabaseCircuitOpenError,
|
|
9
|
+
DatabaseCommitUnknownError,
|
|
10
|
+
DatabaseConfigurationError,
|
|
11
|
+
DatabaseConnectionError,
|
|
12
|
+
DatabaseDeadlockError,
|
|
13
|
+
DatabaseError,
|
|
14
|
+
DatabaseForeignKeyViolationError,
|
|
15
|
+
DatabaseIntegrityError,
|
|
16
|
+
DatabaseLockTimeoutError,
|
|
17
|
+
DatabaseMappingError,
|
|
18
|
+
DatabaseNotNullViolationError,
|
|
19
|
+
DatabaseOverloadedError,
|
|
20
|
+
DatabasePermissionError,
|
|
21
|
+
DatabasePoolTimeoutError,
|
|
22
|
+
DatabaseProgrammingError,
|
|
23
|
+
DatabaseQueryTimeoutError,
|
|
24
|
+
DatabaseReadOnlyError,
|
|
25
|
+
DatabaseResultError,
|
|
26
|
+
DatabaseRoutingError,
|
|
27
|
+
DatabaseSerializationError,
|
|
28
|
+
DatabaseSyntaxError,
|
|
29
|
+
DatabaseTransactionError,
|
|
30
|
+
DatabaseUnavailableError,
|
|
31
|
+
DatabaseUniqueViolationError,
|
|
32
|
+
DatabaseUnsupportedOperationError,
|
|
33
|
+
ErrorCategory,
|
|
34
|
+
)
|
|
35
|
+
from .classifier import classify, is_connection_error
|
|
36
|
+
from .redaction import redact_dsn, redact_params, sanitize_message
|
|
37
|
+
from .sqlstate import error_class_for_sqlstate
|
|
38
|
+
|
|
39
|
+
__all__ = [
|
|
40
|
+
"DatabaseCancellationError",
|
|
41
|
+
"DatabaseCheckViolationError",
|
|
42
|
+
"DatabaseCircuitOpenError",
|
|
43
|
+
"DatabaseCommitUnknownError",
|
|
44
|
+
"DatabaseConfigurationError",
|
|
45
|
+
"DatabaseConnectionError",
|
|
46
|
+
"DatabaseDeadlockError",
|
|
47
|
+
"DatabaseError",
|
|
48
|
+
"DatabaseForeignKeyViolationError",
|
|
49
|
+
"DatabaseIntegrityError",
|
|
50
|
+
"DatabaseLockTimeoutError",
|
|
51
|
+
"DatabaseMappingError",
|
|
52
|
+
"DatabaseNotNullViolationError",
|
|
53
|
+
"DatabaseOverloadedError",
|
|
54
|
+
"DatabasePermissionError",
|
|
55
|
+
"DatabasePoolTimeoutError",
|
|
56
|
+
"DatabaseProgrammingError",
|
|
57
|
+
"DatabaseQueryTimeoutError",
|
|
58
|
+
"DatabaseReadOnlyError",
|
|
59
|
+
"DatabaseResultError",
|
|
60
|
+
"DatabaseRoutingError",
|
|
61
|
+
"DatabaseSerializationError",
|
|
62
|
+
"DatabaseSyntaxError",
|
|
63
|
+
"DatabaseTransactionError",
|
|
64
|
+
"DatabaseUnavailableError",
|
|
65
|
+
"DatabaseUniqueViolationError",
|
|
66
|
+
"DatabaseUnsupportedOperationError",
|
|
67
|
+
"ErrorCategory",
|
|
68
|
+
"classify",
|
|
69
|
+
"error_class_for_sqlstate",
|
|
70
|
+
"is_connection_error",
|
|
71
|
+
"redact_dsn",
|
|
72
|
+
"redact_params",
|
|
73
|
+
"sanitize_message",
|
|
74
|
+
]
|