dbkit-sql 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbkit/__init__.py +77 -0
- dbkit/_async/__init__.py +17 -0
- dbkit/_async/_compat.py +111 -0
- dbkit/_async/connection.py +328 -0
- dbkit/_async/database.py +884 -0
- dbkit/_async/engine.py +232 -0
- dbkit/_async/executor.py +329 -0
- dbkit/_async/health.py +53 -0
- dbkit/_async/resilience.py +148 -0
- dbkit/_async/streaming.py +137 -0
- dbkit/_async/transaction.py +264 -0
- dbkit/_core/__init__.py +1 -0
- dbkit/_core/bulk.py +91 -0
- dbkit/_core/circuit.py +130 -0
- dbkit/_core/config.py +613 -0
- dbkit/_core/errors/__init__.py +74 -0
- dbkit/_core/errors/base.py +344 -0
- dbkit/_core/errors/classifier.py +138 -0
- dbkit/_core/errors/redaction.py +96 -0
- dbkit/_core/errors/sqlstate.py +74 -0
- dbkit/_core/idempotency_lint.py +57 -0
- dbkit/_core/keys.py +29 -0
- dbkit/_core/policies.py +105 -0
- dbkit/_core/query.py +142 -0
- dbkit/_core/result.py +109 -0
- dbkit/_core/routing.py +248 -0
- dbkit/_pool.py +202 -0
- dbkit/_sync/__init__.py +20 -0
- dbkit/_sync/_compat.py +82 -0
- dbkit/_sync/connection.py +331 -0
- dbkit/_sync/database.py +887 -0
- dbkit/_sync/engine.py +235 -0
- dbkit/_sync/executor.py +332 -0
- dbkit/_sync/health.py +56 -0
- dbkit/_sync/resilience.py +151 -0
- dbkit/_sync/streaming.py +140 -0
- dbkit/_sync/transaction.py +267 -0
- dbkit/cli/__init__.py +1 -0
- dbkit/cli/main.py +256 -0
- dbkit/errors.py +6 -0
- dbkit/integrations/__init__.py +21 -0
- dbkit/integrations/batch.py +81 -0
- dbkit/integrations/inbox.py +153 -0
- dbkit/observability/__init__.py +21 -0
- dbkit/observability/logging.py +154 -0
- dbkit/observability/metrics.py +110 -0
- dbkit/observability/otel_metrics.py +69 -0
- dbkit/observability/prometheus.py +78 -0
- dbkit/observability/tracing.py +128 -0
- dbkit/postgres/__init__.py +1 -0
- dbkit/postgres/copy.py +77 -0
- dbkit/postgres/pipeline.py +48 -0
- dbkit/postgres/unnest.py +73 -0
- dbkit/testing/__init__.py +1 -0
- dbkit_sql-0.1.0.dist-info/METADATA +555 -0
- dbkit_sql-0.1.0.dist-info/RECORD +59 -0
- dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
- dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
- dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
dbkit/_sync/engine.py
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# This file is GENERATED from ../_async/engine.py by tools/run_unasync.py.
|
|
2
|
+
# Do not edit by hand. Run `make unasync` after changing the async source.
|
|
3
|
+
|
|
4
|
+
"""Async engine registry (§9).
|
|
5
|
+
|
|
6
|
+
One :class:`~sqlalchemy.Engine` per unique
|
|
7
|
+
``environment:database:shard:role:driver`` key, created lazily and disposed on shutdown.
|
|
8
|
+
SQLAlchemy owns pooling and dialects; this module just keys, builds, instruments, and caps
|
|
9
|
+
engines.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from sqlalchemy import NullPool
|
|
20
|
+
from sqlalchemy import Engine, create_engine
|
|
21
|
+
|
|
22
|
+
from .._core.config import DbkitConfig, PoolConfig, TargetConfig
|
|
23
|
+
from .._core.errors import DatabaseConfigurationError, DatabaseRoutingError
|
|
24
|
+
from .._core.keys import EngineKey
|
|
25
|
+
from .._core.routing import ResolvedRoute
|
|
26
|
+
from .._pool import PoolInstrumentation, PoolSnapshot
|
|
27
|
+
from ..observability import metrics as m
|
|
28
|
+
from ..observability.metrics import MetricsSink, NoopMetrics
|
|
29
|
+
from ._compat import sync_engine_of
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class EngineEntry:
|
|
34
|
+
"""One cached engine plus the bookkeeping :class:`EngineRegistry` needs for it."""
|
|
35
|
+
|
|
36
|
+
key: EngineKey
|
|
37
|
+
engine: Engine
|
|
38
|
+
target: TargetConfig
|
|
39
|
+
instrumentation: PoolInstrumentation
|
|
40
|
+
labels: dict[str, str]
|
|
41
|
+
last_used: float = field(default_factory=time.monotonic)
|
|
42
|
+
|
|
43
|
+
def snapshot(self) -> PoolSnapshot:
|
|
44
|
+
"""This engine's current connection-pool snapshot."""
|
|
45
|
+
return self.instrumentation.snapshot(sync_engine_of(self.engine).pool)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _connect_args(target: TargetConfig, pool: PoolConfig) -> dict[str, Any]:
|
|
49
|
+
driver = target.driver
|
|
50
|
+
if driver in ("psycopg", "psycopg2"):
|
|
51
|
+
args: dict[str, Any] = {"connect_timeout": int(max(pool.connect_timeout_seconds, 1))}
|
|
52
|
+
if pool.pgbouncer_compatible:
|
|
53
|
+
# Disable server-side prepared-statement autoprep — required under PgBouncer
|
|
54
|
+
# transaction pooling, where a connection may hit a different backend each
|
|
55
|
+
# transaction (§ pgbouncer_compatible docstring on PoolConfig).
|
|
56
|
+
args["prepare_threshold"] = None
|
|
57
|
+
return args
|
|
58
|
+
if driver == "asyncpg":
|
|
59
|
+
# asyncpg names its connection establishment timeout "timeout".
|
|
60
|
+
args = {"timeout": pool.connect_timeout_seconds}
|
|
61
|
+
if pool.pgbouncer_compatible:
|
|
62
|
+
args["statement_cache_size"] = 0
|
|
63
|
+
return args
|
|
64
|
+
return {}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class EngineRegistry:
|
|
68
|
+
"""Creates and caches engines; enforces a maximum engine count (§9, §22.4).
|
|
69
|
+
|
|
70
|
+
When ``max_engines`` is set and ``evict_lru`` is True, reaching the cap evicts (disposes)
|
|
71
|
+
the least-recently-used engine instead of failing — the pattern for dynamic per-tenant
|
|
72
|
+
databases where the number of distinct tenants ever seen may be unbounded, but only a
|
|
73
|
+
bounded number should have live connections at once. The default (``evict_lru=False``)
|
|
74
|
+
is a hard cap: exceeding it is a configuration error, not a silent eviction.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
def __init__(
|
|
78
|
+
self,
|
|
79
|
+
config: DbkitConfig,
|
|
80
|
+
*,
|
|
81
|
+
metrics: MetricsSink | None = None,
|
|
82
|
+
max_engines: int | None = None,
|
|
83
|
+
evict_lru: bool = False,
|
|
84
|
+
) -> None:
|
|
85
|
+
"""A registry backed by ``config``, empty until :meth:`get` creates engines lazily."""
|
|
86
|
+
self._config = config
|
|
87
|
+
self._metrics = metrics or NoopMetrics()
|
|
88
|
+
self._max_engines = max_engines
|
|
89
|
+
self._evict_lru = evict_lru
|
|
90
|
+
self._entries: dict[str, EngineEntry] = {}
|
|
91
|
+
self._lock = threading.Lock()
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def count(self) -> int:
|
|
95
|
+
"""Number of currently live engines."""
|
|
96
|
+
return len(self._entries)
|
|
97
|
+
|
|
98
|
+
def _target_for(self, route: ResolvedRoute) -> TargetConfig:
|
|
99
|
+
db = self._config.databases.get(route.database)
|
|
100
|
+
if db is None:
|
|
101
|
+
raise DatabaseRoutingError(f"unknown database {route.database!r}")
|
|
102
|
+
if route.role == "replica":
|
|
103
|
+
for r in db.replicas:
|
|
104
|
+
if r.name == route.replica_name:
|
|
105
|
+
return r
|
|
106
|
+
raise DatabaseRoutingError(
|
|
107
|
+
f"unknown replica {route.replica_name!r} for database {route.database!r}"
|
|
108
|
+
)
|
|
109
|
+
return db.primary
|
|
110
|
+
|
|
111
|
+
def _key_for(self, route: ResolvedRoute, target: TargetConfig) -> EngineKey:
|
|
112
|
+
role = "primary" if route.role == "primary" else f"replica:{route.replica_name}"
|
|
113
|
+
return EngineKey(
|
|
114
|
+
environment=self._config.environment,
|
|
115
|
+
database=route.database,
|
|
116
|
+
shard_id=route.shard_id,
|
|
117
|
+
role=role,
|
|
118
|
+
driver=target.driver,
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
def _build_engine(self, target: TargetConfig) -> Engine:
|
|
122
|
+
pool = target.resolved_pool(self._config.defaults)
|
|
123
|
+
kwargs: dict[str, Any] = {
|
|
124
|
+
"connect_args": _connect_args(target, pool),
|
|
125
|
+
"pool_pre_ping": pool.pre_ping,
|
|
126
|
+
}
|
|
127
|
+
if pool.disable_pooling:
|
|
128
|
+
kwargs["poolclass"] = NullPool
|
|
129
|
+
else:
|
|
130
|
+
reset = None if pool.reset_on_return in (None, "none") else pool.reset_on_return
|
|
131
|
+
kwargs.update(
|
|
132
|
+
pool_size=pool.size,
|
|
133
|
+
max_overflow=pool.max_overflow,
|
|
134
|
+
pool_timeout=pool.timeout_seconds,
|
|
135
|
+
pool_recycle=pool.recycle_seconds,
|
|
136
|
+
pool_use_lifo=pool.use_lifo,
|
|
137
|
+
pool_reset_on_return=reset,
|
|
138
|
+
)
|
|
139
|
+
try:
|
|
140
|
+
return create_engine(target.url, **kwargs)
|
|
141
|
+
except Exception as exc:
|
|
142
|
+
raise DatabaseConfigurationError(
|
|
143
|
+
f"failed to create engine for {target.name!r}: {exc}"
|
|
144
|
+
) from exc
|
|
145
|
+
|
|
146
|
+
def get(self, route: ResolvedRoute) -> EngineEntry:
|
|
147
|
+
"""The cached engine for ``route``, creating (and, if needed, evicting) one lazily."""
|
|
148
|
+
target = self._target_for(route)
|
|
149
|
+
key = self._key_for(route, target)
|
|
150
|
+
key_str = str(key)
|
|
151
|
+
entry = self._entries.get(key_str)
|
|
152
|
+
if entry is not None:
|
|
153
|
+
entry.last_used = time.monotonic()
|
|
154
|
+
return entry
|
|
155
|
+
victim: EngineEntry | None = None
|
|
156
|
+
with self._lock:
|
|
157
|
+
entry = self._entries.get(key_str) # double-checked
|
|
158
|
+
if entry is None:
|
|
159
|
+
if self._max_engines is not None and len(self._entries) >= self._max_engines:
|
|
160
|
+
if not self._evict_lru:
|
|
161
|
+
raise DatabaseConfigurationError(
|
|
162
|
+
f"engine limit reached ({self._max_engines}); cannot create {key_str!r}"
|
|
163
|
+
)
|
|
164
|
+
# Pop the victim under the lock, but dispose it *after* releasing the lock
|
|
165
|
+
# (below) — disposing while holding the lock would block every concurrent
|
|
166
|
+
# get() that only misses the fast path on I/O it has nothing to do with
|
|
167
|
+
# (performance review §2 Finding #4).
|
|
168
|
+
victim = self._pop_lru_locked()
|
|
169
|
+
labels = {
|
|
170
|
+
"environment": key.environment,
|
|
171
|
+
"database": key.database,
|
|
172
|
+
"shard": key.shard_id,
|
|
173
|
+
"role": key.role,
|
|
174
|
+
}
|
|
175
|
+
engine = self._build_engine(target)
|
|
176
|
+
pool = target.resolved_pool(self._config.defaults)
|
|
177
|
+
instrumentation = PoolInstrumentation(
|
|
178
|
+
key=key_str,
|
|
179
|
+
labels=labels,
|
|
180
|
+
long_hold_warning_seconds=pool.long_hold_warning_seconds,
|
|
181
|
+
metrics=self._metrics,
|
|
182
|
+
)
|
|
183
|
+
instrumentation.attach(sync_engine_of(engine))
|
|
184
|
+
entry = EngineEntry(
|
|
185
|
+
key=key,
|
|
186
|
+
engine=engine,
|
|
187
|
+
target=target,
|
|
188
|
+
instrumentation=instrumentation,
|
|
189
|
+
labels=labels,
|
|
190
|
+
)
|
|
191
|
+
self._entries[key_str] = entry
|
|
192
|
+
entry.last_used = time.monotonic()
|
|
193
|
+
if victim is not None:
|
|
194
|
+
victim.engine.dispose()
|
|
195
|
+
self._metrics.incr(m.CONN_CLOSED, labels=victim.labels)
|
|
196
|
+
return entry
|
|
197
|
+
|
|
198
|
+
def _pop_lru_locked(self) -> EngineEntry | None:
|
|
199
|
+
"""Pop (but don't dispose) the least-recently-used engine. Caller must hold
|
|
200
|
+
``self._lock``; the caller is responsible for disposing the popped engine after
|
|
201
|
+
releasing the lock."""
|
|
202
|
+
if not self._entries:
|
|
203
|
+
return None
|
|
204
|
+
oldest_key = min(self._entries, key=lambda k: self._entries[k].last_used)
|
|
205
|
+
return self._entries.pop(oldest_key)
|
|
206
|
+
|
|
207
|
+
def snapshots(self) -> list[PoolSnapshot]:
|
|
208
|
+
"""A pool snapshot for every currently live engine."""
|
|
209
|
+
return [entry.snapshot() for entry in self._entries.values()]
|
|
210
|
+
|
|
211
|
+
def dispose_all(self) -> None:
|
|
212
|
+
"""Dispose every engine and clear the registry (called by ``Database.close``)."""
|
|
213
|
+
with self._lock:
|
|
214
|
+
entries = list(self._entries.values())
|
|
215
|
+
self._entries.clear()
|
|
216
|
+
for entry in entries:
|
|
217
|
+
entry.engine.dispose()
|
|
218
|
+
|
|
219
|
+
def dispose_one(self, key: str) -> bool:
|
|
220
|
+
"""Dispose one engine by its snapshot key (e.g. ``"env:database:shard:role:driver"``, as
|
|
221
|
+
printed by :meth:`snapshots`/``dbkit pools``), forcing subsequent calls to rebuild it
|
|
222
|
+
with fresh connections. Returns ``False`` if no live engine has that key.
|
|
223
|
+
|
|
224
|
+
Only closes idle pooled connections, exactly like the LRU-eviction path
|
|
225
|
+
(:meth:`_pop_lru_locked`) — a connection already checked out by another in-flight
|
|
226
|
+
coroutine keeps working until the caller releases it (verified by the same regression
|
|
227
|
+
test covering LRU eviction under concurrent use).
|
|
228
|
+
"""
|
|
229
|
+
with self._lock:
|
|
230
|
+
entry = self._entries.pop(key, None)
|
|
231
|
+
if entry is None:
|
|
232
|
+
return False
|
|
233
|
+
entry.engine.dispose()
|
|
234
|
+
self._metrics.incr(m.CONN_CLOSED, labels=entry.labels)
|
|
235
|
+
return True
|
dbkit/_sync/executor.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
# This file is GENERATED from ../_async/executor.py by tools/run_unasync.py.
|
|
2
|
+
# Do not edit by hand. Run `make unasync` after changing the async source.
|
|
3
|
+
|
|
4
|
+
"""Resilience-orchestration collaborator for :class:`~dbkit._sync.database.Database`
|
|
5
|
+
(§11.1, §14, §16, §17).
|
|
6
|
+
|
|
7
|
+
Extracted from the facade so it stays a thin dispatcher over fetch/execute/bulk/stream/
|
|
8
|
+
transaction methods, each delegating the actual "acquire a connection and run one statement,
|
|
9
|
+
with concurrency limits + circuit breaker + retries" mechanics to one focused collaborator —
|
|
10
|
+
mirroring how bulk writes, streaming, and transactions already live in their own modules. This
|
|
11
|
+
module owns exactly what the facade's own ``# -- resilience --`` / ``# -- acquisition --``
|
|
12
|
+
section comments already called out as one boundary; nothing here changes behavior, only where
|
|
13
|
+
the code lives.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import contextlib
|
|
19
|
+
import time
|
|
20
|
+
from collections.abc import Iterator, Awaitable, Callable
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
from sqlalchemy import Connection, Engine
|
|
24
|
+
|
|
25
|
+
from .._core.circuit import CircuitBreaker
|
|
26
|
+
from .._core.config import DbkitConfig
|
|
27
|
+
from .._core.errors import (
|
|
28
|
+
DatabaseCommitUnknownError,
|
|
29
|
+
DatabaseError,
|
|
30
|
+
classify,
|
|
31
|
+
is_connection_error,
|
|
32
|
+
)
|
|
33
|
+
from .._core.policies import effective_timeout
|
|
34
|
+
from .._core.query import Query
|
|
35
|
+
from .._core.routing import DatabaseTarget, ResolvedRoute
|
|
36
|
+
from ..observability import logging as obslog
|
|
37
|
+
from ..observability import metrics as m
|
|
38
|
+
from ..observability.metrics import MetricsSink
|
|
39
|
+
from ..observability.tracing import Tracer
|
|
40
|
+
from ._compat import API_LABEL
|
|
41
|
+
from .connection import ConnectionScope
|
|
42
|
+
from .engine import EngineRegistry, EngineEntry
|
|
43
|
+
from .resilience import ConcurrencyLimiter, run_with_retries
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class ResilientExecutor:
|
|
47
|
+
"""Resolves a target to an engine, acquires a connection, and runs one statement under
|
|
48
|
+
the configured concurrency limits, circuit breaker, and retry policy.
|
|
49
|
+
|
|
50
|
+
``resolve`` is injected (rather than this class doing its own routing) because routing
|
|
51
|
+
(shard/replica selection, the read-your-writes override) is a facade concern, not a
|
|
52
|
+
resilience one — this collaborator only needs "given a target, which route/engine."
|
|
53
|
+
"""
|
|
54
|
+
|
|
55
|
+
def __init__(
|
|
56
|
+
self,
|
|
57
|
+
config: DbkitConfig,
|
|
58
|
+
*,
|
|
59
|
+
registry: EngineRegistry,
|
|
60
|
+
resolve: Callable[[DatabaseTarget], ResolvedRoute],
|
|
61
|
+
metrics: MetricsSink,
|
|
62
|
+
tracer: Tracer,
|
|
63
|
+
) -> None:
|
|
64
|
+
self._config = config
|
|
65
|
+
self._registry = registry
|
|
66
|
+
self._resolve = resolve
|
|
67
|
+
self._metrics = metrics
|
|
68
|
+
self._tracer = tracer
|
|
69
|
+
self._breakers: dict[str, CircuitBreaker] = {}
|
|
70
|
+
self._limiters: dict[str, ConcurrencyLimiter] = {}
|
|
71
|
+
|
|
72
|
+
# -- routing lookup ------------------------------------------------------------- #
|
|
73
|
+
|
|
74
|
+
def entry(self, target: DatabaseTarget) -> tuple[EngineEntry, ResolvedRoute]:
|
|
75
|
+
"""The engine entry and resolved route for ``target``, creating the engine if needed."""
|
|
76
|
+
route = self._resolve(target)
|
|
77
|
+
entry = self._registry.get(route)
|
|
78
|
+
return entry, route
|
|
79
|
+
|
|
80
|
+
def labels(self, entry: EngineEntry, query_name: str, operation: str) -> dict[str, str]:
|
|
81
|
+
"""The metrics/tracing label set for one operation against ``entry``."""
|
|
82
|
+
return {
|
|
83
|
+
"environment": entry.key.environment,
|
|
84
|
+
"database": entry.key.database,
|
|
85
|
+
"shard": entry.key.shard_id,
|
|
86
|
+
"role": entry.key.role,
|
|
87
|
+
"query_name": query_name,
|
|
88
|
+
"operation": operation,
|
|
89
|
+
"api": API_LABEL,
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
@staticmethod
|
|
93
|
+
def is_postgres(entry: EngineEntry) -> bool:
|
|
94
|
+
"""Whether ``entry``'s target dialect is PostgreSQL."""
|
|
95
|
+
return entry.target.dialect == "postgresql"
|
|
96
|
+
|
|
97
|
+
@staticmethod
|
|
98
|
+
def query_meta(query: object) -> tuple[str, str, Query | None]:
|
|
99
|
+
"""``(query_name, operation, Query | None)`` for a caller-supplied query object."""
|
|
100
|
+
if isinstance(query, Query):
|
|
101
|
+
return query.name, query.operation, query
|
|
102
|
+
return "adhoc", "read", None
|
|
103
|
+
|
|
104
|
+
# -- resilience ------------------------------------------------------------------ #
|
|
105
|
+
|
|
106
|
+
def breaker_for(self, entry: EngineEntry) -> CircuitBreaker | None:
|
|
107
|
+
"""One circuit breaker per db+shard+role, created lazily if enabled (§16)."""
|
|
108
|
+
cb = self._config.defaults.circuit_breaker
|
|
109
|
+
if not cb.enabled:
|
|
110
|
+
return None
|
|
111
|
+
key = str(entry.key)
|
|
112
|
+
breaker = self._breakers.get(key)
|
|
113
|
+
if breaker is None:
|
|
114
|
+
breaker = CircuitBreaker(
|
|
115
|
+
failure_threshold=cb.failure_threshold,
|
|
116
|
+
window_seconds=cb.window_seconds,
|
|
117
|
+
open_seconds=cb.open_seconds,
|
|
118
|
+
half_open_max_calls=cb.half_open_max_calls,
|
|
119
|
+
)
|
|
120
|
+
self._breakers[key] = breaker
|
|
121
|
+
return breaker
|
|
122
|
+
|
|
123
|
+
def limiter_for(self, database: str) -> ConcurrencyLimiter:
|
|
124
|
+
"""One concurrency limiter per named database, from its concurrency config (§17)."""
|
|
125
|
+
limiter = self._limiters.get(database)
|
|
126
|
+
if limiter is None:
|
|
127
|
+
cc = self._config.databases[database].concurrency
|
|
128
|
+
limiter = ConcurrencyLimiter(
|
|
129
|
+
{
|
|
130
|
+
"database": cc.database,
|
|
131
|
+
"reads": cc.reads,
|
|
132
|
+
"writes": cc.writes,
|
|
133
|
+
"bulk": cc.bulk_writes,
|
|
134
|
+
"expensive": cc.expensive_queries,
|
|
135
|
+
}
|
|
136
|
+
)
|
|
137
|
+
self._limiters[database] = limiter
|
|
138
|
+
return limiter
|
|
139
|
+
|
|
140
|
+
def execute_with_resilience(
|
|
141
|
+
self,
|
|
142
|
+
target: DatabaseTarget,
|
|
143
|
+
query: object,
|
|
144
|
+
op: Callable[[ConnectionScope], Awaitable[Any]],
|
|
145
|
+
*,
|
|
146
|
+
commit: bool,
|
|
147
|
+
timeout: float | None,
|
|
148
|
+
deadline: float | None,
|
|
149
|
+
) -> Any:
|
|
150
|
+
"""Run ``op`` under concurrency limiting, circuit breaking, and retries (§14, §16, §17).
|
|
151
|
+
|
|
152
|
+
Concurrency is acquired *inside* each attempt (before pool checkout) so queueing
|
|
153
|
+
happens in cheap waiters and a retry backoff does not hold a slot.
|
|
154
|
+
"""
|
|
155
|
+
entry, _route = self.entry(target)
|
|
156
|
+
query_name, operation, q = self.query_meta(query)
|
|
157
|
+
labels = self.labels(entry, query_name, operation)
|
|
158
|
+
breaker = self.breaker_for(entry)
|
|
159
|
+
limiter = self.limiter_for(entry.key.database)
|
|
160
|
+
tier = "writes" if (q and q.is_write) else "reads"
|
|
161
|
+
|
|
162
|
+
def attempt() -> Any:
|
|
163
|
+
# Bound the concurrency-limiter wait by the same effective timeout the query
|
|
164
|
+
# itself would get, so a saturated tier fails with a classified error instead of
|
|
165
|
+
# queueing invisibly forever (§17).
|
|
166
|
+
limiter_timeout = effective_timeout(
|
|
167
|
+
timeout, q, self._config.defaults.query_timeout_seconds, deadline, time.monotonic()
|
|
168
|
+
)
|
|
169
|
+
with contextlib.ExitStack() as stack:
|
|
170
|
+
stack.enter_context(
|
|
171
|
+
limiter.acquire("database", timeout=limiter_timeout)
|
|
172
|
+
)
|
|
173
|
+
stack.enter_context(limiter.acquire(tier, timeout=limiter_timeout))
|
|
174
|
+
if q is not None and q.expensive:
|
|
175
|
+
# A separate, additional tier for a small number of known-heavy queries
|
|
176
|
+
# (§17) — acquired on top of the normal reads/writes tier, not instead of
|
|
177
|
+
# it, so an expensive query still counts against ordinary capacity too.
|
|
178
|
+
stack.enter_context(
|
|
179
|
+
limiter.acquire("expensive", timeout=limiter_timeout)
|
|
180
|
+
)
|
|
181
|
+
scope = stack.enter_context(
|
|
182
|
+
self.scope(
|
|
183
|
+
target,
|
|
184
|
+
query,
|
|
185
|
+
call_timeout=timeout,
|
|
186
|
+
deadline=deadline,
|
|
187
|
+
commit=commit,
|
|
188
|
+
entry=entry,
|
|
189
|
+
labels=labels,
|
|
190
|
+
)
|
|
191
|
+
)
|
|
192
|
+
return op(scope)
|
|
193
|
+
|
|
194
|
+
return run_with_retries(
|
|
195
|
+
attempt,
|
|
196
|
+
query=q,
|
|
197
|
+
retry=self._config.defaults.retry,
|
|
198
|
+
breaker=breaker,
|
|
199
|
+
metrics=self._metrics,
|
|
200
|
+
labels=labels,
|
|
201
|
+
deadline=deadline,
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
# -- acquisition ------------------------------------------------------------------ #
|
|
205
|
+
|
|
206
|
+
@contextlib.contextmanager
|
|
207
|
+
def scope(
|
|
208
|
+
self,
|
|
209
|
+
target: DatabaseTarget,
|
|
210
|
+
query: object,
|
|
211
|
+
*,
|
|
212
|
+
call_timeout: float | None,
|
|
213
|
+
deadline: float | None,
|
|
214
|
+
commit: bool,
|
|
215
|
+
entry: EngineEntry | None = None,
|
|
216
|
+
labels: dict[str, str] | None = None,
|
|
217
|
+
) -> Iterator[ConnectionScope]:
|
|
218
|
+
"""Acquire a short-lived connection scope, measure pool wait, emit metrics (§11.1).
|
|
219
|
+
|
|
220
|
+
``entry``/``labels`` let a caller that already resolved them (``execute_with_resilience``,
|
|
221
|
+
which needs them itself for the breaker/limiter/retry-labels) pass them straight through
|
|
222
|
+
instead of this method redoing the registry lookup and rebuilding an identical labels
|
|
223
|
+
dict a second time per operation (performance review §4/§11 Finding #5). Callers that
|
|
224
|
+
invoke ``scope()`` directly (``db.connection()``, ``db.transaction()``, bulk executemany)
|
|
225
|
+
don't have them yet, so both remain optional and are resolved here when omitted.
|
|
226
|
+
"""
|
|
227
|
+
if entry is None:
|
|
228
|
+
entry, _route = self.entry(target)
|
|
229
|
+
query_name, operation, q = self.query_meta(query)
|
|
230
|
+
if labels is None:
|
|
231
|
+
labels = self.labels(entry, query_name, operation)
|
|
232
|
+
timeout = effective_timeout(
|
|
233
|
+
call_timeout,
|
|
234
|
+
q,
|
|
235
|
+
self._config.defaults.query_timeout_seconds,
|
|
236
|
+
deadline,
|
|
237
|
+
time.monotonic(),
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
with self._tracer.span(
|
|
241
|
+
f"dbkit.{operation}",
|
|
242
|
+
operation_type=operation,
|
|
243
|
+
query_name=query_name,
|
|
244
|
+
database=entry.key.database,
|
|
245
|
+
shard=entry.key.shard_id,
|
|
246
|
+
role=entry.key.role,
|
|
247
|
+
) as span:
|
|
248
|
+
wait_start = time.monotonic()
|
|
249
|
+
conn = self.acquire(entry.engine, labels, query_name)
|
|
250
|
+
pool_wait = time.monotonic() - wait_start
|
|
251
|
+
self._metrics.observe(m.POOL_WAIT_SECONDS, pool_wait, labels=labels)
|
|
252
|
+
span.set_attribute("db.pool.wait_ms", pool_wait * 1000)
|
|
253
|
+
|
|
254
|
+
scope = ConnectionScope(
|
|
255
|
+
conn,
|
|
256
|
+
is_postgres=self.is_postgres(entry),
|
|
257
|
+
default_timeout=timeout,
|
|
258
|
+
database=entry.key.database,
|
|
259
|
+
shard_id=entry.key.shard_id,
|
|
260
|
+
role=entry.key.role,
|
|
261
|
+
)
|
|
262
|
+
op_start = time.monotonic()
|
|
263
|
+
try:
|
|
264
|
+
yield scope
|
|
265
|
+
except BaseException as exc:
|
|
266
|
+
with contextlib.suppress(Exception):
|
|
267
|
+
conn.rollback()
|
|
268
|
+
if isinstance(exc, DatabaseError):
|
|
269
|
+
self._metrics.incr(
|
|
270
|
+
m.OP_ERRORS, labels={**labels, "error_category": exc.category.value}
|
|
271
|
+
)
|
|
272
|
+
raise
|
|
273
|
+
else:
|
|
274
|
+
if commit:
|
|
275
|
+
try:
|
|
276
|
+
conn.commit()
|
|
277
|
+
except BaseException as commit_exc:
|
|
278
|
+
# Mirror _TransactionManager._commit(): a failure during COMMIT
|
|
279
|
+
# may mean the write committed anyway (§15) — never silently swallow
|
|
280
|
+
# that ambiguity, and never leave the exception unclassified either
|
|
281
|
+
# (both would previously happen only on this auto-commit write path).
|
|
282
|
+
if is_connection_error(commit_exc):
|
|
283
|
+
err: DatabaseError = DatabaseCommitUnknownError(
|
|
284
|
+
"connection failed during COMMIT; transaction outcome is unknown",
|
|
285
|
+
original=commit_exc,
|
|
286
|
+
database_name=entry.key.database,
|
|
287
|
+
shard_id=entry.key.shard_id,
|
|
288
|
+
role=entry.key.role,
|
|
289
|
+
query_name=query_name,
|
|
290
|
+
)
|
|
291
|
+
else:
|
|
292
|
+
with contextlib.suppress(Exception):
|
|
293
|
+
conn.rollback()
|
|
294
|
+
err = classify(
|
|
295
|
+
commit_exc,
|
|
296
|
+
query_name=query_name,
|
|
297
|
+
database_name=entry.key.database,
|
|
298
|
+
role=entry.key.role,
|
|
299
|
+
)
|
|
300
|
+
self._metrics.incr(
|
|
301
|
+
m.OP_ERRORS, labels={**labels, "error_category": err.category.value}
|
|
302
|
+
)
|
|
303
|
+
raise err from commit_exc
|
|
304
|
+
finally:
|
|
305
|
+
duration = time.monotonic() - op_start
|
|
306
|
+
with contextlib.suppress(Exception):
|
|
307
|
+
conn.close()
|
|
308
|
+
self._metrics.incr(m.OP_TOTAL, labels=labels)
|
|
309
|
+
self._metrics.observe(m.OP_DURATION, duration, labels=labels)
|
|
310
|
+
if duration * 1000 >= self._config.defaults.observability.slow_query_ms:
|
|
311
|
+
obslog.slow_query_warning(
|
|
312
|
+
query_name=query_name,
|
|
313
|
+
duration_ms=duration * 1000,
|
|
314
|
+
threshold_ms=self._config.defaults.observability.slow_query_ms,
|
|
315
|
+
database=entry.key.database,
|
|
316
|
+
pool_wait_ms=pool_wait * 1000,
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
def acquire(
|
|
320
|
+
self, engine: Engine, labels: dict[str, str], query_name: str
|
|
321
|
+
) -> Connection:
|
|
322
|
+
"""Check out a connection, classifying any failure; tags it for leak diagnostics."""
|
|
323
|
+
try:
|
|
324
|
+
conn = engine.connect()
|
|
325
|
+
except Exception as exc:
|
|
326
|
+
err = classify(exc, query_name=query_name, database_name=labels.get("database"))
|
|
327
|
+
self._metrics.incr(m.OP_ERRORS, labels={**labels, "error_category": err.category.value})
|
|
328
|
+
raise err from exc
|
|
329
|
+
# Best-effort context tag for leak diagnostics (§10.5).
|
|
330
|
+
with contextlib.suppress(Exception):
|
|
331
|
+
conn.info["dbkit_context"] = query_name
|
|
332
|
+
return conn
|
dbkit/_sync/health.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# This file is GENERATED from ../_async/health.py by tools/run_unasync.py.
|
|
2
|
+
# Do not edit by hand. Run `make unasync` after changing the async source.
|
|
3
|
+
|
|
4
|
+
"""Liveness and readiness checks (§26).
|
|
5
|
+
|
|
6
|
+
Liveness verifies the process/event loop is functioning and must not require any database.
|
|
7
|
+
Readiness verifies that *required* database targets answer a trivial ``SELECT 1`` within a
|
|
8
|
+
strict, short timeout.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
|
|
15
|
+
from sqlalchemy import text
|
|
16
|
+
from sqlalchemy import Engine
|
|
17
|
+
|
|
18
|
+
from .._core.errors import classify
|
|
19
|
+
from ._compat import timeout_scope
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class TargetHealth:
|
|
24
|
+
"""Readiness result for one required target."""
|
|
25
|
+
|
|
26
|
+
key: str
|
|
27
|
+
healthy: bool
|
|
28
|
+
error: str | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class HealthReport:
|
|
33
|
+
"""The overall result of :meth:`~dbkit.Database.health`."""
|
|
34
|
+
|
|
35
|
+
live: bool
|
|
36
|
+
ready: bool
|
|
37
|
+
targets: list[TargetHealth] = field(default_factory=list)
|
|
38
|
+
|
|
39
|
+
def to_dict(self) -> dict[str, object]:
|
|
40
|
+
"""A JSON-serializable representation, e.g. for the CLI/HTTP health endpoint."""
|
|
41
|
+
return {
|
|
42
|
+
"live": self.live,
|
|
43
|
+
"ready": self.ready,
|
|
44
|
+
"targets": [
|
|
45
|
+
{"key": t.key, "healthy": t.healthy, "error": t.error} for t in self.targets
|
|
46
|
+
],
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def ping(engine: Engine, *, timeout: float) -> None:
|
|
51
|
+
"""Run ``SELECT 1`` with a strict timeout; raises a classified error on failure (§26.3)."""
|
|
52
|
+
try:
|
|
53
|
+
with timeout_scope(timeout), engine.connect() as conn:
|
|
54
|
+
conn.execute(text("SELECT 1"))
|
|
55
|
+
except Exception as exc:
|
|
56
|
+
raise classify(exc) from exc
|