dbkit-sql 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbkit/__init__.py +77 -0
- dbkit/_async/__init__.py +17 -0
- dbkit/_async/_compat.py +111 -0
- dbkit/_async/connection.py +328 -0
- dbkit/_async/database.py +884 -0
- dbkit/_async/engine.py +232 -0
- dbkit/_async/executor.py +329 -0
- dbkit/_async/health.py +53 -0
- dbkit/_async/resilience.py +148 -0
- dbkit/_async/streaming.py +137 -0
- dbkit/_async/transaction.py +264 -0
- dbkit/_core/__init__.py +1 -0
- dbkit/_core/bulk.py +91 -0
- dbkit/_core/circuit.py +130 -0
- dbkit/_core/config.py +613 -0
- dbkit/_core/errors/__init__.py +74 -0
- dbkit/_core/errors/base.py +344 -0
- dbkit/_core/errors/classifier.py +138 -0
- dbkit/_core/errors/redaction.py +96 -0
- dbkit/_core/errors/sqlstate.py +74 -0
- dbkit/_core/idempotency_lint.py +57 -0
- dbkit/_core/keys.py +29 -0
- dbkit/_core/policies.py +105 -0
- dbkit/_core/query.py +142 -0
- dbkit/_core/result.py +109 -0
- dbkit/_core/routing.py +248 -0
- dbkit/_pool.py +202 -0
- dbkit/_sync/__init__.py +20 -0
- dbkit/_sync/_compat.py +82 -0
- dbkit/_sync/connection.py +331 -0
- dbkit/_sync/database.py +887 -0
- dbkit/_sync/engine.py +235 -0
- dbkit/_sync/executor.py +332 -0
- dbkit/_sync/health.py +56 -0
- dbkit/_sync/resilience.py +151 -0
- dbkit/_sync/streaming.py +140 -0
- dbkit/_sync/transaction.py +267 -0
- dbkit/cli/__init__.py +1 -0
- dbkit/cli/main.py +256 -0
- dbkit/errors.py +6 -0
- dbkit/integrations/__init__.py +21 -0
- dbkit/integrations/batch.py +81 -0
- dbkit/integrations/inbox.py +153 -0
- dbkit/observability/__init__.py +21 -0
- dbkit/observability/logging.py +154 -0
- dbkit/observability/metrics.py +110 -0
- dbkit/observability/otel_metrics.py +69 -0
- dbkit/observability/prometheus.py +78 -0
- dbkit/observability/tracing.py +128 -0
- dbkit/postgres/__init__.py +1 -0
- dbkit/postgres/copy.py +77 -0
- dbkit/postgres/pipeline.py +48 -0
- dbkit/postgres/unnest.py +73 -0
- dbkit/testing/__init__.py +1 -0
- dbkit_sql-0.1.0.dist-info/METADATA +555 -0
- dbkit_sql-0.1.0.dist-info/RECORD +59 -0
- dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
- dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
- dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
dbkit/_core/routing.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"""Routing: targets, resolved routes, and resolver protocols (§21-23).
|
|
2
|
+
|
|
3
|
+
Shard and replica resolution implement :class:`ShardResolver` / :class:`ReplicaSelector`
|
|
4
|
+
without changing the public :class:`DatabaseTarget` surface (§22, §23).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import random as _random
|
|
11
|
+
from collections import OrderedDict
|
|
12
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Literal, Protocol, runtime_checkable
|
|
15
|
+
|
|
16
|
+
from .errors import DatabaseConfigurationError, DatabaseRoutingError
|
|
17
|
+
|
|
18
|
+
Role = Literal["write", "read", "prefer_replica", "primary_only"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True, slots=True)
|
|
22
|
+
class DatabaseTarget:
|
|
23
|
+
"""Where an operation should run (§8.2, §22.1).
|
|
24
|
+
|
|
25
|
+
``role`` selects primary vs replica. ``shard_key`` (any hashable business value, e.g. an
|
|
26
|
+
organization id) is resolved to a concrete shard by a :class:`ShardResolver` in Phase 4;
|
|
27
|
+
in Phase 1 it is carried through and ignored by the single-shard resolver.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
database: str
|
|
31
|
+
role: Role = "write"
|
|
32
|
+
shard_key: object | None = None
|
|
33
|
+
|
|
34
|
+
def __post_init__(self) -> None:
|
|
35
|
+
"""Raise :class:`DatabaseRoutingError` if ``database`` is empty."""
|
|
36
|
+
if not self.database:
|
|
37
|
+
raise DatabaseRoutingError("DatabaseTarget.database must be a non-empty name")
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def wants_replica(self) -> bool:
|
|
41
|
+
"""Whether ``role`` prefers a replica (``read``/``prefer_replica``)."""
|
|
42
|
+
return self.role in ("read", "prefer_replica")
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def requires_primary(self) -> bool:
|
|
46
|
+
"""Whether ``role`` must hit the primary (``write``/``primary_only``)."""
|
|
47
|
+
return self.role in ("write", "primary_only")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True, slots=True)
|
|
51
|
+
class ResolvedRoute:
|
|
52
|
+
"""The concrete destination an operation was routed to — used for engine keying,
|
|
53
|
+
metrics labels, and trace attributes."""
|
|
54
|
+
|
|
55
|
+
database: str
|
|
56
|
+
shard_id: str
|
|
57
|
+
role: Literal["primary", "replica"]
|
|
58
|
+
replica_name: str | None = None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@runtime_checkable
|
|
62
|
+
class ShardResolver(Protocol):
|
|
63
|
+
"""Resolve a ``(database, shard_key)`` pair to a concrete shard id (§22.2)."""
|
|
64
|
+
|
|
65
|
+
def resolve(self, database: str, shard_key: object) -> str:
|
|
66
|
+
"""Return the shard id ``shard_key`` belongs to within ``database``."""
|
|
67
|
+
...
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@runtime_checkable
|
|
71
|
+
class ReplicaSelector(Protocol):
|
|
72
|
+
"""Choose a replica name for a read, or ``None`` to use the primary (§23)."""
|
|
73
|
+
|
|
74
|
+
def select(self, database: str, shard_id: str) -> str | None:
|
|
75
|
+
"""Return a replica name for this shard, or ``None`` to route to the primary."""
|
|
76
|
+
...
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class SingleShardResolver:
|
|
80
|
+
"""Default resolver for single-shard deployments: every key maps to ``"default"``."""
|
|
81
|
+
|
|
82
|
+
def resolve(self, database: str, shard_key: object) -> str:
|
|
83
|
+
"""Always return ``"default"``, regardless of ``shard_key``."""
|
|
84
|
+
return "default"
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class HashShardResolver:
|
|
88
|
+
"""Hash-based sharding (§22.2): ``shard_key`` -> ``{prefix}-NNN``.
|
|
89
|
+
|
|
90
|
+
Uses SHA-256 rather than the builtin ``hash()`` — Python randomizes ``hash()`` for
|
|
91
|
+
str/bytes per process (``PYTHONHASHSEED``), which would route the same key to different
|
|
92
|
+
shards across restarts or app instances. Write routing must be deterministic (§22.3).
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
#: Bounded per-instance cache of ``shard_key -> resolved bucket name``. The result is a
|
|
96
|
+
#: pure function of ``(shard_key, num_shards, prefix)``, all fixed once the resolver is
|
|
97
|
+
#: constructed, so caching is safe — bounded (not a plain unbounded dict) because a
|
|
98
|
+
#: high-cardinality key space (e.g. one key per request) must not grow this without limit,
|
|
99
|
+
#: the same concern the engine registry's own ``max_engines``/``evict_lru`` addresses
|
|
100
|
+
#: (performance review §10 Finding: SHA-256 was recomputed fresh on every call).
|
|
101
|
+
_CACHE_SIZE = 4096
|
|
102
|
+
|
|
103
|
+
def __init__(self, num_shards: int, *, prefix: str = "shard") -> None:
|
|
104
|
+
"""``num_shards`` fixed buckets, named ``{prefix}-000``, ``{prefix}-001``, etc."""
|
|
105
|
+
if num_shards < 1:
|
|
106
|
+
raise DatabaseConfigurationError("num_shards must be >= 1")
|
|
107
|
+
self.num_shards = num_shards
|
|
108
|
+
self.prefix = prefix
|
|
109
|
+
self._cache: OrderedDict[object, str] = OrderedDict()
|
|
110
|
+
|
|
111
|
+
def resolve(self, database: str, shard_key: object) -> str:
|
|
112
|
+
"""Hash ``shard_key`` (via SHA-256) into one of ``num_shards`` buckets."""
|
|
113
|
+
if shard_key is None:
|
|
114
|
+
raise DatabaseRoutingError(
|
|
115
|
+
f"hash sharding requires a shard_key for database {database!r}"
|
|
116
|
+
)
|
|
117
|
+
cached = self._cache.get(shard_key)
|
|
118
|
+
if cached is not None:
|
|
119
|
+
self._cache.move_to_end(shard_key)
|
|
120
|
+
return cached
|
|
121
|
+
digest = hashlib.sha256(str(shard_key).encode("utf-8")).digest()
|
|
122
|
+
bucket = int.from_bytes(digest[:8], "big") % self.num_shards
|
|
123
|
+
result = f"{self.prefix}-{bucket:03d}"
|
|
124
|
+
self._cache[shard_key] = result
|
|
125
|
+
if len(self._cache) > self._CACHE_SIZE:
|
|
126
|
+
self._cache.popitem(last=False)
|
|
127
|
+
return result
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True, slots=True)
|
|
131
|
+
class ShardRange:
|
|
132
|
+
"""A range boundary: keys ``< upper_bound`` (and >= the previous range's bound) map here."""
|
|
133
|
+
|
|
134
|
+
upper_bound: int
|
|
135
|
+
shard_id: str
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class RangeShardResolver:
|
|
139
|
+
"""Range-based sharding (§22.2): an ordered set of upper bounds, e.g. by tenant-id range."""
|
|
140
|
+
|
|
141
|
+
def __init__(self, ranges: Sequence[ShardRange]) -> None:
|
|
142
|
+
"""``ranges`` need not be pre-sorted; they're sorted by ``upper_bound`` here."""
|
|
143
|
+
if not ranges:
|
|
144
|
+
raise DatabaseConfigurationError("RangeShardResolver requires at least one range")
|
|
145
|
+
self._ranges = sorted(ranges, key=lambda r: r.upper_bound)
|
|
146
|
+
|
|
147
|
+
def resolve(self, database: str, shard_key: object) -> str:
|
|
148
|
+
"""The shard id of the first range whose ``upper_bound`` exceeds ``shard_key``."""
|
|
149
|
+
if shard_key is None:
|
|
150
|
+
raise DatabaseRoutingError(
|
|
151
|
+
f"range sharding requires a shard_key for database {database!r}"
|
|
152
|
+
)
|
|
153
|
+
for r in self._ranges:
|
|
154
|
+
if shard_key < r.upper_bound: # type: ignore[operator]
|
|
155
|
+
return r.shard_id
|
|
156
|
+
raise DatabaseRoutingError(
|
|
157
|
+
f"shard_key {shard_key!r} exceeds the highest configured range "
|
|
158
|
+
f"for database {database!r}"
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
class DirectoryShardResolver:
|
|
163
|
+
"""Explicit ``shard_key -> shard_id`` directory lookup (§22.2).
|
|
164
|
+
|
|
165
|
+
Missing mappings fail closed (§22.3) rather than silently defaulting to some shard.
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
def __init__(self, directory: Mapping[object, str]) -> None:
|
|
169
|
+
"""Seed the resolver with an initial ``{shard_key: shard_id}`` mapping."""
|
|
170
|
+
self._directory: dict[object, str] = dict(directory)
|
|
171
|
+
|
|
172
|
+
def resolve(self, database: str, shard_key: object) -> str:
|
|
173
|
+
"""Look up ``shard_key``; raises :class:`DatabaseRoutingError` if unmapped."""
|
|
174
|
+
try:
|
|
175
|
+
return self._directory[shard_key]
|
|
176
|
+
except KeyError:
|
|
177
|
+
raise DatabaseRoutingError(
|
|
178
|
+
f"no shard mapping for key {shard_key!r} in database {database!r}"
|
|
179
|
+
) from None
|
|
180
|
+
|
|
181
|
+
def set_mapping(self, shard_key: object, shard_id: str) -> None:
|
|
182
|
+
"""Update the directory, e.g. after an explicit tenant-migration workflow (§22.3)."""
|
|
183
|
+
self._directory[shard_key] = shard_id
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class CallableShardResolver:
|
|
187
|
+
"""Adapts a plain ``fn(database, shard_key) -> shard_id`` callback to :class:`ShardResolver`."""
|
|
188
|
+
|
|
189
|
+
def __init__(self, fn: Callable[[str, object], str]) -> None:
|
|
190
|
+
"""Wrap ``fn`` so it satisfies the :class:`ShardResolver` protocol."""
|
|
191
|
+
self._fn = fn
|
|
192
|
+
|
|
193
|
+
def resolve(self, database: str, shard_key: object) -> str:
|
|
194
|
+
"""Delegate to the wrapped callback."""
|
|
195
|
+
return self._fn(database, shard_key)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class RoundRobinReplicaSelector:
|
|
199
|
+
"""Cycles through a database's configured replica names in order (§23).
|
|
200
|
+
|
|
201
|
+
Constructed with an explicit ``{database: [replica_name, ...]}`` mapping rather than a
|
|
202
|
+
config object, so it stays decoupled and independently testable.
|
|
203
|
+
"""
|
|
204
|
+
|
|
205
|
+
def __init__(self, replicas: Mapping[str, Sequence[str]]) -> None:
|
|
206
|
+
"""``replicas`` maps each database name to its ordered list of replica names."""
|
|
207
|
+
self._names: dict[str, list[str]] = {db: list(names) for db, names in replicas.items()}
|
|
208
|
+
self._counters: dict[str, int] = dict.fromkeys(self._names, 0)
|
|
209
|
+
|
|
210
|
+
def select(self, database: str, shard_id: str) -> str | None:
|
|
211
|
+
"""The next replica in rotation for ``database``, or ``None`` if it has none configured."""
|
|
212
|
+
names = self._names.get(database) or []
|
|
213
|
+
if not names:
|
|
214
|
+
return None
|
|
215
|
+
idx = self._counters.get(database, 0) % len(names)
|
|
216
|
+
self._counters[database] = self._counters.get(database, 0) + 1
|
|
217
|
+
return names[idx]
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class WeightedReplicaSelector:
|
|
221
|
+
"""Weighted-random replica selection using each replica's configured weight (§23)."""
|
|
222
|
+
|
|
223
|
+
def __init__(
|
|
224
|
+
self,
|
|
225
|
+
replicas: Mapping[str, Sequence[tuple[str, int]]],
|
|
226
|
+
*,
|
|
227
|
+
rand: Callable[[], float] | None = None,
|
|
228
|
+
) -> None:
|
|
229
|
+
"""``replicas`` maps each database name to its ``(replica_name, weight)`` pairs.
|
|
230
|
+
``rand`` overrides the random source (e.g. for deterministic tests)."""
|
|
231
|
+
self._rand = rand or _random.random
|
|
232
|
+
self._replicas: dict[str, list[tuple[str, int]]] = {
|
|
233
|
+
db: list(specs) for db, specs in replicas.items()
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
def select(self, database: str, shard_id: str) -> str | None:
|
|
237
|
+
"""A replica for ``database`` chosen at random, weighted by its configured weight."""
|
|
238
|
+
specs = self._replicas.get(database) or []
|
|
239
|
+
if not specs:
|
|
240
|
+
return None
|
|
241
|
+
total = sum(max(w, 1) for _, w in specs)
|
|
242
|
+
r = self._rand() * total
|
|
243
|
+
upto = 0.0
|
|
244
|
+
for name, weight in specs:
|
|
245
|
+
upto += max(weight, 1)
|
|
246
|
+
if r < upto:
|
|
247
|
+
return name
|
|
248
|
+
return specs[-1][0]
|
dbkit/_pool.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Connection-pool instrumentation, protection, and leak detection (§10.4-10.6).
|
|
2
|
+
|
|
3
|
+
SQLAlchemy fires pool events (``connect``/``checkout``/``checkin``/``invalidate``/``close``)
|
|
4
|
+
as *synchronous* callbacks on the sync engine — even for an ``AsyncEngine`` (they run on
|
|
5
|
+
``engine.sync_engine``). So this instrumentation is identical for both frontends and lives in
|
|
6
|
+
one shared, non-generated module.
|
|
7
|
+
|
|
8
|
+
It does not manage connections itself — SQLAlchemy's pool does that. It observes the pool,
|
|
9
|
+
feeds ``db_pool_*`` metrics, powers :meth:`AsyncDatabase.pool_status`, and detects leaks /
|
|
10
|
+
long-held connections.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import logging
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
from sqlalchemy import event
|
|
22
|
+
|
|
23
|
+
from .observability import logging as obslog
|
|
24
|
+
from .observability import metrics as m
|
|
25
|
+
from .observability.metrics import MetricsSink, NoopMetrics
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class PoolSnapshot:
|
|
30
|
+
"""A point-in-time view of a single engine's pool, for ``pool_status`` / health (§10.4)."""
|
|
31
|
+
|
|
32
|
+
key: str
|
|
33
|
+
size: int
|
|
34
|
+
checked_out: int
|
|
35
|
+
checked_in: int
|
|
36
|
+
overflow: int
|
|
37
|
+
max_overflow: int
|
|
38
|
+
created: int
|
|
39
|
+
closed: int
|
|
40
|
+
invalidations: int
|
|
41
|
+
longest_current_hold_seconds: float
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def total_capacity(self) -> int:
|
|
45
|
+
"""``size + max_overflow`` — the most connections this pool will ever open."""
|
|
46
|
+
return self.size + max(self.max_overflow, 0)
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def utilization(self) -> float:
|
|
50
|
+
"""``checked_out / total_capacity``, or ``0.0`` if capacity is unbounded/zero."""
|
|
51
|
+
cap = self.total_capacity
|
|
52
|
+
return (self.checked_out / cap) if cap else 0.0
|
|
53
|
+
|
|
54
|
+
def to_dict(self) -> dict[str, Any]:
|
|
55
|
+
"""A JSON-serializable representation, e.g. for the CLI's ``pools`` command."""
|
|
56
|
+
return {
|
|
57
|
+
"key": self.key,
|
|
58
|
+
"size": self.size,
|
|
59
|
+
"checked_out": self.checked_out,
|
|
60
|
+
"checked_in": self.checked_in,
|
|
61
|
+
"overflow": self.overflow,
|
|
62
|
+
"created": self.created,
|
|
63
|
+
"closed": self.closed,
|
|
64
|
+
"invalidations": self.invalidations,
|
|
65
|
+
"utilization": round(self.utilization, 3),
|
|
66
|
+
"longest_current_hold_seconds": round(self.longest_current_hold_seconds, 3),
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class _Checkout:
|
|
72
|
+
since: float
|
|
73
|
+
context: str | None
|
|
74
|
+
owner: str | None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class PoolInstrumentation:
|
|
78
|
+
"""Attaches pool-event listeners to one engine and tracks its live pool state."""
|
|
79
|
+
|
|
80
|
+
def __init__(
|
|
81
|
+
self,
|
|
82
|
+
*,
|
|
83
|
+
key: str,
|
|
84
|
+
labels: dict[str, str],
|
|
85
|
+
long_hold_warning_seconds: float,
|
|
86
|
+
metrics: MetricsSink | None = None,
|
|
87
|
+
) -> None:
|
|
88
|
+
"""Create instrumentation for one engine; call :meth:`attach` to start observing it."""
|
|
89
|
+
self.key = key
|
|
90
|
+
self._labels = labels
|
|
91
|
+
self._long_hold = long_hold_warning_seconds
|
|
92
|
+
self._metrics = metrics or NoopMetrics()
|
|
93
|
+
self._lock = threading.Lock()
|
|
94
|
+
self._created = 0
|
|
95
|
+
self._closed = 0
|
|
96
|
+
self._invalidations = 0
|
|
97
|
+
# record-id -> checkout metadata (only currently-held connections)
|
|
98
|
+
self._checked_out: dict[int, _Checkout] = {}
|
|
99
|
+
|
|
100
|
+
# -- event handlers (run synchronously inside SQLAlchemy) --------------------- #
|
|
101
|
+
|
|
102
|
+
def _on_connect(self, _dbapi_conn: Any, _record: Any) -> None:
|
|
103
|
+
with self._lock:
|
|
104
|
+
self._created += 1
|
|
105
|
+
self._metrics.incr(m.CONN_CREATED, labels=self._labels)
|
|
106
|
+
|
|
107
|
+
def _on_checkout(self, _dbapi_conn: Any, record: Any, _proxy: Any) -> None:
|
|
108
|
+
info = record.info
|
|
109
|
+
info["dbkit_checkout_at"] = time.monotonic()
|
|
110
|
+
with self._lock:
|
|
111
|
+
self._checked_out[id(record)] = _Checkout(
|
|
112
|
+
since=info["dbkit_checkout_at"],
|
|
113
|
+
context=info.get("dbkit_context"),
|
|
114
|
+
owner=info.get("dbkit_owner"),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def _on_checkin(self, _dbapi_conn: Any, record: Any) -> None:
|
|
118
|
+
now = time.monotonic()
|
|
119
|
+
started = record.info.pop("dbkit_checkout_at", None)
|
|
120
|
+
record.info.pop("dbkit_context", None)
|
|
121
|
+
record.info.pop("dbkit_owner", None)
|
|
122
|
+
with self._lock:
|
|
123
|
+
self._checked_out.pop(id(record), None)
|
|
124
|
+
if started is not None:
|
|
125
|
+
hold = now - started
|
|
126
|
+
self._metrics.observe(m.CONN_HOLD_DURATION, hold, labels=self._labels)
|
|
127
|
+
if hold >= self._long_hold:
|
|
128
|
+
obslog.log_event(
|
|
129
|
+
logging.WARNING,
|
|
130
|
+
"database.pool.long_hold",
|
|
131
|
+
database=self._labels.get("database"),
|
|
132
|
+
role=self._labels.get("role"),
|
|
133
|
+
duration_ms=round(hold * 1000, 3),
|
|
134
|
+
pool=self.key,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
def _on_invalidate(self, _dbapi_conn: Any, _record: Any, _exc: Any) -> None:
|
|
138
|
+
with self._lock:
|
|
139
|
+
self._invalidations += 1
|
|
140
|
+
self._metrics.incr(m.POOL_INVALIDATIONS, labels=self._labels)
|
|
141
|
+
|
|
142
|
+
def _on_close(self, _dbapi_conn: Any, _record: Any) -> None:
|
|
143
|
+
with self._lock:
|
|
144
|
+
self._closed += 1
|
|
145
|
+
self._metrics.incr(m.CONN_CLOSED, labels=self._labels)
|
|
146
|
+
|
|
147
|
+
def attach(self, target: Any) -> None:
|
|
148
|
+
"""Register listeners on ``target`` (a sync ``Engine`` / ``Pool``)."""
|
|
149
|
+
event.listen(target, "connect", self._on_connect)
|
|
150
|
+
event.listen(target, "checkout", self._on_checkout)
|
|
151
|
+
event.listen(target, "checkin", self._on_checkin)
|
|
152
|
+
event.listen(target, "invalidate", self._on_invalidate)
|
|
153
|
+
event.listen(target, "close", self._on_close)
|
|
154
|
+
|
|
155
|
+
# -- introspection ------------------------------------------------------------ #
|
|
156
|
+
|
|
157
|
+
def longest_current_hold(self) -> float:
|
|
158
|
+
"""Seconds the longest-held currently-checked-out connection has been held, or 0.0."""
|
|
159
|
+
now = time.monotonic()
|
|
160
|
+
with self._lock:
|
|
161
|
+
if not self._checked_out:
|
|
162
|
+
return 0.0
|
|
163
|
+
oldest = min(c.since for c in self._checked_out.values())
|
|
164
|
+
return now - oldest
|
|
165
|
+
|
|
166
|
+
def snapshot(self, pool: Any) -> PoolSnapshot:
|
|
167
|
+
"""Build a :class:`PoolSnapshot` from a live SQLAlchemy pool object."""
|
|
168
|
+
|
|
169
|
+
# QueuePool exposes size()/checkedout()/checkedin()/overflow(); other pools may not.
|
|
170
|
+
def _call(name: str, default: int = 0) -> int:
|
|
171
|
+
fn = getattr(pool, name, None)
|
|
172
|
+
try:
|
|
173
|
+
return int(fn()) if callable(fn) else default
|
|
174
|
+
except Exception:
|
|
175
|
+
return default
|
|
176
|
+
|
|
177
|
+
size = _call("size")
|
|
178
|
+
checked_out = _call("checkedout")
|
|
179
|
+
checked_in = _call("checkedin")
|
|
180
|
+
# QueuePool.overflow() is negative while base slots remain free; report only the
|
|
181
|
+
# connections actually opened beyond the base pool size.
|
|
182
|
+
overflow = max(_call("overflow", 0), 0)
|
|
183
|
+
max_overflow = getattr(pool, "_max_overflow", 0) or 0
|
|
184
|
+
with self._lock:
|
|
185
|
+
created, closed, invalidations = self._created, self._closed, self._invalidations
|
|
186
|
+
snap = PoolSnapshot(
|
|
187
|
+
key=self.key,
|
|
188
|
+
size=size,
|
|
189
|
+
checked_out=checked_out,
|
|
190
|
+
checked_in=checked_in,
|
|
191
|
+
overflow=overflow,
|
|
192
|
+
max_overflow=int(max_overflow),
|
|
193
|
+
created=created,
|
|
194
|
+
closed=closed,
|
|
195
|
+
invalidations=invalidations,
|
|
196
|
+
longest_current_hold_seconds=self.longest_current_hold(),
|
|
197
|
+
)
|
|
198
|
+
# Emit gauges opportunistically whenever status is sampled.
|
|
199
|
+
self._metrics.gauge(m.POOL_SIZE, snap.size, labels=self._labels)
|
|
200
|
+
self._metrics.gauge(m.POOL_CHECKED_OUT, snap.checked_out, labels=self._labels)
|
|
201
|
+
self._metrics.gauge(m.POOL_OVERFLOW, snap.overflow, labels=self._labels)
|
|
202
|
+
return snap
|
dbkit/_sync/__init__.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# This file is GENERATED from ../_async/__init__.py by tools/run_unasync.py.
|
|
2
|
+
# Do not edit by hand. Run `make unasync` after changing the async source.
|
|
3
|
+
|
|
4
|
+
"""Async frontend (hand-written source of truth; the sync frontend is generated from it)."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from .connection import ConnectionScope
|
|
9
|
+
from .database import Database
|
|
10
|
+
from .engine import EngineRegistry
|
|
11
|
+
from .health import HealthReport
|
|
12
|
+
from .transaction import TransactionScope
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"ConnectionScope",
|
|
16
|
+
"Database",
|
|
17
|
+
"EngineRegistry",
|
|
18
|
+
"TransactionScope",
|
|
19
|
+
"HealthReport",
|
|
20
|
+
]
|
dbkit/_sync/_compat.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""Sync counterpart of ``_async/_compat.py`` — hand-written, skipped by the unasync generator.
|
|
2
|
+
|
|
3
|
+
The sync build has no client-side timeout primitive (there is no ``asyncio.timeout``
|
|
4
|
+
equivalent that can interrupt a blocking driver call); it relies on the server-side
|
|
5
|
+
``statement_timeout`` set per operation instead (§12.1). Cancellation is not a concept in the
|
|
6
|
+
sync world, so the cancellation helpers are inert.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
from typing import Any, Iterator, TypeVar
|
|
15
|
+
|
|
16
|
+
IS_ASYNC = False
|
|
17
|
+
|
|
18
|
+
#: Metric/label value distinguishing this frontend (§25.1).
|
|
19
|
+
API_LABEL = "sync"
|
|
20
|
+
|
|
21
|
+
T = TypeVar("T")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def sync_engine_of(engine: Any) -> Any:
|
|
25
|
+
"""A sync ``Engine`` already owns its pool and fires pool events directly."""
|
|
26
|
+
return engine
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sleep(seconds: float) -> None:
|
|
30
|
+
"""Frontend-appropriate sleep (async ``asyncio.sleep``; sync ``time.sleep``)."""
|
|
31
|
+
time.sleep(seconds)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def stream_mappings(conn: Any, statement: Any, params: Any, batch_size: int) -> Iterator[Any]:
|
|
35
|
+
"""Yield row mappings from a server-side cursor without buffering the full result.
|
|
36
|
+
|
|
37
|
+
The sync build uses ``yield_per`` execution options so SQLAlchemy streams rows in
|
|
38
|
+
``batch_size`` chunks rather than materializing the whole result (§20).
|
|
39
|
+
"""
|
|
40
|
+
result = conn.execution_options(yield_per=batch_size).execute(statement, params or {})
|
|
41
|
+
yield from result.mappings()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def copy_from_records(sa_conn: Any, table: str, columns: Any, records: Any) -> int:
|
|
45
|
+
"""Dispatch to the sync PostgreSQL COPY implementation (§19.2)."""
|
|
46
|
+
from ..postgres.copy import copy_records_sync
|
|
47
|
+
|
|
48
|
+
return copy_records_sync(sa_conn, table, columns, records)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def pipeline_scope(sa_conn: Any) -> contextlib.AbstractContextManager[None]:
|
|
52
|
+
"""Dispatch to the sync psycopg pipeline-mode implementation (§7.3)."""
|
|
53
|
+
from ..postgres.pipeline import pipeline_scope_sync
|
|
54
|
+
|
|
55
|
+
return pipeline_scope_sync(sa_conn)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def timeout_scope(seconds: float | None) -> contextlib.AbstractContextManager[None]:
|
|
59
|
+
"""No client-side deadline in the sync build; server ``statement_timeout`` does the work."""
|
|
60
|
+
return contextlib.nullcontext()
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def semaphore_acquire(sem: threading.Semaphore, timeout: float | None) -> bool:
|
|
64
|
+
"""Acquire ``sem``, returning False (never raising) if ``timeout`` elapses first.
|
|
65
|
+
|
|
66
|
+
``threading.Semaphore.acquire(timeout=...)`` already returns a bool instead of raising,
|
|
67
|
+
unlike ``asyncio.Semaphore`` (§17) — this just gives both frontends the same contract.
|
|
68
|
+
"""
|
|
69
|
+
return sem.acquire(timeout=timeout)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def is_cancellation(exc: BaseException) -> bool:
|
|
73
|
+
"""No cooperative cancellation in synchronous code."""
|
|
74
|
+
return False
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def shield_from_cancellation(result: T) -> T:
|
|
78
|
+
"""Passthrough — the sync frontend has no cancellation to shield against: by the time this
|
|
79
|
+
is called, the protected cleanup (a plain function call, evaluated as this call's argument)
|
|
80
|
+
has already run to completion (mirrors ``_async/_compat.py``'s real ``asyncio.shield``-based
|
|
81
|
+
version, which the async build genuinely needs)."""
|
|
82
|
+
return result
|