dbkit-sql 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbkit/__init__.py +77 -0
- dbkit/_async/__init__.py +17 -0
- dbkit/_async/_compat.py +111 -0
- dbkit/_async/connection.py +328 -0
- dbkit/_async/database.py +884 -0
- dbkit/_async/engine.py +232 -0
- dbkit/_async/executor.py +329 -0
- dbkit/_async/health.py +53 -0
- dbkit/_async/resilience.py +148 -0
- dbkit/_async/streaming.py +137 -0
- dbkit/_async/transaction.py +264 -0
- dbkit/_core/__init__.py +1 -0
- dbkit/_core/bulk.py +91 -0
- dbkit/_core/circuit.py +130 -0
- dbkit/_core/config.py +613 -0
- dbkit/_core/errors/__init__.py +74 -0
- dbkit/_core/errors/base.py +344 -0
- dbkit/_core/errors/classifier.py +138 -0
- dbkit/_core/errors/redaction.py +96 -0
- dbkit/_core/errors/sqlstate.py +74 -0
- dbkit/_core/idempotency_lint.py +57 -0
- dbkit/_core/keys.py +29 -0
- dbkit/_core/policies.py +105 -0
- dbkit/_core/query.py +142 -0
- dbkit/_core/result.py +109 -0
- dbkit/_core/routing.py +248 -0
- dbkit/_pool.py +202 -0
- dbkit/_sync/__init__.py +20 -0
- dbkit/_sync/_compat.py +82 -0
- dbkit/_sync/connection.py +331 -0
- dbkit/_sync/database.py +887 -0
- dbkit/_sync/engine.py +235 -0
- dbkit/_sync/executor.py +332 -0
- dbkit/_sync/health.py +56 -0
- dbkit/_sync/resilience.py +151 -0
- dbkit/_sync/streaming.py +140 -0
- dbkit/_sync/transaction.py +267 -0
- dbkit/cli/__init__.py +1 -0
- dbkit/cli/main.py +256 -0
- dbkit/errors.py +6 -0
- dbkit/integrations/__init__.py +21 -0
- dbkit/integrations/batch.py +81 -0
- dbkit/integrations/inbox.py +153 -0
- dbkit/observability/__init__.py +21 -0
- dbkit/observability/logging.py +154 -0
- dbkit/observability/metrics.py +110 -0
- dbkit/observability/otel_metrics.py +69 -0
- dbkit/observability/prometheus.py +78 -0
- dbkit/observability/tracing.py +128 -0
- dbkit/postgres/__init__.py +1 -0
- dbkit/postgres/copy.py +77 -0
- dbkit/postgres/pipeline.py +48 -0
- dbkit/postgres/unnest.py +73 -0
- dbkit/testing/__init__.py +1 -0
- dbkit_sql-0.1.0.dist-info/METADATA +555 -0
- dbkit_sql-0.1.0.dist-info/RECORD +59 -0
- dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
- dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
- dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
dbkit/_async/database.py
ADDED
|
@@ -0,0 +1,884 @@
|
|
|
1
|
+
"""The ``AsyncDatabase`` facade — the primary public entrypoint (§8.1).
|
|
2
|
+
|
|
3
|
+
Orchestrates target resolution, engine lookup, connection acquisition (measuring pool wait),
|
|
4
|
+
execution, resilience (retries/circuit breaker/concurrency limits), streaming, bulk writes,
|
|
5
|
+
COPY, metrics, and graceful startup/shutdown.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import contextlib
|
|
11
|
+
import contextvars
|
|
12
|
+
import logging
|
|
13
|
+
import time
|
|
14
|
+
from collections.abc import AsyncIterator, Mapping, Sequence
|
|
15
|
+
from typing import Any, Literal
|
|
16
|
+
|
|
17
|
+
from sqlalchemy import Table, insert
|
|
18
|
+
from sqlalchemy.dialects import postgresql
|
|
19
|
+
|
|
20
|
+
from .._core import bulk as bulk_mod
|
|
21
|
+
from .._core.config import DbkitConfig
|
|
22
|
+
from .._core.errors import (
|
|
23
|
+
DatabaseError,
|
|
24
|
+
DatabaseRoutingError,
|
|
25
|
+
DatabaseUnsupportedOperationError,
|
|
26
|
+
classify,
|
|
27
|
+
)
|
|
28
|
+
from .._core.query import coerce_statement
|
|
29
|
+
from .._core.result import ExecutionResult
|
|
30
|
+
from .._core.routing import (
|
|
31
|
+
DatabaseTarget,
|
|
32
|
+
ReplicaSelector,
|
|
33
|
+
ResolvedRoute,
|
|
34
|
+
RoundRobinReplicaSelector,
|
|
35
|
+
ShardResolver,
|
|
36
|
+
SingleShardResolver,
|
|
37
|
+
)
|
|
38
|
+
from .._pool import PoolSnapshot
|
|
39
|
+
from ..observability import logging as obslog
|
|
40
|
+
from ..observability import metrics as m
|
|
41
|
+
from ..observability.metrics import MetricsSink, NoopMetrics
|
|
42
|
+
from ..observability.tracing import Tracer, make_tracer
|
|
43
|
+
from ..postgres import unnest as unnest_mod
|
|
44
|
+
from ._compat import copy_from_records
|
|
45
|
+
from .connection import AsyncConnectionScope, run_execute
|
|
46
|
+
from .engine import AsyncEngineRegistry, EngineEntry
|
|
47
|
+
from .executor import ResilientExecutor
|
|
48
|
+
from .health import HealthReport, TargetHealth, ping
|
|
49
|
+
from .streaming import AsyncResultStream
|
|
50
|
+
from .transaction import _AsyncTransactionManager
|
|
51
|
+
|
|
52
|
+
#: Task-local (safe across concurrent operations) consistency-scope override for read routing.
|
|
53
|
+
_consistency_mode: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
|
54
|
+
"dbkit_consistency_mode", default=None
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class AsyncDatabase:
|
|
59
|
+
"""Async, SQL-first database facade over SQLAlchemy Core (§8.1)."""
|
|
60
|
+
|
|
61
|
+
def __init__(
|
|
62
|
+
self,
|
|
63
|
+
config: DbkitConfig,
|
|
64
|
+
*,
|
|
65
|
+
metrics: MetricsSink | None = None,
|
|
66
|
+
tracer: Tracer | None = None,
|
|
67
|
+
shard_resolver: ShardResolver | None = None,
|
|
68
|
+
replica_selector: ReplicaSelector | None = None,
|
|
69
|
+
) -> None:
|
|
70
|
+
"""Build a facade from an already-validated :class:`DbkitConfig`.
|
|
71
|
+
|
|
72
|
+
Prefer :meth:`from_config` unless you need to pass a config object you built/validated
|
|
73
|
+
yourself. Does not connect to any database — call :meth:`start` before use.
|
|
74
|
+
"""
|
|
75
|
+
self._config = config
|
|
76
|
+
if metrics is None:
|
|
77
|
+
metrics = (
|
|
78
|
+
_make_default_metrics(config)
|
|
79
|
+
if config.defaults.observability.metrics
|
|
80
|
+
else NoopMetrics()
|
|
81
|
+
)
|
|
82
|
+
self._metrics = metrics
|
|
83
|
+
self._tracer = tracer or make_tracer(config.defaults.observability.tracing)
|
|
84
|
+
self._registry = AsyncEngineRegistry(
|
|
85
|
+
config,
|
|
86
|
+
metrics=self._metrics,
|
|
87
|
+
max_engines=config.max_engines,
|
|
88
|
+
evict_lru=config.evict_lru_engines,
|
|
89
|
+
)
|
|
90
|
+
self._shards = shard_resolver or SingleShardResolver()
|
|
91
|
+
self._replicas = replica_selector or RoundRobinReplicaSelector(
|
|
92
|
+
{name: [r.name for r in db.replicas] for name, db in config.databases.items()}
|
|
93
|
+
)
|
|
94
|
+
self._executor = ResilientExecutor(
|
|
95
|
+
config,
|
|
96
|
+
registry=self._registry,
|
|
97
|
+
resolve=self._resolve,
|
|
98
|
+
metrics=self._metrics,
|
|
99
|
+
tracer=self._tracer,
|
|
100
|
+
)
|
|
101
|
+
self._started = False
|
|
102
|
+
|
|
103
|
+
# -- construction ------------------------------------------------------------- #
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def from_config(
|
|
107
|
+
cls,
|
|
108
|
+
config: DbkitConfig | Mapping[str, Any],
|
|
109
|
+
*,
|
|
110
|
+
metrics: MetricsSink | None = None,
|
|
111
|
+
tracer: Tracer | None = None,
|
|
112
|
+
shard_resolver: ShardResolver | None = None,
|
|
113
|
+
replica_selector: ReplicaSelector | None = None,
|
|
114
|
+
) -> AsyncDatabase:
|
|
115
|
+
"""Build a facade from a :class:`DbkitConfig` or a plain dict (validated via
|
|
116
|
+
:meth:`DbkitConfig.from_dict`). The usual entry point."""
|
|
117
|
+
cfg = config if isinstance(config, DbkitConfig) else DbkitConfig.from_dict(config)
|
|
118
|
+
return cls(
|
|
119
|
+
cfg,
|
|
120
|
+
metrics=metrics,
|
|
121
|
+
tracer=tracer,
|
|
122
|
+
shard_resolver=shard_resolver,
|
|
123
|
+
replica_selector=replica_selector,
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def config(self) -> DbkitConfig:
|
|
128
|
+
"""The validated configuration this facade was built from."""
|
|
129
|
+
return self._config
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def engine_count(self) -> int:
|
|
133
|
+
"""Number of live SQLAlchemy engines currently held (one per target)."""
|
|
134
|
+
return self._registry.count
|
|
135
|
+
|
|
136
|
+
# -- lifecycle ---------------------------------------------------------------- #
|
|
137
|
+
|
|
138
|
+
async def start(self, *, warm: bool = False) -> None:
|
|
139
|
+
"""Create required engines and (optionally) warm connections (§27.1)."""
|
|
140
|
+
if self._started:
|
|
141
|
+
return
|
|
142
|
+
self._config.validate()
|
|
143
|
+
for name, db in self._config.databases.items():
|
|
144
|
+
if db.primary.required:
|
|
145
|
+
entry = await self._registry.get(
|
|
146
|
+
ResolvedRoute(database=name, shard_id="default", role="primary")
|
|
147
|
+
)
|
|
148
|
+
if warm:
|
|
149
|
+
await ping(entry.engine, timeout=self._config.defaults.query_timeout_seconds)
|
|
150
|
+
self._started = True
|
|
151
|
+
obslog.log_event(logging.INFO, "database.started", engines=self._registry.count)
|
|
152
|
+
|
|
153
|
+
async def require_ready(self) -> None:
|
|
154
|
+
"""Raise unless every required target is reachable (§27.1)."""
|
|
155
|
+
report = await self.health()
|
|
156
|
+
if not report.ready:
|
|
157
|
+
failed = [t.key for t in report.targets if not t.healthy]
|
|
158
|
+
raise DatabaseError(f"required databases not ready: {failed}")
|
|
159
|
+
|
|
160
|
+
async def close(self, grace_period: float = 10.0) -> None:
|
|
161
|
+
"""Dispose all engines (§27.2). ``grace_period`` reserved for in-flight draining."""
|
|
162
|
+
await self._registry.dispose_all()
|
|
163
|
+
self._started = False
|
|
164
|
+
obslog.log_event(logging.INFO, "database.closed")
|
|
165
|
+
|
|
166
|
+
async def __aenter__(self) -> AsyncDatabase:
|
|
167
|
+
"""``async with AsyncDatabase(...) as db:`` — calls :meth:`start`."""
|
|
168
|
+
await self.start()
|
|
169
|
+
return self
|
|
170
|
+
|
|
171
|
+
async def __aexit__(self, *exc: object) -> None:
|
|
172
|
+
"""Calls :meth:`close`."""
|
|
173
|
+
await self.close()
|
|
174
|
+
|
|
175
|
+
# -- routing ------------------------------------------------------------------ #
|
|
176
|
+
|
|
177
|
+
def _resolve(self, target: DatabaseTarget) -> ResolvedRoute:
|
|
178
|
+
db = self._config.databases.get(target.database)
|
|
179
|
+
if db is None:
|
|
180
|
+
raise DatabaseRoutingError(f"unknown database {target.database!r}")
|
|
181
|
+
shard_id = (
|
|
182
|
+
self._shards.resolve(target.database, target.shard_key)
|
|
183
|
+
if target.shard_key is not None
|
|
184
|
+
else "default"
|
|
185
|
+
)
|
|
186
|
+
# A consistency_scope(read_your_writes) override forces reads to the primary so they
|
|
187
|
+
# observe writes made earlier in the same scope (§23).
|
|
188
|
+
if target.wants_replica and db.replicas and _consistency_mode.get() is None:
|
|
189
|
+
replica_name = self._replicas.select(target.database, shard_id)
|
|
190
|
+
if replica_name is not None:
|
|
191
|
+
return ResolvedRoute(
|
|
192
|
+
database=target.database,
|
|
193
|
+
shard_id=shard_id,
|
|
194
|
+
role="replica",
|
|
195
|
+
replica_name=replica_name,
|
|
196
|
+
)
|
|
197
|
+
# Explicit fallback: selector found no replica for this database -> primary (§23).
|
|
198
|
+
return ResolvedRoute(database=target.database, shard_id=shard_id, role="primary")
|
|
199
|
+
|
|
200
|
+
# -- read API ----------------------------------------------------------------- #
|
|
201
|
+
|
|
202
|
+
async def fetch_all(
|
|
203
|
+
self,
|
|
204
|
+
query: object,
|
|
205
|
+
params: Mapping[str, Any] | None = None,
|
|
206
|
+
*,
|
|
207
|
+
target: DatabaseTarget,
|
|
208
|
+
map_to: Any = None,
|
|
209
|
+
timeout: float | None = None,
|
|
210
|
+
deadline: float | None = None,
|
|
211
|
+
) -> list[Any]:
|
|
212
|
+
"""Run a one-shot read, acquiring and releasing its own connection (§11.1).
|
|
213
|
+
|
|
214
|
+
Returns every row, mapped to ``map_to``, with no cardinality constraint. Retries and
|
|
215
|
+
the circuit breaker apply per the resolved ``target``'s policy.
|
|
216
|
+
"""
|
|
217
|
+
rows: list[Any] = await self._executor.execute_with_resilience(
|
|
218
|
+
target,
|
|
219
|
+
query,
|
|
220
|
+
lambda scope: scope.fetch_all(query, params, map_to=map_to),
|
|
221
|
+
commit=False,
|
|
222
|
+
timeout=timeout,
|
|
223
|
+
deadline=deadline,
|
|
224
|
+
)
|
|
225
|
+
return rows
|
|
226
|
+
|
|
227
|
+
async def fetch_one(
|
|
228
|
+
self,
|
|
229
|
+
query: object,
|
|
230
|
+
params: Mapping[str, Any] | None = None,
|
|
231
|
+
*,
|
|
232
|
+
target: DatabaseTarget,
|
|
233
|
+
map_to: Any = None,
|
|
234
|
+
timeout: float | None = None,
|
|
235
|
+
deadline: float | None = None,
|
|
236
|
+
) -> Any:
|
|
237
|
+
"""Run a one-shot read expecting exactly one row.
|
|
238
|
+
|
|
239
|
+
Raises :class:`~dbkit.errors.DatabaseResultError` if zero or more than one row comes
|
|
240
|
+
back (via SQLAlchemy's own ``Result.one()``).
|
|
241
|
+
"""
|
|
242
|
+
return await self._executor.execute_with_resilience(
|
|
243
|
+
target,
|
|
244
|
+
query,
|
|
245
|
+
lambda scope: scope.fetch_one(query, params, map_to=map_to),
|
|
246
|
+
commit=False,
|
|
247
|
+
timeout=timeout,
|
|
248
|
+
deadline=deadline,
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
async def fetch_optional(
|
|
252
|
+
self,
|
|
253
|
+
query: object,
|
|
254
|
+
params: Mapping[str, Any] | None = None,
|
|
255
|
+
*,
|
|
256
|
+
target: DatabaseTarget,
|
|
257
|
+
map_to: Any = None,
|
|
258
|
+
timeout: float | None = None,
|
|
259
|
+
deadline: float | None = None,
|
|
260
|
+
) -> Any | None:
|
|
261
|
+
"""Run a one-shot read expecting zero or one row; ``None`` if empty."""
|
|
262
|
+
return await self._executor.execute_with_resilience(
|
|
263
|
+
target,
|
|
264
|
+
query,
|
|
265
|
+
lambda scope: scope.fetch_optional(query, params, map_to=map_to),
|
|
266
|
+
commit=False,
|
|
267
|
+
timeout=timeout,
|
|
268
|
+
deadline=deadline,
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
async def fetch_value(
|
|
272
|
+
self,
|
|
273
|
+
query: object,
|
|
274
|
+
params: Mapping[str, Any] | None = None,
|
|
275
|
+
*,
|
|
276
|
+
target: DatabaseTarget,
|
|
277
|
+
timeout: float | None = None,
|
|
278
|
+
deadline: float | None = None,
|
|
279
|
+
) -> Any:
|
|
280
|
+
"""Run a one-shot read expecting exactly one row and return its first column."""
|
|
281
|
+
return await self._executor.execute_with_resilience(
|
|
282
|
+
target,
|
|
283
|
+
query,
|
|
284
|
+
lambda scope: scope.fetch_value(query, params),
|
|
285
|
+
commit=False,
|
|
286
|
+
timeout=timeout,
|
|
287
|
+
deadline=deadline,
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
async def fetch_values(
|
|
291
|
+
self,
|
|
292
|
+
query: object,
|
|
293
|
+
params: Mapping[str, Any] | None = None,
|
|
294
|
+
*,
|
|
295
|
+
target: DatabaseTarget,
|
|
296
|
+
timeout: float | None = None,
|
|
297
|
+
deadline: float | None = None,
|
|
298
|
+
) -> list[Any]:
|
|
299
|
+
"""Run a one-shot read and return the first column of every row."""
|
|
300
|
+
values: list[Any] = await self._executor.execute_with_resilience(
|
|
301
|
+
target,
|
|
302
|
+
query,
|
|
303
|
+
lambda scope: scope.fetch_values(query, params),
|
|
304
|
+
commit=False,
|
|
305
|
+
timeout=timeout,
|
|
306
|
+
deadline=deadline,
|
|
307
|
+
)
|
|
308
|
+
return values
|
|
309
|
+
|
|
310
|
+
# -- write API ---------------------------------------------------------------- #
|
|
311
|
+
|
|
312
|
+
async def execute(
|
|
313
|
+
self,
|
|
314
|
+
query: object,
|
|
315
|
+
params: Mapping[str, Any] | None = None,
|
|
316
|
+
*,
|
|
317
|
+
target: DatabaseTarget,
|
|
318
|
+
timeout: float | None = None,
|
|
319
|
+
deadline: float | None = None,
|
|
320
|
+
) -> ExecutionResult:
|
|
321
|
+
"""Run a one-shot write/DDL statement, auto-committed on success (§11.1)."""
|
|
322
|
+
query_name, _, _ = self._executor.query_meta(query)
|
|
323
|
+
start = time.monotonic()
|
|
324
|
+
row_count = await self._executor.execute_with_resilience(
|
|
325
|
+
target,
|
|
326
|
+
query,
|
|
327
|
+
lambda scope: scope.execute(query, params),
|
|
328
|
+
commit=True,
|
|
329
|
+
timeout=timeout,
|
|
330
|
+
deadline=deadline,
|
|
331
|
+
)
|
|
332
|
+
return ExecutionResult(
|
|
333
|
+
row_count=row_count,
|
|
334
|
+
query_name=query_name,
|
|
335
|
+
database_name=target.database,
|
|
336
|
+
duration_ms=(time.monotonic() - start) * 1000,
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
async def execute_many(
|
|
340
|
+
self,
|
|
341
|
+
query: object,
|
|
342
|
+
params_seq: Sequence[Mapping[str, Any]],
|
|
343
|
+
*,
|
|
344
|
+
target: DatabaseTarget,
|
|
345
|
+
timeout: float | None = None,
|
|
346
|
+
deadline: float | None = None,
|
|
347
|
+
) -> ExecutionResult:
|
|
348
|
+
"""Run one statement against many parameter sets (driver executemany)."""
|
|
349
|
+
query_name, _op, _q = self._executor.query_meta(query)
|
|
350
|
+
statement = coerce_statement(query)
|
|
351
|
+
start = time.monotonic()
|
|
352
|
+
async with self._executor.scope(
|
|
353
|
+
target, query, call_timeout=timeout, deadline=deadline, commit=True
|
|
354
|
+
) as scope:
|
|
355
|
+
try:
|
|
356
|
+
cursor = await run_execute(
|
|
357
|
+
scope.raw,
|
|
358
|
+
statement,
|
|
359
|
+
list(params_seq),
|
|
360
|
+
timeout=scope._default_timeout,
|
|
361
|
+
is_postgres=scope._is_postgres,
|
|
362
|
+
)
|
|
363
|
+
row_count = cursor.rowcount
|
|
364
|
+
except Exception as exc:
|
|
365
|
+
raise classify(exc, query_name=query_name, database_name=target.database) from exc
|
|
366
|
+
return ExecutionResult(
|
|
367
|
+
row_count=row_count,
|
|
368
|
+
query_name=query_name,
|
|
369
|
+
database_name=target.database,
|
|
370
|
+
duration_ms=(time.monotonic() - start) * 1000,
|
|
371
|
+
)
|
|
372
|
+
|
|
373
|
+
# -- explicit scopes ---------------------------------------------------------- #
|
|
374
|
+
|
|
375
|
+
@contextlib.asynccontextmanager
|
|
376
|
+
async def connection(
|
|
377
|
+
self, *, target: DatabaseTarget, timeout: float | None = None
|
|
378
|
+
) -> AsyncIterator[AsyncConnectionScope]:
|
|
379
|
+
"""A held connection with commit-on-success / rollback-on-error semantics (§11.2)."""
|
|
380
|
+
async with self._executor.scope(
|
|
381
|
+
target, "adhoc", call_timeout=timeout, deadline=None, commit=True
|
|
382
|
+
) as scope:
|
|
383
|
+
yield scope
|
|
384
|
+
|
|
385
|
+
@contextlib.asynccontextmanager
|
|
386
|
+
async def transaction(
|
|
387
|
+
self,
|
|
388
|
+
*,
|
|
389
|
+
target: DatabaseTarget,
|
|
390
|
+
isolation: str | None = None,
|
|
391
|
+
read_only: bool = False,
|
|
392
|
+
deferrable: bool = False,
|
|
393
|
+
timeout: float | None = None,
|
|
394
|
+
lock_timeout: float | None = None,
|
|
395
|
+
) -> AsyncIterator[Any]:
|
|
396
|
+
"""An explicit transaction with isolation/timeout options (§11.3).
|
|
397
|
+
|
|
398
|
+
``deferrable`` (PostgreSQL only) is only meaningful together with
|
|
399
|
+
``isolation="serializable"`` and ``read_only=True``: it lets a read-only serializable
|
|
400
|
+
transaction wait for a safe snapshot instead of risking a serialization failure.
|
|
401
|
+
|
|
402
|
+
Usage::
|
|
403
|
+
|
|
404
|
+
async with db.transaction(target=write_target) as tx:
|
|
405
|
+
await tx.execute(INSERT, params)
|
|
406
|
+
"""
|
|
407
|
+
route = self._resolve(target)
|
|
408
|
+
entry = await self._registry.get(route)
|
|
409
|
+
labels = self._executor.labels(entry, "transaction", "transaction")
|
|
410
|
+
with self._tracer.span(
|
|
411
|
+
"dbkit.transaction",
|
|
412
|
+
operation_type="transaction",
|
|
413
|
+
query_name="transaction",
|
|
414
|
+
database=entry.key.database,
|
|
415
|
+
shard=entry.key.shard_id,
|
|
416
|
+
role=entry.key.role,
|
|
417
|
+
) as span:
|
|
418
|
+
manager = _AsyncTransactionManager(
|
|
419
|
+
entry.engine,
|
|
420
|
+
is_postgres=self._executor.is_postgres(entry),
|
|
421
|
+
default_timeout=self._config.defaults.transaction_timeout_seconds,
|
|
422
|
+
database=entry.key.database,
|
|
423
|
+
shard_id=entry.key.shard_id,
|
|
424
|
+
role=entry.key.role,
|
|
425
|
+
isolation=isolation,
|
|
426
|
+
read_only=read_only,
|
|
427
|
+
deferrable=deferrable,
|
|
428
|
+
timeout=timeout,
|
|
429
|
+
lock_timeout=lock_timeout,
|
|
430
|
+
query_name="transaction",
|
|
431
|
+
metrics=self._metrics,
|
|
432
|
+
labels=labels,
|
|
433
|
+
long_transaction_warning_seconds=(
|
|
434
|
+
self._config.defaults.long_transaction_warning_seconds
|
|
435
|
+
),
|
|
436
|
+
span=span,
|
|
437
|
+
)
|
|
438
|
+
async with manager as scope:
|
|
439
|
+
yield scope
|
|
440
|
+
|
|
441
|
+
@contextlib.asynccontextmanager
|
|
442
|
+
async def consistency_scope(
|
|
443
|
+
self, *, mode: Literal["read_your_writes"] = "read_your_writes"
|
|
444
|
+
) -> AsyncIterator[None]:
|
|
445
|
+
"""Force reads within this scope to the primary so they observe writes made earlier
|
|
446
|
+
in the same scope — read-your-writes over replica routing (§23)::
|
|
447
|
+
|
|
448
|
+
async with db.consistency_scope(mode="read_your_writes"):
|
|
449
|
+
await db.execute(write_query, target=write_target)
|
|
450
|
+
row = await db.fetch_one(read_query, target=read_target) # sees the write
|
|
451
|
+
|
|
452
|
+
The override is a ``contextvars.ContextVar``, so it never leaks across concurrent
|
|
453
|
+
operations in other tasks — but it also does not cross a thread boundary (a plain
|
|
454
|
+
``threading.Thread``, ``loop.run_in_executor``, or the sync ``Database`` facade driven
|
|
455
|
+
from a worker thread does not inherit it). See "Read-your-writes across threads" in
|
|
456
|
+
``docs/troubleshooting.md`` if you need the override to apply there too.
|
|
457
|
+
"""
|
|
458
|
+
token = _consistency_mode.set(mode)
|
|
459
|
+
try:
|
|
460
|
+
yield
|
|
461
|
+
finally:
|
|
462
|
+
_consistency_mode.reset(token)
|
|
463
|
+
|
|
464
|
+
# -- health & introspection --------------------------------------------------- #
|
|
465
|
+
|
|
466
|
+
async def health(self) -> HealthReport:
|
|
467
|
+
"""Readiness check: ``SELECT 1`` against every required target (§26)."""
|
|
468
|
+
targets: list[TargetHealth] = []
|
|
469
|
+
ready = True
|
|
470
|
+
for name, db in self._config.databases.items():
|
|
471
|
+
if not db.primary.required:
|
|
472
|
+
continue
|
|
473
|
+
try:
|
|
474
|
+
entry = await self._registry.get(
|
|
475
|
+
ResolvedRoute(database=name, shard_id="default", role="primary")
|
|
476
|
+
)
|
|
477
|
+
await ping(entry.engine, timeout=self._config.defaults.query_timeout_seconds)
|
|
478
|
+
targets.append(TargetHealth(key=f"{name}.primary", healthy=True))
|
|
479
|
+
except Exception as exc:
|
|
480
|
+
ready = False
|
|
481
|
+
targets.append(TargetHealth(key=f"{name}.primary", healthy=False, error=str(exc)))
|
|
482
|
+
return HealthReport(live=True, ready=ready, targets=targets)
|
|
483
|
+
|
|
484
|
+
def pool_status(self) -> list[PoolSnapshot]:
|
|
485
|
+
"""A point-in-time snapshot of every currently live engine's connection pool."""
|
|
486
|
+
return self._registry.snapshots()
|
|
487
|
+
|
|
488
|
+
async def drain_engine(self, key: str) -> bool:
|
|
489
|
+
"""Force-dispose one engine's idle pooled connections by its snapshot ``key`` (as printed
|
|
490
|
+
by :meth:`pool_status`, e.g. ``"prod:app:default:primary:psycopg"``), so the *next* call
|
|
491
|
+
routed to it rebuilds a fresh engine with fresh connections. Returns ``False`` if no live
|
|
492
|
+
engine currently has that key.
|
|
493
|
+
|
|
494
|
+
Useful right before a planned failover/topology change: rather than waiting for
|
|
495
|
+
connections to naturally recycle (``PoolConfig.recycle_seconds``), force them closed so
|
|
496
|
+
subsequent traffic reaches the new backend immediately. Only idle pooled connections are
|
|
497
|
+
closed — one already checked out by an in-flight call keeps working until released,
|
|
498
|
+
exactly like LRU eviction (`AsyncEngineRegistry(evict_lru=True)`).
|
|
499
|
+
|
|
500
|
+
This must be called from *within the running application process* — there is no
|
|
501
|
+
``dbkit`` CLI command for this, deliberately: each CLI invocation is a fresh, separate
|
|
502
|
+
process with its own empty engine registry, so it has no way to reach into an
|
|
503
|
+
already-running application's live engines. Wire this to a signal handler or an admin
|
|
504
|
+
HTTP route in your own application instead (see ``docs/troubleshooting.md``).
|
|
505
|
+
"""
|
|
506
|
+
return await self._registry.dispose_one(key)
|
|
507
|
+
|
|
508
|
+
# -- streaming ---------------------------------------------------------------- #
|
|
509
|
+
|
|
510
|
+
async def stream(
|
|
511
|
+
self,
|
|
512
|
+
query: object,
|
|
513
|
+
params: Mapping[str, Any] | None = None,
|
|
514
|
+
*,
|
|
515
|
+
target: DatabaseTarget,
|
|
516
|
+
batch_size: int = 1000,
|
|
517
|
+
map_to: Any = None,
|
|
518
|
+
max_duration: float | None = None,
|
|
519
|
+
) -> AsyncResultStream:
|
|
520
|
+
"""Stream a large result set with a server-side cursor, bounded memory (§20).
|
|
521
|
+
|
|
522
|
+
Usage::
|
|
523
|
+
|
|
524
|
+
async with await db.stream(EXPORT, params, target=t, batch_size=1000) as rows:
|
|
525
|
+
async for row in rows:
|
|
526
|
+
...
|
|
527
|
+
|
|
528
|
+
The stream owns its connection until the context exits, so it bypasses auto-retry.
|
|
529
|
+
"""
|
|
530
|
+
entry, _route = await self._executor.entry(target)
|
|
531
|
+
statement = coerce_statement(query)
|
|
532
|
+
query_name, operation, _q = self._executor.query_meta(query)
|
|
533
|
+
labels = self._executor.labels(entry, query_name, operation)
|
|
534
|
+
return AsyncResultStream(
|
|
535
|
+
entry.engine,
|
|
536
|
+
statement,
|
|
537
|
+
params,
|
|
538
|
+
batch_size=batch_size,
|
|
539
|
+
map_to=map_to,
|
|
540
|
+
database=entry.key.database,
|
|
541
|
+
role=entry.key.role,
|
|
542
|
+
query_name=query_name,
|
|
543
|
+
metrics=self._metrics,
|
|
544
|
+
labels=labels,
|
|
545
|
+
max_duration=max_duration,
|
|
546
|
+
tracer=self._tracer,
|
|
547
|
+
shard_id=entry.key.shard_id,
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
# -- bulk writes -------------------------------------------------------------- #
|
|
551
|
+
|
|
552
|
+
async def insert_many(
|
|
553
|
+
self,
|
|
554
|
+
table: Table,
|
|
555
|
+
rows: Sequence[Mapping[str, Any]],
|
|
556
|
+
*,
|
|
557
|
+
target: DatabaseTarget,
|
|
558
|
+
mode: bulk_mod.FailureMode = "atomic",
|
|
559
|
+
strategy: bulk_mod.InsertStrategy = "execute_many",
|
|
560
|
+
batch_size: int | None = None,
|
|
561
|
+
timeout: float | None = None,
|
|
562
|
+
) -> ExecutionResult:
|
|
563
|
+
"""Insert many rows in adaptively-sized batches (§19). ``rows`` are param dicts.
|
|
564
|
+
|
|
565
|
+
``strategy="unnest"`` (PostgreSQL only) binds one array parameter per column instead
|
|
566
|
+
of one parameter per column per row, so batch size isn't limited by the bind-parameter
|
|
567
|
+
ceiling — a mid-tier option between ``execute_many`` and COPY.
|
|
568
|
+
"""
|
|
569
|
+
return await self._bulk_write(
|
|
570
|
+
table,
|
|
571
|
+
insert(table),
|
|
572
|
+
rows,
|
|
573
|
+
target=target,
|
|
574
|
+
mode=mode,
|
|
575
|
+
strategy=strategy,
|
|
576
|
+
batch_size=batch_size,
|
|
577
|
+
timeout=timeout,
|
|
578
|
+
query_name=f"{table.name}.insert_many",
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
async def upsert_many(
|
|
582
|
+
self,
|
|
583
|
+
table: Table,
|
|
584
|
+
rows: Sequence[Mapping[str, Any]],
|
|
585
|
+
*,
|
|
586
|
+
target: DatabaseTarget,
|
|
587
|
+
conflict_index_elements: Sequence[str],
|
|
588
|
+
update_columns: Sequence[str] | None = None,
|
|
589
|
+
mode: bulk_mod.FailureMode = "atomic",
|
|
590
|
+
strategy: bulk_mod.InsertStrategy = "execute_many",
|
|
591
|
+
batch_size: int | None = None,
|
|
592
|
+
timeout: float | None = None,
|
|
593
|
+
) -> ExecutionResult:
|
|
594
|
+
"""Upsert many rows via PostgreSQL ``ON CONFLICT`` (§19). ``update_columns=None`` means
|
|
595
|
+
``DO NOTHING``; otherwise those columns are updated from the proposed row."""
|
|
596
|
+
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
597
|
+
|
|
598
|
+
base = pg_insert(table)
|
|
599
|
+
if update_columns is None:
|
|
600
|
+
stmt = base.on_conflict_do_nothing(index_elements=list(conflict_index_elements))
|
|
601
|
+
else:
|
|
602
|
+
stmt = base.on_conflict_do_update(
|
|
603
|
+
index_elements=list(conflict_index_elements),
|
|
604
|
+
set_={c: base.excluded[c] for c in update_columns},
|
|
605
|
+
)
|
|
606
|
+
return await self._bulk_write(
|
|
607
|
+
table,
|
|
608
|
+
stmt,
|
|
609
|
+
rows,
|
|
610
|
+
target=target,
|
|
611
|
+
mode=mode,
|
|
612
|
+
strategy=strategy,
|
|
613
|
+
batch_size=batch_size,
|
|
614
|
+
timeout=timeout,
|
|
615
|
+
query_name=f"{table.name}.upsert_many",
|
|
616
|
+
conflict_index_elements=conflict_index_elements,
|
|
617
|
+
update_columns=update_columns,
|
|
618
|
+
)
|
|
619
|
+
|
|
620
|
+
async def _bulk_write(
|
|
621
|
+
self,
|
|
622
|
+
table: Table,
|
|
623
|
+
statement: Any,
|
|
624
|
+
rows: Sequence[Mapping[str, Any]],
|
|
625
|
+
*,
|
|
626
|
+
target: DatabaseTarget,
|
|
627
|
+
mode: bulk_mod.FailureMode,
|
|
628
|
+
batch_size: int | None,
|
|
629
|
+
timeout: float | None,
|
|
630
|
+
query_name: str,
|
|
631
|
+
strategy: bulk_mod.InsertStrategy = "execute_many",
|
|
632
|
+
conflict_index_elements: Sequence[str] | None = None,
|
|
633
|
+
update_columns: Sequence[str] | None = None,
|
|
634
|
+
) -> ExecutionResult:
|
|
635
|
+
start = time.monotonic()
|
|
636
|
+
if not rows:
|
|
637
|
+
return ExecutionResult(
|
|
638
|
+
row_count=0, query_name=query_name, database_name=target.database, duration_ms=0.0
|
|
639
|
+
)
|
|
640
|
+
entry, _route = await self._executor.entry(target)
|
|
641
|
+
labels = self._executor.labels(entry, query_name, "write")
|
|
642
|
+
limiter = self._executor.limiter_for(entry.key.database)
|
|
643
|
+
bulk_cfg = self._config.defaults.bulk
|
|
644
|
+
columns = bulk_mod.column_names(list(rows))
|
|
645
|
+
|
|
646
|
+
if strategy == "unnest":
|
|
647
|
+
if not self._executor.is_postgres(entry):
|
|
648
|
+
raise DatabaseUnsupportedOperationError(
|
|
649
|
+
"strategy='unnest' requires PostgreSQL; this target uses a different dialect"
|
|
650
|
+
)
|
|
651
|
+
column_types = {
|
|
652
|
+
c: table.c[c].type.compile(dialect=postgresql.dialect()) # type: ignore[no-untyped-call]
|
|
653
|
+
for c in columns
|
|
654
|
+
}
|
|
655
|
+
unnest_stmt = unnest_mod.unnest_insert_sql(
|
|
656
|
+
table.name,
|
|
657
|
+
columns,
|
|
658
|
+
column_types,
|
|
659
|
+
conflict_index_elements=conflict_index_elements,
|
|
660
|
+
update_columns=update_columns,
|
|
661
|
+
)
|
|
662
|
+
plan = _BulkPlan(unnest_stmt, columns=columns)
|
|
663
|
+
else:
|
|
664
|
+
plan = _BulkPlan(statement, columns=None)
|
|
665
|
+
|
|
666
|
+
batch_rows = bulk_mod.resolve_batch_rows(
|
|
667
|
+
len(columns),
|
|
668
|
+
batch_size or bulk_cfg.default_batch_rows,
|
|
669
|
+
bulk_mod.BulkLimits(
|
|
670
|
+
max_rows=bulk_cfg.max_batch_rows, max_payload_bytes=bulk_cfg.max_payload_bytes
|
|
671
|
+
),
|
|
672
|
+
sample_row=rows[0] if rows else None,
|
|
673
|
+
)
|
|
674
|
+
batches = list(bulk_mod.iter_batches(list(rows), batch_rows))
|
|
675
|
+
self._metrics.observe(m.BULK_BATCH_SIZE, batch_rows, labels=labels)
|
|
676
|
+
eff_timeout = timeout or self._config.defaults.transaction_timeout_seconds
|
|
677
|
+
|
|
678
|
+
written = 0
|
|
679
|
+
with self._tracer.span(
|
|
680
|
+
"dbkit.bulk_write",
|
|
681
|
+
operation_type="write",
|
|
682
|
+
query_name=query_name,
|
|
683
|
+
database=entry.key.database,
|
|
684
|
+
shard=entry.key.shard_id,
|
|
685
|
+
role=entry.key.role,
|
|
686
|
+
) as span:
|
|
687
|
+
async with limiter.acquire("database"), limiter.acquire("bulk"):
|
|
688
|
+
if mode == "atomic":
|
|
689
|
+
async with self.transaction(target=target, timeout=eff_timeout) as tx:
|
|
690
|
+
for batch in batches:
|
|
691
|
+
stmt, params = plan.for_batch(batch)
|
|
692
|
+
try:
|
|
693
|
+
cursor = await run_execute(
|
|
694
|
+
tx.raw,
|
|
695
|
+
stmt,
|
|
696
|
+
params,
|
|
697
|
+
timeout=eff_timeout,
|
|
698
|
+
is_postgres=self._executor.is_postgres(entry),
|
|
699
|
+
)
|
|
700
|
+
except Exception as exc:
|
|
701
|
+
raise classify(
|
|
702
|
+
exc, query_name=query_name, database_name=target.database
|
|
703
|
+
) from exc
|
|
704
|
+
written += (
|
|
705
|
+
cursor.rowcount
|
|
706
|
+
if cursor.rowcount and cursor.rowcount > 0
|
|
707
|
+
else len(batch)
|
|
708
|
+
)
|
|
709
|
+
else:
|
|
710
|
+
written = await self._bulk_best_effort(
|
|
711
|
+
entry,
|
|
712
|
+
plan,
|
|
713
|
+
batches,
|
|
714
|
+
mode=mode,
|
|
715
|
+
timeout=eff_timeout,
|
|
716
|
+
query_name=query_name,
|
|
717
|
+
)
|
|
718
|
+
span.set_attribute("db.rows_affected", written)
|
|
719
|
+
|
|
720
|
+
self._metrics.incr(m.BULK_ROWS, written, labels=labels)
|
|
721
|
+
return ExecutionResult(
|
|
722
|
+
row_count=written,
|
|
723
|
+
query_name=query_name,
|
|
724
|
+
database_name=target.database,
|
|
725
|
+
duration_ms=(time.monotonic() - start) * 1000,
|
|
726
|
+
)
|
|
727
|
+
|
|
728
|
+
async def _bulk_best_effort(
|
|
729
|
+
self,
|
|
730
|
+
entry: EngineEntry,
|
|
731
|
+
plan: _BulkPlan,
|
|
732
|
+
batches: list[Any],
|
|
733
|
+
*,
|
|
734
|
+
mode: bulk_mod.FailureMode,
|
|
735
|
+
timeout: float,
|
|
736
|
+
query_name: str,
|
|
737
|
+
) -> int:
|
|
738
|
+
"""best_effort: each batch commits independently; split_on_failure additionally retries a
|
|
739
|
+
failed batch row-by-row to isolate the bad rows (§19.3)."""
|
|
740
|
+
target = DatabaseTarget(database=entry.key.database, role="write")
|
|
741
|
+
written = 0
|
|
742
|
+
for batch in batches:
|
|
743
|
+
try:
|
|
744
|
+
stmt, params = plan.for_batch(batch)
|
|
745
|
+
async with self._executor.scope(
|
|
746
|
+
target, query_name, call_timeout=timeout, deadline=None, commit=True
|
|
747
|
+
) as scope:
|
|
748
|
+
cursor = await run_execute(
|
|
749
|
+
scope.raw,
|
|
750
|
+
stmt,
|
|
751
|
+
params,
|
|
752
|
+
timeout=timeout,
|
|
753
|
+
is_postgres=self._executor.is_postgres(entry),
|
|
754
|
+
)
|
|
755
|
+
written += (
|
|
756
|
+
cursor.rowcount if cursor.rowcount and cursor.rowcount > 0 else len(batch)
|
|
757
|
+
)
|
|
758
|
+
except Exception as exc:
|
|
759
|
+
if mode != "split_on_failure":
|
|
760
|
+
# best_effort: this whole batch is dropped, never retried -- the only
|
|
761
|
+
# signal a caller gets is this log line + metric (performance review §9).
|
|
762
|
+
category = classify(exc, query_name=query_name).category.value
|
|
763
|
+
obslog.bulk_batch_dropped_warning(
|
|
764
|
+
query_name=query_name,
|
|
765
|
+
database=entry.key.database,
|
|
766
|
+
mode=mode,
|
|
767
|
+
rows_dropped=len(batch),
|
|
768
|
+
error_category=category,
|
|
769
|
+
)
|
|
770
|
+
self._metrics.incr(
|
|
771
|
+
m.BULK_ROWS_DROPPED,
|
|
772
|
+
len(batch),
|
|
773
|
+
labels={"database": entry.key.database, "query_name": query_name},
|
|
774
|
+
)
|
|
775
|
+
continue
|
|
776
|
+
for row in batch:
|
|
777
|
+
try:
|
|
778
|
+
row_stmt, row_params = plan.for_batch([row])
|
|
779
|
+
async with self._executor.scope(
|
|
780
|
+
target, query_name, call_timeout=timeout, deadline=None, commit=True
|
|
781
|
+
) as scope:
|
|
782
|
+
await run_execute(
|
|
783
|
+
scope.raw,
|
|
784
|
+
row_stmt,
|
|
785
|
+
row_params,
|
|
786
|
+
timeout=timeout,
|
|
787
|
+
is_postgres=self._executor.is_postgres(entry),
|
|
788
|
+
)
|
|
789
|
+
written += 1
|
|
790
|
+
except Exception as row_exc:
|
|
791
|
+
# split_on_failure: this one row is dropped after its batch already
|
|
792
|
+
# failed -- same silent-loss risk, narrowed to a single row.
|
|
793
|
+
category = classify(row_exc, query_name=query_name).category.value
|
|
794
|
+
obslog.bulk_batch_dropped_warning(
|
|
795
|
+
query_name=query_name,
|
|
796
|
+
database=entry.key.database,
|
|
797
|
+
mode=mode,
|
|
798
|
+
rows_dropped=1,
|
|
799
|
+
error_category=category,
|
|
800
|
+
)
|
|
801
|
+
self._metrics.incr(
|
|
802
|
+
m.BULK_ROWS_DROPPED,
|
|
803
|
+
1,
|
|
804
|
+
labels={"database": entry.key.database, "query_name": query_name},
|
|
805
|
+
)
|
|
806
|
+
continue
|
|
807
|
+
return written
|
|
808
|
+
|
|
809
|
+
# -- COPY (PostgreSQL fast bulk ingest) --------------------------------------- #
|
|
810
|
+
|
|
811
|
+
async def copy_records(
|
|
812
|
+
self,
|
|
813
|
+
table: str,
|
|
814
|
+
columns: Sequence[str],
|
|
815
|
+
records: Any,
|
|
816
|
+
*,
|
|
817
|
+
target: DatabaseTarget,
|
|
818
|
+
timeout: float | None = None,
|
|
819
|
+
) -> ExecutionResult:
|
|
820
|
+
"""Bulk-ingest ``records`` into ``table`` via PostgreSQL COPY (§19.2).
|
|
821
|
+
|
|
822
|
+
``records`` is an iterable (async or sync) of row sequences matching ``columns``.
|
|
823
|
+
Fastest path for large ingests; PostgreSQL + psycopg only.
|
|
824
|
+
"""
|
|
825
|
+
entry, _route = await self._executor.entry(target)
|
|
826
|
+
if not self._executor.is_postgres(entry):
|
|
827
|
+
raise DatabaseUnsupportedOperationError("COPY is only supported on PostgreSQL")
|
|
828
|
+
labels = self._executor.labels(entry, f"{table}.copy", "write")
|
|
829
|
+
limiter = self._executor.limiter_for(entry.key.database)
|
|
830
|
+
eff_timeout = timeout or self._config.defaults.transaction_timeout_seconds
|
|
831
|
+
start = time.monotonic()
|
|
832
|
+
written = 0
|
|
833
|
+
# COPY runs on the raw driver connection, so it must sit inside an explicit
|
|
834
|
+
# transaction (which begins the SQLAlchemy transaction) for the commit to persist it.
|
|
835
|
+
with self._tracer.span(
|
|
836
|
+
"dbkit.copy",
|
|
837
|
+
operation_type="write",
|
|
838
|
+
query_name=f"{table}.copy",
|
|
839
|
+
database=entry.key.database,
|
|
840
|
+
shard=entry.key.shard_id,
|
|
841
|
+
role=entry.key.role,
|
|
842
|
+
) as span:
|
|
843
|
+
async with (
|
|
844
|
+
limiter.acquire("database"),
|
|
845
|
+
limiter.acquire("bulk"),
|
|
846
|
+
self.transaction(target=target, timeout=eff_timeout) as tx,
|
|
847
|
+
):
|
|
848
|
+
try:
|
|
849
|
+
written = await copy_from_records(tx.raw, table, list(columns), records)
|
|
850
|
+
except Exception as exc:
|
|
851
|
+
raise classify(
|
|
852
|
+
exc, query_name=f"{table}.copy", database_name=target.database
|
|
853
|
+
) from exc
|
|
854
|
+
span.set_attribute("db.rows_affected", written)
|
|
855
|
+
self._metrics.incr(m.BULK_ROWS, written, labels=labels)
|
|
856
|
+
return ExecutionResult(
|
|
857
|
+
row_count=written,
|
|
858
|
+
query_name=f"{table}.copy",
|
|
859
|
+
database_name=target.database,
|
|
860
|
+
duration_ms=(time.monotonic() - start) * 1000,
|
|
861
|
+
)
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
class _BulkPlan:
|
|
865
|
+
"""How to turn a batch of row-dicts into ``(statement, params)`` for one execution —
|
|
866
|
+
abstracts over ``execute_many`` (list-of-dicts params) vs ``unnest`` (one array bind per
|
|
867
|
+
column) so :meth:`AsyncDatabase._bulk_write` / `_bulk_best_effort` stay strategy-agnostic.
|
|
868
|
+
"""
|
|
869
|
+
|
|
870
|
+
def __init__(self, statement: Any, *, columns: Sequence[str] | None) -> None:
|
|
871
|
+
"""``columns`` non-``None`` selects the ``unnest`` strategy; ``None`` selects
|
|
872
|
+
``execute_many``."""
|
|
873
|
+
self._statement = statement
|
|
874
|
+
self._columns = columns
|
|
875
|
+
|
|
876
|
+
def for_batch(self, batch: Sequence[Mapping[str, Any]]) -> tuple[Any, Any]:
|
|
877
|
+
"""The ``(statement, params)`` pair to execute for this batch."""
|
|
878
|
+
if self._columns is not None:
|
|
879
|
+
return self._statement, unnest_mod.columnar_params(batch, self._columns)
|
|
880
|
+
return self._statement, list(batch)
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
def _make_default_metrics(config: DbkitConfig) -> MetricsSink:
|
|
884
|
+
return NoopMetrics()
|