dbkit-sql 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. dbkit/__init__.py +77 -0
  2. dbkit/_async/__init__.py +17 -0
  3. dbkit/_async/_compat.py +111 -0
  4. dbkit/_async/connection.py +328 -0
  5. dbkit/_async/database.py +884 -0
  6. dbkit/_async/engine.py +232 -0
  7. dbkit/_async/executor.py +329 -0
  8. dbkit/_async/health.py +53 -0
  9. dbkit/_async/resilience.py +148 -0
  10. dbkit/_async/streaming.py +137 -0
  11. dbkit/_async/transaction.py +264 -0
  12. dbkit/_core/__init__.py +1 -0
  13. dbkit/_core/bulk.py +91 -0
  14. dbkit/_core/circuit.py +130 -0
  15. dbkit/_core/config.py +613 -0
  16. dbkit/_core/errors/__init__.py +74 -0
  17. dbkit/_core/errors/base.py +344 -0
  18. dbkit/_core/errors/classifier.py +138 -0
  19. dbkit/_core/errors/redaction.py +96 -0
  20. dbkit/_core/errors/sqlstate.py +74 -0
  21. dbkit/_core/idempotency_lint.py +57 -0
  22. dbkit/_core/keys.py +29 -0
  23. dbkit/_core/policies.py +105 -0
  24. dbkit/_core/query.py +142 -0
  25. dbkit/_core/result.py +109 -0
  26. dbkit/_core/routing.py +248 -0
  27. dbkit/_pool.py +202 -0
  28. dbkit/_sync/__init__.py +20 -0
  29. dbkit/_sync/_compat.py +82 -0
  30. dbkit/_sync/connection.py +331 -0
  31. dbkit/_sync/database.py +887 -0
  32. dbkit/_sync/engine.py +235 -0
  33. dbkit/_sync/executor.py +332 -0
  34. dbkit/_sync/health.py +56 -0
  35. dbkit/_sync/resilience.py +151 -0
  36. dbkit/_sync/streaming.py +140 -0
  37. dbkit/_sync/transaction.py +267 -0
  38. dbkit/cli/__init__.py +1 -0
  39. dbkit/cli/main.py +256 -0
  40. dbkit/errors.py +6 -0
  41. dbkit/integrations/__init__.py +21 -0
  42. dbkit/integrations/batch.py +81 -0
  43. dbkit/integrations/inbox.py +153 -0
  44. dbkit/observability/__init__.py +21 -0
  45. dbkit/observability/logging.py +154 -0
  46. dbkit/observability/metrics.py +110 -0
  47. dbkit/observability/otel_metrics.py +69 -0
  48. dbkit/observability/prometheus.py +78 -0
  49. dbkit/observability/tracing.py +128 -0
  50. dbkit/postgres/__init__.py +1 -0
  51. dbkit/postgres/copy.py +77 -0
  52. dbkit/postgres/pipeline.py +48 -0
  53. dbkit/postgres/unnest.py +73 -0
  54. dbkit/testing/__init__.py +1 -0
  55. dbkit_sql-0.1.0.dist-info/METADATA +555 -0
  56. dbkit_sql-0.1.0.dist-info/RECORD +59 -0
  57. dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
  58. dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
  59. dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,884 @@
1
+ """The ``AsyncDatabase`` facade — the primary public entrypoint (§8.1).
2
+
3
+ Orchestrates target resolution, engine lookup, connection acquisition (measuring pool wait),
4
+ execution, resilience (retries/circuit breaker/concurrency limits), streaming, bulk writes,
5
+ COPY, metrics, and graceful startup/shutdown.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import contextlib
11
+ import contextvars
12
+ import logging
13
+ import time
14
+ from collections.abc import AsyncIterator, Mapping, Sequence
15
+ from typing import Any, Literal
16
+
17
+ from sqlalchemy import Table, insert
18
+ from sqlalchemy.dialects import postgresql
19
+
20
+ from .._core import bulk as bulk_mod
21
+ from .._core.config import DbkitConfig
22
+ from .._core.errors import (
23
+ DatabaseError,
24
+ DatabaseRoutingError,
25
+ DatabaseUnsupportedOperationError,
26
+ classify,
27
+ )
28
+ from .._core.query import coerce_statement
29
+ from .._core.result import ExecutionResult
30
+ from .._core.routing import (
31
+ DatabaseTarget,
32
+ ReplicaSelector,
33
+ ResolvedRoute,
34
+ RoundRobinReplicaSelector,
35
+ ShardResolver,
36
+ SingleShardResolver,
37
+ )
38
+ from .._pool import PoolSnapshot
39
+ from ..observability import logging as obslog
40
+ from ..observability import metrics as m
41
+ from ..observability.metrics import MetricsSink, NoopMetrics
42
+ from ..observability.tracing import Tracer, make_tracer
43
+ from ..postgres import unnest as unnest_mod
44
+ from ._compat import copy_from_records
45
+ from .connection import AsyncConnectionScope, run_execute
46
+ from .engine import AsyncEngineRegistry, EngineEntry
47
+ from .executor import ResilientExecutor
48
+ from .health import HealthReport, TargetHealth, ping
49
+ from .streaming import AsyncResultStream
50
+ from .transaction import _AsyncTransactionManager
51
+
52
+ #: Task-local (safe across concurrent operations) consistency-scope override for read routing.
53
+ _consistency_mode: contextvars.ContextVar[str | None] = contextvars.ContextVar(
54
+ "dbkit_consistency_mode", default=None
55
+ )
56
+
57
+
58
+ class AsyncDatabase:
59
+ """Async, SQL-first database facade over SQLAlchemy Core (§8.1)."""
60
+
61
+ def __init__(
62
+ self,
63
+ config: DbkitConfig,
64
+ *,
65
+ metrics: MetricsSink | None = None,
66
+ tracer: Tracer | None = None,
67
+ shard_resolver: ShardResolver | None = None,
68
+ replica_selector: ReplicaSelector | None = None,
69
+ ) -> None:
70
+ """Build a facade from an already-validated :class:`DbkitConfig`.
71
+
72
+ Prefer :meth:`from_config` unless you need to pass a config object you built/validated
73
+ yourself. Does not connect to any database — call :meth:`start` before use.
74
+ """
75
+ self._config = config
76
+ if metrics is None:
77
+ metrics = (
78
+ _make_default_metrics(config)
79
+ if config.defaults.observability.metrics
80
+ else NoopMetrics()
81
+ )
82
+ self._metrics = metrics
83
+ self._tracer = tracer or make_tracer(config.defaults.observability.tracing)
84
+ self._registry = AsyncEngineRegistry(
85
+ config,
86
+ metrics=self._metrics,
87
+ max_engines=config.max_engines,
88
+ evict_lru=config.evict_lru_engines,
89
+ )
90
+ self._shards = shard_resolver or SingleShardResolver()
91
+ self._replicas = replica_selector or RoundRobinReplicaSelector(
92
+ {name: [r.name for r in db.replicas] for name, db in config.databases.items()}
93
+ )
94
+ self._executor = ResilientExecutor(
95
+ config,
96
+ registry=self._registry,
97
+ resolve=self._resolve,
98
+ metrics=self._metrics,
99
+ tracer=self._tracer,
100
+ )
101
+ self._started = False
102
+
103
+ # -- construction ------------------------------------------------------------- #
104
+
105
+ @classmethod
106
+ def from_config(
107
+ cls,
108
+ config: DbkitConfig | Mapping[str, Any],
109
+ *,
110
+ metrics: MetricsSink | None = None,
111
+ tracer: Tracer | None = None,
112
+ shard_resolver: ShardResolver | None = None,
113
+ replica_selector: ReplicaSelector | None = None,
114
+ ) -> AsyncDatabase:
115
+ """Build a facade from a :class:`DbkitConfig` or a plain dict (validated via
116
+ :meth:`DbkitConfig.from_dict`). The usual entry point."""
117
+ cfg = config if isinstance(config, DbkitConfig) else DbkitConfig.from_dict(config)
118
+ return cls(
119
+ cfg,
120
+ metrics=metrics,
121
+ tracer=tracer,
122
+ shard_resolver=shard_resolver,
123
+ replica_selector=replica_selector,
124
+ )
125
+
126
+ @property
127
+ def config(self) -> DbkitConfig:
128
+ """The validated configuration this facade was built from."""
129
+ return self._config
130
+
131
+ @property
132
+ def engine_count(self) -> int:
133
+ """Number of live SQLAlchemy engines currently held (one per target)."""
134
+ return self._registry.count
135
+
136
+ # -- lifecycle ---------------------------------------------------------------- #
137
+
138
+ async def start(self, *, warm: bool = False) -> None:
139
+ """Create required engines and (optionally) warm connections (§27.1)."""
140
+ if self._started:
141
+ return
142
+ self._config.validate()
143
+ for name, db in self._config.databases.items():
144
+ if db.primary.required:
145
+ entry = await self._registry.get(
146
+ ResolvedRoute(database=name, shard_id="default", role="primary")
147
+ )
148
+ if warm:
149
+ await ping(entry.engine, timeout=self._config.defaults.query_timeout_seconds)
150
+ self._started = True
151
+ obslog.log_event(logging.INFO, "database.started", engines=self._registry.count)
152
+
153
+ async def require_ready(self) -> None:
154
+ """Raise unless every required target is reachable (§27.1)."""
155
+ report = await self.health()
156
+ if not report.ready:
157
+ failed = [t.key for t in report.targets if not t.healthy]
158
+ raise DatabaseError(f"required databases not ready: {failed}")
159
+
160
+ async def close(self, grace_period: float = 10.0) -> None:
161
+ """Dispose all engines (§27.2). ``grace_period`` reserved for in-flight draining."""
162
+ await self._registry.dispose_all()
163
+ self._started = False
164
+ obslog.log_event(logging.INFO, "database.closed")
165
+
166
+ async def __aenter__(self) -> AsyncDatabase:
167
+ """``async with AsyncDatabase(...) as db:`` — calls :meth:`start`."""
168
+ await self.start()
169
+ return self
170
+
171
+ async def __aexit__(self, *exc: object) -> None:
172
+ """Calls :meth:`close`."""
173
+ await self.close()
174
+
175
+ # -- routing ------------------------------------------------------------------ #
176
+
177
+ def _resolve(self, target: DatabaseTarget) -> ResolvedRoute:
178
+ db = self._config.databases.get(target.database)
179
+ if db is None:
180
+ raise DatabaseRoutingError(f"unknown database {target.database!r}")
181
+ shard_id = (
182
+ self._shards.resolve(target.database, target.shard_key)
183
+ if target.shard_key is not None
184
+ else "default"
185
+ )
186
+ # A consistency_scope(read_your_writes) override forces reads to the primary so they
187
+ # observe writes made earlier in the same scope (§23).
188
+ if target.wants_replica and db.replicas and _consistency_mode.get() is None:
189
+ replica_name = self._replicas.select(target.database, shard_id)
190
+ if replica_name is not None:
191
+ return ResolvedRoute(
192
+ database=target.database,
193
+ shard_id=shard_id,
194
+ role="replica",
195
+ replica_name=replica_name,
196
+ )
197
+ # Explicit fallback: selector found no replica for this database -> primary (§23).
198
+ return ResolvedRoute(database=target.database, shard_id=shard_id, role="primary")
199
+
200
+ # -- read API ----------------------------------------------------------------- #
201
+
202
+ async def fetch_all(
203
+ self,
204
+ query: object,
205
+ params: Mapping[str, Any] | None = None,
206
+ *,
207
+ target: DatabaseTarget,
208
+ map_to: Any = None,
209
+ timeout: float | None = None,
210
+ deadline: float | None = None,
211
+ ) -> list[Any]:
212
+ """Run a one-shot read, acquiring and releasing its own connection (§11.1).
213
+
214
+ Returns every row, mapped to ``map_to``, with no cardinality constraint. Retries and
215
+ the circuit breaker apply per the resolved ``target``'s policy.
216
+ """
217
+ rows: list[Any] = await self._executor.execute_with_resilience(
218
+ target,
219
+ query,
220
+ lambda scope: scope.fetch_all(query, params, map_to=map_to),
221
+ commit=False,
222
+ timeout=timeout,
223
+ deadline=deadline,
224
+ )
225
+ return rows
226
+
227
+ async def fetch_one(
228
+ self,
229
+ query: object,
230
+ params: Mapping[str, Any] | None = None,
231
+ *,
232
+ target: DatabaseTarget,
233
+ map_to: Any = None,
234
+ timeout: float | None = None,
235
+ deadline: float | None = None,
236
+ ) -> Any:
237
+ """Run a one-shot read expecting exactly one row.
238
+
239
+ Raises :class:`~dbkit.errors.DatabaseResultError` if zero or more than one row comes
240
+ back (via SQLAlchemy's own ``Result.one()``).
241
+ """
242
+ return await self._executor.execute_with_resilience(
243
+ target,
244
+ query,
245
+ lambda scope: scope.fetch_one(query, params, map_to=map_to),
246
+ commit=False,
247
+ timeout=timeout,
248
+ deadline=deadline,
249
+ )
250
+
251
+ async def fetch_optional(
252
+ self,
253
+ query: object,
254
+ params: Mapping[str, Any] | None = None,
255
+ *,
256
+ target: DatabaseTarget,
257
+ map_to: Any = None,
258
+ timeout: float | None = None,
259
+ deadline: float | None = None,
260
+ ) -> Any | None:
261
+ """Run a one-shot read expecting zero or one row; ``None`` if empty."""
262
+ return await self._executor.execute_with_resilience(
263
+ target,
264
+ query,
265
+ lambda scope: scope.fetch_optional(query, params, map_to=map_to),
266
+ commit=False,
267
+ timeout=timeout,
268
+ deadline=deadline,
269
+ )
270
+
271
+ async def fetch_value(
272
+ self,
273
+ query: object,
274
+ params: Mapping[str, Any] | None = None,
275
+ *,
276
+ target: DatabaseTarget,
277
+ timeout: float | None = None,
278
+ deadline: float | None = None,
279
+ ) -> Any:
280
+ """Run a one-shot read expecting exactly one row and return its first column."""
281
+ return await self._executor.execute_with_resilience(
282
+ target,
283
+ query,
284
+ lambda scope: scope.fetch_value(query, params),
285
+ commit=False,
286
+ timeout=timeout,
287
+ deadline=deadline,
288
+ )
289
+
290
+ async def fetch_values(
291
+ self,
292
+ query: object,
293
+ params: Mapping[str, Any] | None = None,
294
+ *,
295
+ target: DatabaseTarget,
296
+ timeout: float | None = None,
297
+ deadline: float | None = None,
298
+ ) -> list[Any]:
299
+ """Run a one-shot read and return the first column of every row."""
300
+ values: list[Any] = await self._executor.execute_with_resilience(
301
+ target,
302
+ query,
303
+ lambda scope: scope.fetch_values(query, params),
304
+ commit=False,
305
+ timeout=timeout,
306
+ deadline=deadline,
307
+ )
308
+ return values
309
+
310
+ # -- write API ---------------------------------------------------------------- #
311
+
312
+ async def execute(
313
+ self,
314
+ query: object,
315
+ params: Mapping[str, Any] | None = None,
316
+ *,
317
+ target: DatabaseTarget,
318
+ timeout: float | None = None,
319
+ deadline: float | None = None,
320
+ ) -> ExecutionResult:
321
+ """Run a one-shot write/DDL statement, auto-committed on success (§11.1)."""
322
+ query_name, _, _ = self._executor.query_meta(query)
323
+ start = time.monotonic()
324
+ row_count = await self._executor.execute_with_resilience(
325
+ target,
326
+ query,
327
+ lambda scope: scope.execute(query, params),
328
+ commit=True,
329
+ timeout=timeout,
330
+ deadline=deadline,
331
+ )
332
+ return ExecutionResult(
333
+ row_count=row_count,
334
+ query_name=query_name,
335
+ database_name=target.database,
336
+ duration_ms=(time.monotonic() - start) * 1000,
337
+ )
338
+
339
+ async def execute_many(
340
+ self,
341
+ query: object,
342
+ params_seq: Sequence[Mapping[str, Any]],
343
+ *,
344
+ target: DatabaseTarget,
345
+ timeout: float | None = None,
346
+ deadline: float | None = None,
347
+ ) -> ExecutionResult:
348
+ """Run one statement against many parameter sets (driver executemany)."""
349
+ query_name, _op, _q = self._executor.query_meta(query)
350
+ statement = coerce_statement(query)
351
+ start = time.monotonic()
352
+ async with self._executor.scope(
353
+ target, query, call_timeout=timeout, deadline=deadline, commit=True
354
+ ) as scope:
355
+ try:
356
+ cursor = await run_execute(
357
+ scope.raw,
358
+ statement,
359
+ list(params_seq),
360
+ timeout=scope._default_timeout,
361
+ is_postgres=scope._is_postgres,
362
+ )
363
+ row_count = cursor.rowcount
364
+ except Exception as exc:
365
+ raise classify(exc, query_name=query_name, database_name=target.database) from exc
366
+ return ExecutionResult(
367
+ row_count=row_count,
368
+ query_name=query_name,
369
+ database_name=target.database,
370
+ duration_ms=(time.monotonic() - start) * 1000,
371
+ )
372
+
373
+ # -- explicit scopes ---------------------------------------------------------- #
374
+
375
+ @contextlib.asynccontextmanager
376
+ async def connection(
377
+ self, *, target: DatabaseTarget, timeout: float | None = None
378
+ ) -> AsyncIterator[AsyncConnectionScope]:
379
+ """A held connection with commit-on-success / rollback-on-error semantics (§11.2)."""
380
+ async with self._executor.scope(
381
+ target, "adhoc", call_timeout=timeout, deadline=None, commit=True
382
+ ) as scope:
383
+ yield scope
384
+
385
+ @contextlib.asynccontextmanager
386
+ async def transaction(
387
+ self,
388
+ *,
389
+ target: DatabaseTarget,
390
+ isolation: str | None = None,
391
+ read_only: bool = False,
392
+ deferrable: bool = False,
393
+ timeout: float | None = None,
394
+ lock_timeout: float | None = None,
395
+ ) -> AsyncIterator[Any]:
396
+ """An explicit transaction with isolation/timeout options (§11.3).
397
+
398
+ ``deferrable`` (PostgreSQL only) is only meaningful together with
399
+ ``isolation="serializable"`` and ``read_only=True``: it lets a read-only serializable
400
+ transaction wait for a safe snapshot instead of risking a serialization failure.
401
+
402
+ Usage::
403
+
404
+ async with db.transaction(target=write_target) as tx:
405
+ await tx.execute(INSERT, params)
406
+ """
407
+ route = self._resolve(target)
408
+ entry = await self._registry.get(route)
409
+ labels = self._executor.labels(entry, "transaction", "transaction")
410
+ with self._tracer.span(
411
+ "dbkit.transaction",
412
+ operation_type="transaction",
413
+ query_name="transaction",
414
+ database=entry.key.database,
415
+ shard=entry.key.shard_id,
416
+ role=entry.key.role,
417
+ ) as span:
418
+ manager = _AsyncTransactionManager(
419
+ entry.engine,
420
+ is_postgres=self._executor.is_postgres(entry),
421
+ default_timeout=self._config.defaults.transaction_timeout_seconds,
422
+ database=entry.key.database,
423
+ shard_id=entry.key.shard_id,
424
+ role=entry.key.role,
425
+ isolation=isolation,
426
+ read_only=read_only,
427
+ deferrable=deferrable,
428
+ timeout=timeout,
429
+ lock_timeout=lock_timeout,
430
+ query_name="transaction",
431
+ metrics=self._metrics,
432
+ labels=labels,
433
+ long_transaction_warning_seconds=(
434
+ self._config.defaults.long_transaction_warning_seconds
435
+ ),
436
+ span=span,
437
+ )
438
+ async with manager as scope:
439
+ yield scope
440
+
441
+ @contextlib.asynccontextmanager
442
+ async def consistency_scope(
443
+ self, *, mode: Literal["read_your_writes"] = "read_your_writes"
444
+ ) -> AsyncIterator[None]:
445
+ """Force reads within this scope to the primary so they observe writes made earlier
446
+ in the same scope — read-your-writes over replica routing (§23)::
447
+
448
+ async with db.consistency_scope(mode="read_your_writes"):
449
+ await db.execute(write_query, target=write_target)
450
+ row = await db.fetch_one(read_query, target=read_target) # sees the write
451
+
452
+ The override is a ``contextvars.ContextVar``, so it never leaks across concurrent
453
+ operations in other tasks — but it also does not cross a thread boundary (a plain
454
+ ``threading.Thread``, ``loop.run_in_executor``, or the sync ``Database`` facade driven
455
+ from a worker thread does not inherit it). See "Read-your-writes across threads" in
456
+ ``docs/troubleshooting.md`` if you need the override to apply there too.
457
+ """
458
+ token = _consistency_mode.set(mode)
459
+ try:
460
+ yield
461
+ finally:
462
+ _consistency_mode.reset(token)
463
+
464
+ # -- health & introspection --------------------------------------------------- #
465
+
466
+ async def health(self) -> HealthReport:
467
+ """Readiness check: ``SELECT 1`` against every required target (§26)."""
468
+ targets: list[TargetHealth] = []
469
+ ready = True
470
+ for name, db in self._config.databases.items():
471
+ if not db.primary.required:
472
+ continue
473
+ try:
474
+ entry = await self._registry.get(
475
+ ResolvedRoute(database=name, shard_id="default", role="primary")
476
+ )
477
+ await ping(entry.engine, timeout=self._config.defaults.query_timeout_seconds)
478
+ targets.append(TargetHealth(key=f"{name}.primary", healthy=True))
479
+ except Exception as exc:
480
+ ready = False
481
+ targets.append(TargetHealth(key=f"{name}.primary", healthy=False, error=str(exc)))
482
+ return HealthReport(live=True, ready=ready, targets=targets)
483
+
484
+ def pool_status(self) -> list[PoolSnapshot]:
485
+ """A point-in-time snapshot of every currently live engine's connection pool."""
486
+ return self._registry.snapshots()
487
+
488
+ async def drain_engine(self, key: str) -> bool:
489
+ """Force-dispose one engine's idle pooled connections by its snapshot ``key`` (as printed
490
+ by :meth:`pool_status`, e.g. ``"prod:app:default:primary:psycopg"``), so the *next* call
491
+ routed to it rebuilds a fresh engine with fresh connections. Returns ``False`` if no live
492
+ engine currently has that key.
493
+
494
+ Useful right before a planned failover/topology change: rather than waiting for
495
+ connections to naturally recycle (``PoolConfig.recycle_seconds``), force them closed so
496
+ subsequent traffic reaches the new backend immediately. Only idle pooled connections are
497
+ closed — one already checked out by an in-flight call keeps working until released,
498
+ exactly like LRU eviction (`AsyncEngineRegistry(evict_lru=True)`).
499
+
500
+ This must be called from *within the running application process* — there is no
501
+ ``dbkit`` CLI command for this, deliberately: each CLI invocation is a fresh, separate
502
+ process with its own empty engine registry, so it has no way to reach into an
503
+ already-running application's live engines. Wire this to a signal handler or an admin
504
+ HTTP route in your own application instead (see ``docs/troubleshooting.md``).
505
+ """
506
+ return await self._registry.dispose_one(key)
507
+
508
+ # -- streaming ---------------------------------------------------------------- #
509
+
510
+ async def stream(
511
+ self,
512
+ query: object,
513
+ params: Mapping[str, Any] | None = None,
514
+ *,
515
+ target: DatabaseTarget,
516
+ batch_size: int = 1000,
517
+ map_to: Any = None,
518
+ max_duration: float | None = None,
519
+ ) -> AsyncResultStream:
520
+ """Stream a large result set with a server-side cursor, bounded memory (§20).
521
+
522
+ Usage::
523
+
524
+ async with await db.stream(EXPORT, params, target=t, batch_size=1000) as rows:
525
+ async for row in rows:
526
+ ...
527
+
528
+ The stream owns its connection until the context exits, so it bypasses auto-retry.
529
+ """
530
+ entry, _route = await self._executor.entry(target)
531
+ statement = coerce_statement(query)
532
+ query_name, operation, _q = self._executor.query_meta(query)
533
+ labels = self._executor.labels(entry, query_name, operation)
534
+ return AsyncResultStream(
535
+ entry.engine,
536
+ statement,
537
+ params,
538
+ batch_size=batch_size,
539
+ map_to=map_to,
540
+ database=entry.key.database,
541
+ role=entry.key.role,
542
+ query_name=query_name,
543
+ metrics=self._metrics,
544
+ labels=labels,
545
+ max_duration=max_duration,
546
+ tracer=self._tracer,
547
+ shard_id=entry.key.shard_id,
548
+ )
549
+
550
+ # -- bulk writes -------------------------------------------------------------- #
551
+
552
+ async def insert_many(
553
+ self,
554
+ table: Table,
555
+ rows: Sequence[Mapping[str, Any]],
556
+ *,
557
+ target: DatabaseTarget,
558
+ mode: bulk_mod.FailureMode = "atomic",
559
+ strategy: bulk_mod.InsertStrategy = "execute_many",
560
+ batch_size: int | None = None,
561
+ timeout: float | None = None,
562
+ ) -> ExecutionResult:
563
+ """Insert many rows in adaptively-sized batches (§19). ``rows`` are param dicts.
564
+
565
+ ``strategy="unnest"`` (PostgreSQL only) binds one array parameter per column instead
566
+ of one parameter per column per row, so batch size isn't limited by the bind-parameter
567
+ ceiling — a mid-tier option between ``execute_many`` and COPY.
568
+ """
569
+ return await self._bulk_write(
570
+ table,
571
+ insert(table),
572
+ rows,
573
+ target=target,
574
+ mode=mode,
575
+ strategy=strategy,
576
+ batch_size=batch_size,
577
+ timeout=timeout,
578
+ query_name=f"{table.name}.insert_many",
579
+ )
580
+
581
+ async def upsert_many(
582
+ self,
583
+ table: Table,
584
+ rows: Sequence[Mapping[str, Any]],
585
+ *,
586
+ target: DatabaseTarget,
587
+ conflict_index_elements: Sequence[str],
588
+ update_columns: Sequence[str] | None = None,
589
+ mode: bulk_mod.FailureMode = "atomic",
590
+ strategy: bulk_mod.InsertStrategy = "execute_many",
591
+ batch_size: int | None = None,
592
+ timeout: float | None = None,
593
+ ) -> ExecutionResult:
594
+ """Upsert many rows via PostgreSQL ``ON CONFLICT`` (§19). ``update_columns=None`` means
595
+ ``DO NOTHING``; otherwise those columns are updated from the proposed row."""
596
+ from sqlalchemy.dialects.postgresql import insert as pg_insert
597
+
598
+ base = pg_insert(table)
599
+ if update_columns is None:
600
+ stmt = base.on_conflict_do_nothing(index_elements=list(conflict_index_elements))
601
+ else:
602
+ stmt = base.on_conflict_do_update(
603
+ index_elements=list(conflict_index_elements),
604
+ set_={c: base.excluded[c] for c in update_columns},
605
+ )
606
+ return await self._bulk_write(
607
+ table,
608
+ stmt,
609
+ rows,
610
+ target=target,
611
+ mode=mode,
612
+ strategy=strategy,
613
+ batch_size=batch_size,
614
+ timeout=timeout,
615
+ query_name=f"{table.name}.upsert_many",
616
+ conflict_index_elements=conflict_index_elements,
617
+ update_columns=update_columns,
618
+ )
619
+
620
+ async def _bulk_write(
621
+ self,
622
+ table: Table,
623
+ statement: Any,
624
+ rows: Sequence[Mapping[str, Any]],
625
+ *,
626
+ target: DatabaseTarget,
627
+ mode: bulk_mod.FailureMode,
628
+ batch_size: int | None,
629
+ timeout: float | None,
630
+ query_name: str,
631
+ strategy: bulk_mod.InsertStrategy = "execute_many",
632
+ conflict_index_elements: Sequence[str] | None = None,
633
+ update_columns: Sequence[str] | None = None,
634
+ ) -> ExecutionResult:
635
+ start = time.monotonic()
636
+ if not rows:
637
+ return ExecutionResult(
638
+ row_count=0, query_name=query_name, database_name=target.database, duration_ms=0.0
639
+ )
640
+ entry, _route = await self._executor.entry(target)
641
+ labels = self._executor.labels(entry, query_name, "write")
642
+ limiter = self._executor.limiter_for(entry.key.database)
643
+ bulk_cfg = self._config.defaults.bulk
644
+ columns = bulk_mod.column_names(list(rows))
645
+
646
+ if strategy == "unnest":
647
+ if not self._executor.is_postgres(entry):
648
+ raise DatabaseUnsupportedOperationError(
649
+ "strategy='unnest' requires PostgreSQL; this target uses a different dialect"
650
+ )
651
+ column_types = {
652
+ c: table.c[c].type.compile(dialect=postgresql.dialect()) # type: ignore[no-untyped-call]
653
+ for c in columns
654
+ }
655
+ unnest_stmt = unnest_mod.unnest_insert_sql(
656
+ table.name,
657
+ columns,
658
+ column_types,
659
+ conflict_index_elements=conflict_index_elements,
660
+ update_columns=update_columns,
661
+ )
662
+ plan = _BulkPlan(unnest_stmt, columns=columns)
663
+ else:
664
+ plan = _BulkPlan(statement, columns=None)
665
+
666
+ batch_rows = bulk_mod.resolve_batch_rows(
667
+ len(columns),
668
+ batch_size or bulk_cfg.default_batch_rows,
669
+ bulk_mod.BulkLimits(
670
+ max_rows=bulk_cfg.max_batch_rows, max_payload_bytes=bulk_cfg.max_payload_bytes
671
+ ),
672
+ sample_row=rows[0] if rows else None,
673
+ )
674
+ batches = list(bulk_mod.iter_batches(list(rows), batch_rows))
675
+ self._metrics.observe(m.BULK_BATCH_SIZE, batch_rows, labels=labels)
676
+ eff_timeout = timeout or self._config.defaults.transaction_timeout_seconds
677
+
678
+ written = 0
679
+ with self._tracer.span(
680
+ "dbkit.bulk_write",
681
+ operation_type="write",
682
+ query_name=query_name,
683
+ database=entry.key.database,
684
+ shard=entry.key.shard_id,
685
+ role=entry.key.role,
686
+ ) as span:
687
+ async with limiter.acquire("database"), limiter.acquire("bulk"):
688
+ if mode == "atomic":
689
+ async with self.transaction(target=target, timeout=eff_timeout) as tx:
690
+ for batch in batches:
691
+ stmt, params = plan.for_batch(batch)
692
+ try:
693
+ cursor = await run_execute(
694
+ tx.raw,
695
+ stmt,
696
+ params,
697
+ timeout=eff_timeout,
698
+ is_postgres=self._executor.is_postgres(entry),
699
+ )
700
+ except Exception as exc:
701
+ raise classify(
702
+ exc, query_name=query_name, database_name=target.database
703
+ ) from exc
704
+ written += (
705
+ cursor.rowcount
706
+ if cursor.rowcount and cursor.rowcount > 0
707
+ else len(batch)
708
+ )
709
+ else:
710
+ written = await self._bulk_best_effort(
711
+ entry,
712
+ plan,
713
+ batches,
714
+ mode=mode,
715
+ timeout=eff_timeout,
716
+ query_name=query_name,
717
+ )
718
+ span.set_attribute("db.rows_affected", written)
719
+
720
+ self._metrics.incr(m.BULK_ROWS, written, labels=labels)
721
+ return ExecutionResult(
722
+ row_count=written,
723
+ query_name=query_name,
724
+ database_name=target.database,
725
+ duration_ms=(time.monotonic() - start) * 1000,
726
+ )
727
+
728
+ async def _bulk_best_effort(
729
+ self,
730
+ entry: EngineEntry,
731
+ plan: _BulkPlan,
732
+ batches: list[Any],
733
+ *,
734
+ mode: bulk_mod.FailureMode,
735
+ timeout: float,
736
+ query_name: str,
737
+ ) -> int:
738
+ """best_effort: each batch commits independently; split_on_failure additionally retries a
739
+ failed batch row-by-row to isolate the bad rows (§19.3)."""
740
+ target = DatabaseTarget(database=entry.key.database, role="write")
741
+ written = 0
742
+ for batch in batches:
743
+ try:
744
+ stmt, params = plan.for_batch(batch)
745
+ async with self._executor.scope(
746
+ target, query_name, call_timeout=timeout, deadline=None, commit=True
747
+ ) as scope:
748
+ cursor = await run_execute(
749
+ scope.raw,
750
+ stmt,
751
+ params,
752
+ timeout=timeout,
753
+ is_postgres=self._executor.is_postgres(entry),
754
+ )
755
+ written += (
756
+ cursor.rowcount if cursor.rowcount and cursor.rowcount > 0 else len(batch)
757
+ )
758
+ except Exception as exc:
759
+ if mode != "split_on_failure":
760
+ # best_effort: this whole batch is dropped, never retried -- the only
761
+ # signal a caller gets is this log line + metric (performance review §9).
762
+ category = classify(exc, query_name=query_name).category.value
763
+ obslog.bulk_batch_dropped_warning(
764
+ query_name=query_name,
765
+ database=entry.key.database,
766
+ mode=mode,
767
+ rows_dropped=len(batch),
768
+ error_category=category,
769
+ )
770
+ self._metrics.incr(
771
+ m.BULK_ROWS_DROPPED,
772
+ len(batch),
773
+ labels={"database": entry.key.database, "query_name": query_name},
774
+ )
775
+ continue
776
+ for row in batch:
777
+ try:
778
+ row_stmt, row_params = plan.for_batch([row])
779
+ async with self._executor.scope(
780
+ target, query_name, call_timeout=timeout, deadline=None, commit=True
781
+ ) as scope:
782
+ await run_execute(
783
+ scope.raw,
784
+ row_stmt,
785
+ row_params,
786
+ timeout=timeout,
787
+ is_postgres=self._executor.is_postgres(entry),
788
+ )
789
+ written += 1
790
+ except Exception as row_exc:
791
+ # split_on_failure: this one row is dropped after its batch already
792
+ # failed -- same silent-loss risk, narrowed to a single row.
793
+ category = classify(row_exc, query_name=query_name).category.value
794
+ obslog.bulk_batch_dropped_warning(
795
+ query_name=query_name,
796
+ database=entry.key.database,
797
+ mode=mode,
798
+ rows_dropped=1,
799
+ error_category=category,
800
+ )
801
+ self._metrics.incr(
802
+ m.BULK_ROWS_DROPPED,
803
+ 1,
804
+ labels={"database": entry.key.database, "query_name": query_name},
805
+ )
806
+ continue
807
+ return written
808
+
809
+ # -- COPY (PostgreSQL fast bulk ingest) --------------------------------------- #
810
+
811
+ async def copy_records(
812
+ self,
813
+ table: str,
814
+ columns: Sequence[str],
815
+ records: Any,
816
+ *,
817
+ target: DatabaseTarget,
818
+ timeout: float | None = None,
819
+ ) -> ExecutionResult:
820
+ """Bulk-ingest ``records`` into ``table`` via PostgreSQL COPY (§19.2).
821
+
822
+ ``records`` is an iterable (async or sync) of row sequences matching ``columns``.
823
+ Fastest path for large ingests; PostgreSQL + psycopg only.
824
+ """
825
+ entry, _route = await self._executor.entry(target)
826
+ if not self._executor.is_postgres(entry):
827
+ raise DatabaseUnsupportedOperationError("COPY is only supported on PostgreSQL")
828
+ labels = self._executor.labels(entry, f"{table}.copy", "write")
829
+ limiter = self._executor.limiter_for(entry.key.database)
830
+ eff_timeout = timeout or self._config.defaults.transaction_timeout_seconds
831
+ start = time.monotonic()
832
+ written = 0
833
+ # COPY runs on the raw driver connection, so it must sit inside an explicit
834
+ # transaction (which begins the SQLAlchemy transaction) for the commit to persist it.
835
+ with self._tracer.span(
836
+ "dbkit.copy",
837
+ operation_type="write",
838
+ query_name=f"{table}.copy",
839
+ database=entry.key.database,
840
+ shard=entry.key.shard_id,
841
+ role=entry.key.role,
842
+ ) as span:
843
+ async with (
844
+ limiter.acquire("database"),
845
+ limiter.acquire("bulk"),
846
+ self.transaction(target=target, timeout=eff_timeout) as tx,
847
+ ):
848
+ try:
849
+ written = await copy_from_records(tx.raw, table, list(columns), records)
850
+ except Exception as exc:
851
+ raise classify(
852
+ exc, query_name=f"{table}.copy", database_name=target.database
853
+ ) from exc
854
+ span.set_attribute("db.rows_affected", written)
855
+ self._metrics.incr(m.BULK_ROWS, written, labels=labels)
856
+ return ExecutionResult(
857
+ row_count=written,
858
+ query_name=f"{table}.copy",
859
+ database_name=target.database,
860
+ duration_ms=(time.monotonic() - start) * 1000,
861
+ )
862
+
863
+
864
+ class _BulkPlan:
865
+ """How to turn a batch of row-dicts into ``(statement, params)`` for one execution —
866
+ abstracts over ``execute_many`` (list-of-dicts params) vs ``unnest`` (one array bind per
867
+ column) so :meth:`AsyncDatabase._bulk_write` / `_bulk_best_effort` stay strategy-agnostic.
868
+ """
869
+
870
+ def __init__(self, statement: Any, *, columns: Sequence[str] | None) -> None:
871
+ """``columns`` non-``None`` selects the ``unnest`` strategy; ``None`` selects
872
+ ``execute_many``."""
873
+ self._statement = statement
874
+ self._columns = columns
875
+
876
+ def for_batch(self, batch: Sequence[Mapping[str, Any]]) -> tuple[Any, Any]:
877
+ """The ``(statement, params)`` pair to execute for this batch."""
878
+ if self._columns is not None:
879
+ return self._statement, unnest_mod.columnar_params(batch, self._columns)
880
+ return self._statement, list(batch)
881
+
882
+
883
+ def _make_default_metrics(config: DbkitConfig) -> MetricsSink:
884
+ return NoopMetrics()