dbkit-sql 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. dbkit/__init__.py +77 -0
  2. dbkit/_async/__init__.py +17 -0
  3. dbkit/_async/_compat.py +111 -0
  4. dbkit/_async/connection.py +328 -0
  5. dbkit/_async/database.py +884 -0
  6. dbkit/_async/engine.py +232 -0
  7. dbkit/_async/executor.py +329 -0
  8. dbkit/_async/health.py +53 -0
  9. dbkit/_async/resilience.py +148 -0
  10. dbkit/_async/streaming.py +137 -0
  11. dbkit/_async/transaction.py +264 -0
  12. dbkit/_core/__init__.py +1 -0
  13. dbkit/_core/bulk.py +91 -0
  14. dbkit/_core/circuit.py +130 -0
  15. dbkit/_core/config.py +613 -0
  16. dbkit/_core/errors/__init__.py +74 -0
  17. dbkit/_core/errors/base.py +344 -0
  18. dbkit/_core/errors/classifier.py +138 -0
  19. dbkit/_core/errors/redaction.py +96 -0
  20. dbkit/_core/errors/sqlstate.py +74 -0
  21. dbkit/_core/idempotency_lint.py +57 -0
  22. dbkit/_core/keys.py +29 -0
  23. dbkit/_core/policies.py +105 -0
  24. dbkit/_core/query.py +142 -0
  25. dbkit/_core/result.py +109 -0
  26. dbkit/_core/routing.py +248 -0
  27. dbkit/_pool.py +202 -0
  28. dbkit/_sync/__init__.py +20 -0
  29. dbkit/_sync/_compat.py +82 -0
  30. dbkit/_sync/connection.py +331 -0
  31. dbkit/_sync/database.py +887 -0
  32. dbkit/_sync/engine.py +235 -0
  33. dbkit/_sync/executor.py +332 -0
  34. dbkit/_sync/health.py +56 -0
  35. dbkit/_sync/resilience.py +151 -0
  36. dbkit/_sync/streaming.py +140 -0
  37. dbkit/_sync/transaction.py +267 -0
  38. dbkit/cli/__init__.py +1 -0
  39. dbkit/cli/main.py +256 -0
  40. dbkit/errors.py +6 -0
  41. dbkit/integrations/__init__.py +21 -0
  42. dbkit/integrations/batch.py +81 -0
  43. dbkit/integrations/inbox.py +153 -0
  44. dbkit/observability/__init__.py +21 -0
  45. dbkit/observability/logging.py +154 -0
  46. dbkit/observability/metrics.py +110 -0
  47. dbkit/observability/otel_metrics.py +69 -0
  48. dbkit/observability/prometheus.py +78 -0
  49. dbkit/observability/tracing.py +128 -0
  50. dbkit/postgres/__init__.py +1 -0
  51. dbkit/postgres/copy.py +77 -0
  52. dbkit/postgres/pipeline.py +48 -0
  53. dbkit/postgres/unnest.py +73 -0
  54. dbkit/testing/__init__.py +1 -0
  55. dbkit_sql-0.1.0.dist-info/METADATA +555 -0
  56. dbkit_sql-0.1.0.dist-info/RECORD +59 -0
  57. dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
  58. dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
  59. dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,887 @@
1
+ # This file is GENERATED from ../_async/database.py by tools/run_unasync.py.
2
+ # Do not edit by hand. Run `make unasync` after changing the async source.
3
+
4
+ """The ``Database`` facade — the primary public entrypoint (§8.1).
5
+
6
+ Orchestrates target resolution, engine lookup, connection acquisition (measuring pool wait),
7
+ execution, resilience (retries/circuit breaker/concurrency limits), streaming, bulk writes,
8
+ COPY, metrics, and graceful startup/shutdown.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import contextlib
14
+ import contextvars
15
+ import logging
16
+ import time
17
+ from collections.abc import Iterator, Mapping, Sequence
18
+ from typing import Any, Literal
19
+
20
+ from sqlalchemy import Table, insert
21
+ from sqlalchemy.dialects import postgresql
22
+
23
+ from .._core import bulk as bulk_mod
24
+ from .._core.config import DbkitConfig
25
+ from .._core.errors import (
26
+ DatabaseError,
27
+ DatabaseRoutingError,
28
+ DatabaseUnsupportedOperationError,
29
+ classify,
30
+ )
31
+ from .._core.query import coerce_statement
32
+ from .._core.result import ExecutionResult
33
+ from .._core.routing import (
34
+ DatabaseTarget,
35
+ ReplicaSelector,
36
+ ResolvedRoute,
37
+ RoundRobinReplicaSelector,
38
+ ShardResolver,
39
+ SingleShardResolver,
40
+ )
41
+ from .._pool import PoolSnapshot
42
+ from ..observability import logging as obslog
43
+ from ..observability import metrics as m
44
+ from ..observability.metrics import MetricsSink, NoopMetrics
45
+ from ..observability.tracing import Tracer, make_tracer
46
+ from ..postgres import unnest as unnest_mod
47
+ from ._compat import copy_from_records
48
+ from .connection import ConnectionScope, run_execute
49
+ from .engine import EngineRegistry, EngineEntry
50
+ from .executor import ResilientExecutor
51
+ from .health import HealthReport, TargetHealth, ping
52
+ from .streaming import ResultStream
53
+ from .transaction import _TransactionManager
54
+
55
+ #: Task-local (safe across concurrent operations) consistency-scope override for read routing.
56
+ _consistency_mode: contextvars.ContextVar[str | None] = contextvars.ContextVar(
57
+ "dbkit_consistency_mode", default=None
58
+ )
59
+
60
+
61
+ class Database:
62
+ """Async, SQL-first database facade over SQLAlchemy Core (§8.1)."""
63
+
64
+ def __init__(
65
+ self,
66
+ config: DbkitConfig,
67
+ *,
68
+ metrics: MetricsSink | None = None,
69
+ tracer: Tracer | None = None,
70
+ shard_resolver: ShardResolver | None = None,
71
+ replica_selector: ReplicaSelector | None = None,
72
+ ) -> None:
73
+ """Build a facade from an already-validated :class:`DbkitConfig`.
74
+
75
+ Prefer :meth:`from_config` unless you need to pass a config object you built/validated
76
+ yourself. Does not connect to any database — call :meth:`start` before use.
77
+ """
78
+ self._config = config
79
+ if metrics is None:
80
+ metrics = (
81
+ _make_default_metrics(config)
82
+ if config.defaults.observability.metrics
83
+ else NoopMetrics()
84
+ )
85
+ self._metrics = metrics
86
+ self._tracer = tracer or make_tracer(config.defaults.observability.tracing)
87
+ self._registry = EngineRegistry(
88
+ config,
89
+ metrics=self._metrics,
90
+ max_engines=config.max_engines,
91
+ evict_lru=config.evict_lru_engines,
92
+ )
93
+ self._shards = shard_resolver or SingleShardResolver()
94
+ self._replicas = replica_selector or RoundRobinReplicaSelector(
95
+ {name: [r.name for r in db.replicas] for name, db in config.databases.items()}
96
+ )
97
+ self._executor = ResilientExecutor(
98
+ config,
99
+ registry=self._registry,
100
+ resolve=self._resolve,
101
+ metrics=self._metrics,
102
+ tracer=self._tracer,
103
+ )
104
+ self._started = False
105
+
106
+ # -- construction ------------------------------------------------------------- #
107
+
108
+ @classmethod
109
+ def from_config(
110
+ cls,
111
+ config: DbkitConfig | Mapping[str, Any],
112
+ *,
113
+ metrics: MetricsSink | None = None,
114
+ tracer: Tracer | None = None,
115
+ shard_resolver: ShardResolver | None = None,
116
+ replica_selector: ReplicaSelector | None = None,
117
+ ) -> Database:
118
+ """Build a facade from a :class:`DbkitConfig` or a plain dict (validated via
119
+ :meth:`DbkitConfig.from_dict`). The usual entry point."""
120
+ cfg = config if isinstance(config, DbkitConfig) else DbkitConfig.from_dict(config)
121
+ return cls(
122
+ cfg,
123
+ metrics=metrics,
124
+ tracer=tracer,
125
+ shard_resolver=shard_resolver,
126
+ replica_selector=replica_selector,
127
+ )
128
+
129
+ @property
130
+ def config(self) -> DbkitConfig:
131
+ """The validated configuration this facade was built from."""
132
+ return self._config
133
+
134
+ @property
135
+ def engine_count(self) -> int:
136
+ """Number of live SQLAlchemy engines currently held (one per target)."""
137
+ return self._registry.count
138
+
139
+ # -- lifecycle ---------------------------------------------------------------- #
140
+
141
+ def start(self, *, warm: bool = False) -> None:
142
+ """Create required engines and (optionally) warm connections (§27.1)."""
143
+ if self._started:
144
+ return
145
+ self._config.validate()
146
+ for name, db in self._config.databases.items():
147
+ if db.primary.required:
148
+ entry = self._registry.get(
149
+ ResolvedRoute(database=name, shard_id="default", role="primary")
150
+ )
151
+ if warm:
152
+ ping(entry.engine, timeout=self._config.defaults.query_timeout_seconds)
153
+ self._started = True
154
+ obslog.log_event(logging.INFO, "database.started", engines=self._registry.count)
155
+
156
+ def require_ready(self) -> None:
157
+ """Raise unless every required target is reachable (§27.1)."""
158
+ report = self.health()
159
+ if not report.ready:
160
+ failed = [t.key for t in report.targets if not t.healthy]
161
+ raise DatabaseError(f"required databases not ready: {failed}")
162
+
163
+ def close(self, grace_period: float = 10.0) -> None:
164
+ """Dispose all engines (§27.2). ``grace_period`` reserved for in-flight draining."""
165
+ self._registry.dispose_all()
166
+ self._started = False
167
+ obslog.log_event(logging.INFO, "database.closed")
168
+
169
+ def __enter__(self) -> Database:
170
+ """``with Database(...) as db:`` — calls :meth:`start`."""
171
+ self.start()
172
+ return self
173
+
174
+ def __exit__(self, *exc: object) -> None:
175
+ """Calls :meth:`close`."""
176
+ self.close()
177
+
178
+ # -- routing ------------------------------------------------------------------ #
179
+
180
+ def _resolve(self, target: DatabaseTarget) -> ResolvedRoute:
181
+ db = self._config.databases.get(target.database)
182
+ if db is None:
183
+ raise DatabaseRoutingError(f"unknown database {target.database!r}")
184
+ shard_id = (
185
+ self._shards.resolve(target.database, target.shard_key)
186
+ if target.shard_key is not None
187
+ else "default"
188
+ )
189
+ # A consistency_scope(read_your_writes) override forces reads to the primary so they
190
+ # observe writes made earlier in the same scope (§23).
191
+ if target.wants_replica and db.replicas and _consistency_mode.get() is None:
192
+ replica_name = self._replicas.select(target.database, shard_id)
193
+ if replica_name is not None:
194
+ return ResolvedRoute(
195
+ database=target.database,
196
+ shard_id=shard_id,
197
+ role="replica",
198
+ replica_name=replica_name,
199
+ )
200
+ # Explicit fallback: selector found no replica for this database -> primary (§23).
201
+ return ResolvedRoute(database=target.database, shard_id=shard_id, role="primary")
202
+
203
+ # -- read API ----------------------------------------------------------------- #
204
+
205
+ def fetch_all(
206
+ self,
207
+ query: object,
208
+ params: Mapping[str, Any] | None = None,
209
+ *,
210
+ target: DatabaseTarget,
211
+ map_to: Any = None,
212
+ timeout: float | None = None,
213
+ deadline: float | None = None,
214
+ ) -> list[Any]:
215
+ """Run a one-shot read, acquiring and releasing its own connection (§11.1).
216
+
217
+ Returns every row, mapped to ``map_to``, with no cardinality constraint. Retries and
218
+ the circuit breaker apply per the resolved ``target``'s policy.
219
+ """
220
+ rows: list[Any] = self._executor.execute_with_resilience(
221
+ target,
222
+ query,
223
+ lambda scope: scope.fetch_all(query, params, map_to=map_to),
224
+ commit=False,
225
+ timeout=timeout,
226
+ deadline=deadline,
227
+ )
228
+ return rows
229
+
230
+ def fetch_one(
231
+ self,
232
+ query: object,
233
+ params: Mapping[str, Any] | None = None,
234
+ *,
235
+ target: DatabaseTarget,
236
+ map_to: Any = None,
237
+ timeout: float | None = None,
238
+ deadline: float | None = None,
239
+ ) -> Any:
240
+ """Run a one-shot read expecting exactly one row.
241
+
242
+ Raises :class:`~dbkit.errors.DatabaseResultError` if zero or more than one row comes
243
+ back (via SQLAlchemy's own ``Result.one()``).
244
+ """
245
+ return self._executor.execute_with_resilience(
246
+ target,
247
+ query,
248
+ lambda scope: scope.fetch_one(query, params, map_to=map_to),
249
+ commit=False,
250
+ timeout=timeout,
251
+ deadline=deadline,
252
+ )
253
+
254
+ def fetch_optional(
255
+ self,
256
+ query: object,
257
+ params: Mapping[str, Any] | None = None,
258
+ *,
259
+ target: DatabaseTarget,
260
+ map_to: Any = None,
261
+ timeout: float | None = None,
262
+ deadline: float | None = None,
263
+ ) -> Any | None:
264
+ """Run a one-shot read expecting zero or one row; ``None`` if empty."""
265
+ return self._executor.execute_with_resilience(
266
+ target,
267
+ query,
268
+ lambda scope: scope.fetch_optional(query, params, map_to=map_to),
269
+ commit=False,
270
+ timeout=timeout,
271
+ deadline=deadline,
272
+ )
273
+
274
+ def fetch_value(
275
+ self,
276
+ query: object,
277
+ params: Mapping[str, Any] | None = None,
278
+ *,
279
+ target: DatabaseTarget,
280
+ timeout: float | None = None,
281
+ deadline: float | None = None,
282
+ ) -> Any:
283
+ """Run a one-shot read expecting exactly one row and return its first column."""
284
+ return self._executor.execute_with_resilience(
285
+ target,
286
+ query,
287
+ lambda scope: scope.fetch_value(query, params),
288
+ commit=False,
289
+ timeout=timeout,
290
+ deadline=deadline,
291
+ )
292
+
293
+ def fetch_values(
294
+ self,
295
+ query: object,
296
+ params: Mapping[str, Any] | None = None,
297
+ *,
298
+ target: DatabaseTarget,
299
+ timeout: float | None = None,
300
+ deadline: float | None = None,
301
+ ) -> list[Any]:
302
+ """Run a one-shot read and return the first column of every row."""
303
+ values: list[Any] = self._executor.execute_with_resilience(
304
+ target,
305
+ query,
306
+ lambda scope: scope.fetch_values(query, params),
307
+ commit=False,
308
+ timeout=timeout,
309
+ deadline=deadline,
310
+ )
311
+ return values
312
+
313
+ # -- write API ---------------------------------------------------------------- #
314
+
315
+ def execute(
316
+ self,
317
+ query: object,
318
+ params: Mapping[str, Any] | None = None,
319
+ *,
320
+ target: DatabaseTarget,
321
+ timeout: float | None = None,
322
+ deadline: float | None = None,
323
+ ) -> ExecutionResult:
324
+ """Run a one-shot write/DDL statement, auto-committed on success (§11.1)."""
325
+ query_name, _, _ = self._executor.query_meta(query)
326
+ start = time.monotonic()
327
+ row_count = self._executor.execute_with_resilience(
328
+ target,
329
+ query,
330
+ lambda scope: scope.execute(query, params),
331
+ commit=True,
332
+ timeout=timeout,
333
+ deadline=deadline,
334
+ )
335
+ return ExecutionResult(
336
+ row_count=row_count,
337
+ query_name=query_name,
338
+ database_name=target.database,
339
+ duration_ms=(time.monotonic() - start) * 1000,
340
+ )
341
+
342
+ def execute_many(
343
+ self,
344
+ query: object,
345
+ params_seq: Sequence[Mapping[str, Any]],
346
+ *,
347
+ target: DatabaseTarget,
348
+ timeout: float | None = None,
349
+ deadline: float | None = None,
350
+ ) -> ExecutionResult:
351
+ """Run one statement against many parameter sets (driver executemany)."""
352
+ query_name, _op, _q = self._executor.query_meta(query)
353
+ statement = coerce_statement(query)
354
+ start = time.monotonic()
355
+ with self._executor.scope(
356
+ target, query, call_timeout=timeout, deadline=deadline, commit=True
357
+ ) as scope:
358
+ try:
359
+ cursor = run_execute(
360
+ scope.raw,
361
+ statement,
362
+ list(params_seq),
363
+ timeout=scope._default_timeout,
364
+ is_postgres=scope._is_postgres,
365
+ )
366
+ row_count = cursor.rowcount
367
+ except Exception as exc:
368
+ raise classify(exc, query_name=query_name, database_name=target.database) from exc
369
+ return ExecutionResult(
370
+ row_count=row_count,
371
+ query_name=query_name,
372
+ database_name=target.database,
373
+ duration_ms=(time.monotonic() - start) * 1000,
374
+ )
375
+
376
+ # -- explicit scopes ---------------------------------------------------------- #
377
+
378
+ @contextlib.contextmanager
379
+ def connection(
380
+ self, *, target: DatabaseTarget, timeout: float | None = None
381
+ ) -> Iterator[ConnectionScope]:
382
+ """A held connection with commit-on-success / rollback-on-error semantics (§11.2)."""
383
+ with self._executor.scope(
384
+ target, "adhoc", call_timeout=timeout, deadline=None, commit=True
385
+ ) as scope:
386
+ yield scope
387
+
388
+ @contextlib.contextmanager
389
+ def transaction(
390
+ self,
391
+ *,
392
+ target: DatabaseTarget,
393
+ isolation: str | None = None,
394
+ read_only: bool = False,
395
+ deferrable: bool = False,
396
+ timeout: float | None = None,
397
+ lock_timeout: float | None = None,
398
+ ) -> Iterator[Any]:
399
+ """An explicit transaction with isolation/timeout options (§11.3).
400
+
401
+ ``deferrable`` (PostgreSQL only) is only meaningful together with
402
+ ``isolation="serializable"`` and ``read_only=True``: it lets a read-only serializable
403
+ transaction wait for a safe snapshot instead of risking a serialization failure.
404
+
405
+ Usage::
406
+
407
+ with db.transaction(target=write_target) as tx:
408
+ tx.execute(INSERT, params)
409
+ """
410
+ route = self._resolve(target)
411
+ entry = self._registry.get(route)
412
+ labels = self._executor.labels(entry, "transaction", "transaction")
413
+ with self._tracer.span(
414
+ "dbkit.transaction",
415
+ operation_type="transaction",
416
+ query_name="transaction",
417
+ database=entry.key.database,
418
+ shard=entry.key.shard_id,
419
+ role=entry.key.role,
420
+ ) as span:
421
+ manager = _TransactionManager(
422
+ entry.engine,
423
+ is_postgres=self._executor.is_postgres(entry),
424
+ default_timeout=self._config.defaults.transaction_timeout_seconds,
425
+ database=entry.key.database,
426
+ shard_id=entry.key.shard_id,
427
+ role=entry.key.role,
428
+ isolation=isolation,
429
+ read_only=read_only,
430
+ deferrable=deferrable,
431
+ timeout=timeout,
432
+ lock_timeout=lock_timeout,
433
+ query_name="transaction",
434
+ metrics=self._metrics,
435
+ labels=labels,
436
+ long_transaction_warning_seconds=(
437
+ self._config.defaults.long_transaction_warning_seconds
438
+ ),
439
+ span=span,
440
+ )
441
+ with manager as scope:
442
+ yield scope
443
+
444
+ @contextlib.contextmanager
445
+ def consistency_scope(
446
+ self, *, mode: Literal["read_your_writes"] = "read_your_writes"
447
+ ) -> Iterator[None]:
448
+ """Force reads within this scope to the primary so they observe writes made earlier
449
+ in the same scope — read-your-writes over replica routing (§23)::
450
+
451
+ with db.consistency_scope(mode="read_your_writes"):
452
+ db.execute(write_query, target=write_target)
453
+ row = db.fetch_one(read_query, target=read_target) # sees the write
454
+
455
+ The override is a ``contextvars.ContextVar``, so it never leaks across concurrent
456
+ operations in other tasks — but it also does not cross a thread boundary (a plain
457
+ ``threading.Thread``, ``loop.run_in_executor``, or the sync ``Database`` facade driven
458
+ from a worker thread does not inherit it). See "Read-your-writes across threads" in
459
+ ``docs/troubleshooting.md`` if you need the override to apply there too.
460
+ """
461
+ token = _consistency_mode.set(mode)
462
+ try:
463
+ yield
464
+ finally:
465
+ _consistency_mode.reset(token)
466
+
467
+ # -- health & introspection --------------------------------------------------- #
468
+
469
+ def health(self) -> HealthReport:
470
+ """Readiness check: ``SELECT 1`` against every required target (§26)."""
471
+ targets: list[TargetHealth] = []
472
+ ready = True
473
+ for name, db in self._config.databases.items():
474
+ if not db.primary.required:
475
+ continue
476
+ try:
477
+ entry = self._registry.get(
478
+ ResolvedRoute(database=name, shard_id="default", role="primary")
479
+ )
480
+ ping(entry.engine, timeout=self._config.defaults.query_timeout_seconds)
481
+ targets.append(TargetHealth(key=f"{name}.primary", healthy=True))
482
+ except Exception as exc:
483
+ ready = False
484
+ targets.append(TargetHealth(key=f"{name}.primary", healthy=False, error=str(exc)))
485
+ return HealthReport(live=True, ready=ready, targets=targets)
486
+
487
+ def pool_status(self) -> list[PoolSnapshot]:
488
+ """A point-in-time snapshot of every currently live engine's connection pool."""
489
+ return self._registry.snapshots()
490
+
491
+ def drain_engine(self, key: str) -> bool:
492
+ """Force-dispose one engine's idle pooled connections by its snapshot ``key`` (as printed
493
+ by :meth:`pool_status`, e.g. ``"prod:app:default:primary:psycopg"``), so the *next* call
494
+ routed to it rebuilds a fresh engine with fresh connections. Returns ``False`` if no live
495
+ engine currently has that key.
496
+
497
+ Useful right before a planned failover/topology change: rather than waiting for
498
+ connections to naturally recycle (``PoolConfig.recycle_seconds``), force them closed so
499
+ subsequent traffic reaches the new backend immediately. Only idle pooled connections are
500
+ closed — one already checked out by an in-flight call keeps working until released,
501
+ exactly like LRU eviction (`EngineRegistry(evict_lru=True)`).
502
+
503
+ This must be called from *within the running application process* — there is no
504
+ ``dbkit`` CLI command for this, deliberately: each CLI invocation is a fresh, separate
505
+ process with its own empty engine registry, so it has no way to reach into an
506
+ already-running application's live engines. Wire this to a signal handler or an admin
507
+ HTTP route in your own application instead (see ``docs/troubleshooting.md``).
508
+ """
509
+ return self._registry.dispose_one(key)
510
+
511
+ # -- streaming ---------------------------------------------------------------- #
512
+
513
+ def stream(
514
+ self,
515
+ query: object,
516
+ params: Mapping[str, Any] | None = None,
517
+ *,
518
+ target: DatabaseTarget,
519
+ batch_size: int = 1000,
520
+ map_to: Any = None,
521
+ max_duration: float | None = None,
522
+ ) -> ResultStream:
523
+ """Stream a large result set with a server-side cursor, bounded memory (§20).
524
+
525
+ Usage::
526
+
527
+ with db.stream(EXPORT, params, target=t, batch_size=1000) as rows:
528
+ for row in rows:
529
+ ...
530
+
531
+ The stream owns its connection until the context exits, so it bypasses auto-retry.
532
+ """
533
+ entry, _route = self._executor.entry(target)
534
+ statement = coerce_statement(query)
535
+ query_name, operation, _q = self._executor.query_meta(query)
536
+ labels = self._executor.labels(entry, query_name, operation)
537
+ return ResultStream(
538
+ entry.engine,
539
+ statement,
540
+ params,
541
+ batch_size=batch_size,
542
+ map_to=map_to,
543
+ database=entry.key.database,
544
+ role=entry.key.role,
545
+ query_name=query_name,
546
+ metrics=self._metrics,
547
+ labels=labels,
548
+ max_duration=max_duration,
549
+ tracer=self._tracer,
550
+ shard_id=entry.key.shard_id,
551
+ )
552
+
553
+ # -- bulk writes -------------------------------------------------------------- #
554
+
555
+ def insert_many(
556
+ self,
557
+ table: Table,
558
+ rows: Sequence[Mapping[str, Any]],
559
+ *,
560
+ target: DatabaseTarget,
561
+ mode: bulk_mod.FailureMode = "atomic",
562
+ strategy: bulk_mod.InsertStrategy = "execute_many",
563
+ batch_size: int | None = None,
564
+ timeout: float | None = None,
565
+ ) -> ExecutionResult:
566
+ """Insert many rows in adaptively-sized batches (§19). ``rows`` are param dicts.
567
+
568
+ ``strategy="unnest"`` (PostgreSQL only) binds one array parameter per column instead
569
+ of one parameter per column per row, so batch size isn't limited by the bind-parameter
570
+ ceiling — a mid-tier option between ``execute_many`` and COPY.
571
+ """
572
+ return self._bulk_write(
573
+ table,
574
+ insert(table),
575
+ rows,
576
+ target=target,
577
+ mode=mode,
578
+ strategy=strategy,
579
+ batch_size=batch_size,
580
+ timeout=timeout,
581
+ query_name=f"{table.name}.insert_many",
582
+ )
583
+
584
+ def upsert_many(
585
+ self,
586
+ table: Table,
587
+ rows: Sequence[Mapping[str, Any]],
588
+ *,
589
+ target: DatabaseTarget,
590
+ conflict_index_elements: Sequence[str],
591
+ update_columns: Sequence[str] | None = None,
592
+ mode: bulk_mod.FailureMode = "atomic",
593
+ strategy: bulk_mod.InsertStrategy = "execute_many",
594
+ batch_size: int | None = None,
595
+ timeout: float | None = None,
596
+ ) -> ExecutionResult:
597
+ """Upsert many rows via PostgreSQL ``ON CONFLICT`` (§19). ``update_columns=None`` means
598
+ ``DO NOTHING``; otherwise those columns are updated from the proposed row."""
599
+ from sqlalchemy.dialects.postgresql import insert as pg_insert
600
+
601
+ base = pg_insert(table)
602
+ if update_columns is None:
603
+ stmt = base.on_conflict_do_nothing(index_elements=list(conflict_index_elements))
604
+ else:
605
+ stmt = base.on_conflict_do_update(
606
+ index_elements=list(conflict_index_elements),
607
+ set_={c: base.excluded[c] for c in update_columns},
608
+ )
609
+ return self._bulk_write(
610
+ table,
611
+ stmt,
612
+ rows,
613
+ target=target,
614
+ mode=mode,
615
+ strategy=strategy,
616
+ batch_size=batch_size,
617
+ timeout=timeout,
618
+ query_name=f"{table.name}.upsert_many",
619
+ conflict_index_elements=conflict_index_elements,
620
+ update_columns=update_columns,
621
+ )
622
+
623
+ def _bulk_write(
624
+ self,
625
+ table: Table,
626
+ statement: Any,
627
+ rows: Sequence[Mapping[str, Any]],
628
+ *,
629
+ target: DatabaseTarget,
630
+ mode: bulk_mod.FailureMode,
631
+ batch_size: int | None,
632
+ timeout: float | None,
633
+ query_name: str,
634
+ strategy: bulk_mod.InsertStrategy = "execute_many",
635
+ conflict_index_elements: Sequence[str] | None = None,
636
+ update_columns: Sequence[str] | None = None,
637
+ ) -> ExecutionResult:
638
+ start = time.monotonic()
639
+ if not rows:
640
+ return ExecutionResult(
641
+ row_count=0, query_name=query_name, database_name=target.database, duration_ms=0.0
642
+ )
643
+ entry, _route = self._executor.entry(target)
644
+ labels = self._executor.labels(entry, query_name, "write")
645
+ limiter = self._executor.limiter_for(entry.key.database)
646
+ bulk_cfg = self._config.defaults.bulk
647
+ columns = bulk_mod.column_names(list(rows))
648
+
649
+ if strategy == "unnest":
650
+ if not self._executor.is_postgres(entry):
651
+ raise DatabaseUnsupportedOperationError(
652
+ "strategy='unnest' requires PostgreSQL; this target uses a different dialect"
653
+ )
654
+ column_types = {
655
+ c: table.c[c].type.compile(dialect=postgresql.dialect()) # type: ignore[no-untyped-call]
656
+ for c in columns
657
+ }
658
+ unnest_stmt = unnest_mod.unnest_insert_sql(
659
+ table.name,
660
+ columns,
661
+ column_types,
662
+ conflict_index_elements=conflict_index_elements,
663
+ update_columns=update_columns,
664
+ )
665
+ plan = _BulkPlan(unnest_stmt, columns=columns)
666
+ else:
667
+ plan = _BulkPlan(statement, columns=None)
668
+
669
+ batch_rows = bulk_mod.resolve_batch_rows(
670
+ len(columns),
671
+ batch_size or bulk_cfg.default_batch_rows,
672
+ bulk_mod.BulkLimits(
673
+ max_rows=bulk_cfg.max_batch_rows, max_payload_bytes=bulk_cfg.max_payload_bytes
674
+ ),
675
+ sample_row=rows[0] if rows else None,
676
+ )
677
+ batches = list(bulk_mod.iter_batches(list(rows), batch_rows))
678
+ self._metrics.observe(m.BULK_BATCH_SIZE, batch_rows, labels=labels)
679
+ eff_timeout = timeout or self._config.defaults.transaction_timeout_seconds
680
+
681
+ written = 0
682
+ with self._tracer.span(
683
+ "dbkit.bulk_write",
684
+ operation_type="write",
685
+ query_name=query_name,
686
+ database=entry.key.database,
687
+ shard=entry.key.shard_id,
688
+ role=entry.key.role,
689
+ ) as span:
690
+ with limiter.acquire("database"), limiter.acquire("bulk"):
691
+ if mode == "atomic":
692
+ with self.transaction(target=target, timeout=eff_timeout) as tx:
693
+ for batch in batches:
694
+ stmt, params = plan.for_batch(batch)
695
+ try:
696
+ cursor = run_execute(
697
+ tx.raw,
698
+ stmt,
699
+ params,
700
+ timeout=eff_timeout,
701
+ is_postgres=self._executor.is_postgres(entry),
702
+ )
703
+ except Exception as exc:
704
+ raise classify(
705
+ exc, query_name=query_name, database_name=target.database
706
+ ) from exc
707
+ written += (
708
+ cursor.rowcount
709
+ if cursor.rowcount and cursor.rowcount > 0
710
+ else len(batch)
711
+ )
712
+ else:
713
+ written = self._bulk_best_effort(
714
+ entry,
715
+ plan,
716
+ batches,
717
+ mode=mode,
718
+ timeout=eff_timeout,
719
+ query_name=query_name,
720
+ )
721
+ span.set_attribute("db.rows_affected", written)
722
+
723
+ self._metrics.incr(m.BULK_ROWS, written, labels=labels)
724
+ return ExecutionResult(
725
+ row_count=written,
726
+ query_name=query_name,
727
+ database_name=target.database,
728
+ duration_ms=(time.monotonic() - start) * 1000,
729
+ )
730
+
731
+ def _bulk_best_effort(
732
+ self,
733
+ entry: EngineEntry,
734
+ plan: _BulkPlan,
735
+ batches: list[Any],
736
+ *,
737
+ mode: bulk_mod.FailureMode,
738
+ timeout: float,
739
+ query_name: str,
740
+ ) -> int:
741
+ """best_effort: each batch commits independently; split_on_failure additionally retries a
742
+ failed batch row-by-row to isolate the bad rows (§19.3)."""
743
+ target = DatabaseTarget(database=entry.key.database, role="write")
744
+ written = 0
745
+ for batch in batches:
746
+ try:
747
+ stmt, params = plan.for_batch(batch)
748
+ with self._executor.scope(
749
+ target, query_name, call_timeout=timeout, deadline=None, commit=True
750
+ ) as scope:
751
+ cursor = run_execute(
752
+ scope.raw,
753
+ stmt,
754
+ params,
755
+ timeout=timeout,
756
+ is_postgres=self._executor.is_postgres(entry),
757
+ )
758
+ written += (
759
+ cursor.rowcount if cursor.rowcount and cursor.rowcount > 0 else len(batch)
760
+ )
761
+ except Exception as exc:
762
+ if mode != "split_on_failure":
763
+ # best_effort: this whole batch is dropped, never retried -- the only
764
+ # signal a caller gets is this log line + metric (performance review §9).
765
+ category = classify(exc, query_name=query_name).category.value
766
+ obslog.bulk_batch_dropped_warning(
767
+ query_name=query_name,
768
+ database=entry.key.database,
769
+ mode=mode,
770
+ rows_dropped=len(batch),
771
+ error_category=category,
772
+ )
773
+ self._metrics.incr(
774
+ m.BULK_ROWS_DROPPED,
775
+ len(batch),
776
+ labels={"database": entry.key.database, "query_name": query_name},
777
+ )
778
+ continue
779
+ for row in batch:
780
+ try:
781
+ row_stmt, row_params = plan.for_batch([row])
782
+ with self._executor.scope(
783
+ target, query_name, call_timeout=timeout, deadline=None, commit=True
784
+ ) as scope:
785
+ run_execute(
786
+ scope.raw,
787
+ row_stmt,
788
+ row_params,
789
+ timeout=timeout,
790
+ is_postgres=self._executor.is_postgres(entry),
791
+ )
792
+ written += 1
793
+ except Exception as row_exc:
794
+ # split_on_failure: this one row is dropped after its batch already
795
+ # failed -- same silent-loss risk, narrowed to a single row.
796
+ category = classify(row_exc, query_name=query_name).category.value
797
+ obslog.bulk_batch_dropped_warning(
798
+ query_name=query_name,
799
+ database=entry.key.database,
800
+ mode=mode,
801
+ rows_dropped=1,
802
+ error_category=category,
803
+ )
804
+ self._metrics.incr(
805
+ m.BULK_ROWS_DROPPED,
806
+ 1,
807
+ labels={"database": entry.key.database, "query_name": query_name},
808
+ )
809
+ continue
810
+ return written
811
+
812
+ # -- COPY (PostgreSQL fast bulk ingest) --------------------------------------- #
813
+
814
+ def copy_records(
815
+ self,
816
+ table: str,
817
+ columns: Sequence[str],
818
+ records: Any,
819
+ *,
820
+ target: DatabaseTarget,
821
+ timeout: float | None = None,
822
+ ) -> ExecutionResult:
823
+ """Bulk-ingest ``records`` into ``table`` via PostgreSQL COPY (§19.2).
824
+
825
+ ``records`` is an iterable (async or sync) of row sequences matching ``columns``.
826
+ Fastest path for large ingests; PostgreSQL + psycopg only.
827
+ """
828
+ entry, _route = self._executor.entry(target)
829
+ if not self._executor.is_postgres(entry):
830
+ raise DatabaseUnsupportedOperationError("COPY is only supported on PostgreSQL")
831
+ labels = self._executor.labels(entry, f"{table}.copy", "write")
832
+ limiter = self._executor.limiter_for(entry.key.database)
833
+ eff_timeout = timeout or self._config.defaults.transaction_timeout_seconds
834
+ start = time.monotonic()
835
+ written = 0
836
+ # COPY runs on the raw driver connection, so it must sit inside an explicit
837
+ # transaction (which begins the SQLAlchemy transaction) for the commit to persist it.
838
+ with self._tracer.span(
839
+ "dbkit.copy",
840
+ operation_type="write",
841
+ query_name=f"{table}.copy",
842
+ database=entry.key.database,
843
+ shard=entry.key.shard_id,
844
+ role=entry.key.role,
845
+ ) as span:
846
+ with (
847
+ limiter.acquire("database"),
848
+ limiter.acquire("bulk"),
849
+ self.transaction(target=target, timeout=eff_timeout) as tx,
850
+ ):
851
+ try:
852
+ written = copy_from_records(tx.raw, table, list(columns), records)
853
+ except Exception as exc:
854
+ raise classify(
855
+ exc, query_name=f"{table}.copy", database_name=target.database
856
+ ) from exc
857
+ span.set_attribute("db.rows_affected", written)
858
+ self._metrics.incr(m.BULK_ROWS, written, labels=labels)
859
+ return ExecutionResult(
860
+ row_count=written,
861
+ query_name=f"{table}.copy",
862
+ database_name=target.database,
863
+ duration_ms=(time.monotonic() - start) * 1000,
864
+ )
865
+
866
+
867
+ class _BulkPlan:
868
+ """How to turn a batch of row-dicts into ``(statement, params)`` for one execution —
869
+ abstracts over ``execute_many`` (list-of-dicts params) vs ``unnest`` (one array bind per
870
+ column) so :meth:`Database._bulk_write` / `_bulk_best_effort` stay strategy-agnostic.
871
+ """
872
+
873
+ def __init__(self, statement: Any, *, columns: Sequence[str] | None) -> None:
874
+ """``columns`` non-``None`` selects the ``unnest`` strategy; ``None`` selects
875
+ ``execute_many``."""
876
+ self._statement = statement
877
+ self._columns = columns
878
+
879
+ def for_batch(self, batch: Sequence[Mapping[str, Any]]) -> tuple[Any, Any]:
880
+ """The ``(statement, params)`` pair to execute for this batch."""
881
+ if self._columns is not None:
882
+ return self._statement, unnest_mod.columnar_params(batch, self._columns)
883
+ return self._statement, list(batch)
884
+
885
+
886
+ def _make_default_metrics(config: DbkitConfig) -> MetricsSink:
887
+ return NoopMetrics()