dbkit-sql 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. dbkit/__init__.py +77 -0
  2. dbkit/_async/__init__.py +17 -0
  3. dbkit/_async/_compat.py +111 -0
  4. dbkit/_async/connection.py +328 -0
  5. dbkit/_async/database.py +884 -0
  6. dbkit/_async/engine.py +232 -0
  7. dbkit/_async/executor.py +329 -0
  8. dbkit/_async/health.py +53 -0
  9. dbkit/_async/resilience.py +148 -0
  10. dbkit/_async/streaming.py +137 -0
  11. dbkit/_async/transaction.py +264 -0
  12. dbkit/_core/__init__.py +1 -0
  13. dbkit/_core/bulk.py +91 -0
  14. dbkit/_core/circuit.py +130 -0
  15. dbkit/_core/config.py +613 -0
  16. dbkit/_core/errors/__init__.py +74 -0
  17. dbkit/_core/errors/base.py +344 -0
  18. dbkit/_core/errors/classifier.py +138 -0
  19. dbkit/_core/errors/redaction.py +96 -0
  20. dbkit/_core/errors/sqlstate.py +74 -0
  21. dbkit/_core/idempotency_lint.py +57 -0
  22. dbkit/_core/keys.py +29 -0
  23. dbkit/_core/policies.py +105 -0
  24. dbkit/_core/query.py +142 -0
  25. dbkit/_core/result.py +109 -0
  26. dbkit/_core/routing.py +248 -0
  27. dbkit/_pool.py +202 -0
  28. dbkit/_sync/__init__.py +20 -0
  29. dbkit/_sync/_compat.py +82 -0
  30. dbkit/_sync/connection.py +331 -0
  31. dbkit/_sync/database.py +887 -0
  32. dbkit/_sync/engine.py +235 -0
  33. dbkit/_sync/executor.py +332 -0
  34. dbkit/_sync/health.py +56 -0
  35. dbkit/_sync/resilience.py +151 -0
  36. dbkit/_sync/streaming.py +140 -0
  37. dbkit/_sync/transaction.py +267 -0
  38. dbkit/cli/__init__.py +1 -0
  39. dbkit/cli/main.py +256 -0
  40. dbkit/errors.py +6 -0
  41. dbkit/integrations/__init__.py +21 -0
  42. dbkit/integrations/batch.py +81 -0
  43. dbkit/integrations/inbox.py +153 -0
  44. dbkit/observability/__init__.py +21 -0
  45. dbkit/observability/logging.py +154 -0
  46. dbkit/observability/metrics.py +110 -0
  47. dbkit/observability/otel_metrics.py +69 -0
  48. dbkit/observability/prometheus.py +78 -0
  49. dbkit/observability/tracing.py +128 -0
  50. dbkit/postgres/__init__.py +1 -0
  51. dbkit/postgres/copy.py +77 -0
  52. dbkit/postgres/pipeline.py +48 -0
  53. dbkit/postgres/unnest.py +73 -0
  54. dbkit/testing/__init__.py +1 -0
  55. dbkit_sql-0.1.0.dist-info/METADATA +555 -0
  56. dbkit_sql-0.1.0.dist-info/RECORD +59 -0
  57. dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
  58. dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
  59. dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,148 @@
1
+ """Retry executor and concurrency limiter (§14, §17).
2
+
3
+ The retry *decisions* (idempotency gating, backoff shape) live in the pure ``_core.policies``
4
+ module; this is the async loop that applies them, respects the caller deadline, and drives the
5
+ circuit breaker. The concurrency limiter bounds how many operations may use the database at
6
+ once, independently of pool size — and is acquired *before* pool checkout so queueing happens
7
+ in cheap asyncio waiters, not held connections (§17).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import contextlib
13
+ import random
14
+ import time
15
+ from collections.abc import AsyncIterator, Awaitable, Callable
16
+ from typing import TypeVar
17
+
18
+ from .._core.circuit import CircuitBreaker, CircuitState, counts_as_failure
19
+ from .._core.config import RetryConfig
20
+ from .._core.errors import DatabaseCircuitOpenError, DatabaseError, DatabaseOverloadedError
21
+ from .._core.policies import backoff_delay_ms, should_retry
22
+ from .._core.query import Query
23
+ from ..observability import metrics as m
24
+ from ..observability.metrics import MetricsSink
25
+ from ._compat import semaphore_acquire, sleep
26
+
27
+ T = TypeVar("T")
28
+
29
+ #: Numeric value of each circuit-breaker state, for the ``CIRCUIT_STATE`` gauge (§16).
30
+ _CIRCUIT_STATE_VALUE = {
31
+ CircuitState.CLOSED: 0.0,
32
+ CircuitState.HALF_OPEN: 1.0,
33
+ CircuitState.OPEN: 2.0,
34
+ }
35
+
36
+
37
+ def _emit_circuit_state(
38
+ breaker: CircuitBreaker, now: float, metrics: MetricsSink, labels: dict[str, str]
39
+ ) -> None:
40
+ metrics.gauge(m.CIRCUIT_STATE, _CIRCUIT_STATE_VALUE[breaker.state(now)], labels=labels)
41
+
42
+
43
+ async def run_with_retries(
44
+ operation: Callable[[], Awaitable[T]],
45
+ *,
46
+ query: Query | None,
47
+ retry: RetryConfig,
48
+ breaker: CircuitBreaker | None,
49
+ metrics: MetricsSink,
50
+ labels: dict[str, str],
51
+ idempotent_override: bool | None = None,
52
+ deadline: float | None = None,
53
+ ) -> T:
54
+ """Run ``operation`` with conservative, idempotency-gated retries (§14).
55
+
56
+ Each attempt runs a *fresh* operation (new connection/transaction) — the caller supplies a
57
+ thunk that re-does the work. Non-retryable errors, exhausted budgets, unknown-commit
58
+ outcomes, and non-idempotent writes propagate immediately.
59
+
60
+ Total time spent retrying (backoff delays only, not attempt execution time) is bounded by
61
+ ``retry.maximum_total_ms`` regardless of ``attempts``/``deadline`` — without this, a caller
62
+ with a generous ``attempts`` count and no explicit ``deadline`` has no bound on how long a
63
+ single logical call can spend retrying (performance review §5/§7).
64
+ """
65
+ attempt = 0
66
+ started = time.monotonic()
67
+ while True:
68
+ attempt += 1
69
+ now = time.monotonic()
70
+ if breaker is not None:
71
+ allowed = breaker.allow(now)
72
+ _emit_circuit_state(breaker, now, metrics, labels)
73
+ if not allowed:
74
+ raise DatabaseCircuitOpenError(
75
+ "circuit is open for this target",
76
+ database_name=labels.get("database"),
77
+ role=labels.get("role"),
78
+ query_name=labels.get("query_name"),
79
+ )
80
+ try:
81
+ result = await operation()
82
+ except DatabaseError as err:
83
+ if breaker is not None:
84
+ if counts_as_failure(err):
85
+ breaker.on_failure(time.monotonic())
86
+ _emit_circuit_state(breaker, time.monotonic(), metrics, labels)
87
+ if not should_retry(
88
+ err,
89
+ query=query,
90
+ config=retry,
91
+ attempt=attempt,
92
+ idempotent_override=idempotent_override,
93
+ ):
94
+ raise
95
+ delay = backoff_delay_ms(attempt, retry, rand=random.random()) / 1000.0
96
+ # never sleep past the caller deadline
97
+ if deadline is not None and time.monotonic() + delay >= deadline:
98
+ raise
99
+ # never sleep past the retry budget, independent of any caller deadline
100
+ elapsed_ms = (time.monotonic() - started) * 1000.0
101
+ if elapsed_ms + delay * 1000.0 >= retry.maximum_total_ms:
102
+ raise
103
+ metrics.incr(m.OP_RETRIES, labels={**labels, "retry_attempt": str(attempt)})
104
+ await sleep(delay)
105
+ continue
106
+ else:
107
+ if breaker is not None:
108
+ breaker.on_success(time.monotonic())
109
+ _emit_circuit_state(breaker, time.monotonic(), metrics, labels)
110
+ return result
111
+
112
+
113
+ class ConcurrencyLimiter:
114
+ """Per-tier semaphores bounding concurrent database use (§17).
115
+
116
+ Tiers: ``"database"`` (overall), ``"reads"``, ``"writes"``, ``"bulk"``. A tier with no
117
+ configured limit is unbounded (a no-op context).
118
+ """
119
+
120
+ def __init__(self, limits: dict[str, int | None]) -> None:
121
+ """``limits`` maps tier name to its cap; a missing/``None``/``<=0`` limit is unbounded."""
122
+ # asyncio.Semaphore in the async build; threading.Semaphore after unasync.
123
+ import asyncio
124
+
125
+ self._sems: dict[str, asyncio.Semaphore] = {
126
+ tier: asyncio.Semaphore(n) for tier, n in limits.items() if n and n > 0
127
+ }
128
+
129
+ @contextlib.asynccontextmanager
130
+ async def acquire(self, tier: str, *, timeout: float | None = None) -> AsyncIterator[None]:
131
+ """Hold one slot of ``tier`` for the block's duration (no-op if unbounded).
132
+
133
+ Raises :class:`DatabaseOverloadedError` if no slot frees up within ``timeout`` —
134
+ without this bound, a saturated tier would queue callers indefinitely, invisibly to
135
+ dbkit's own timeout/deadline machinery (§17).
136
+ """
137
+ sem = self._sems.get(tier)
138
+ if sem is None:
139
+ yield
140
+ return
141
+ if not await semaphore_acquire(sem, timeout):
142
+ raise DatabaseOverloadedError(
143
+ f"concurrency limit reached for tier {tier!r}; no slot freed up within {timeout}s"
144
+ )
145
+ try:
146
+ yield
147
+ finally:
148
+ sem.release()
@@ -0,0 +1,137 @@
1
+ """Streaming large result sets without loading them into memory (§20).
2
+
3
+ ``AsyncResultStream`` holds one connection for the lifetime of the stream and pulls rows from a
4
+ server-side cursor in bounded batches. It is both an async context manager (guaranteeing the
5
+ connection is released) and an async iterator. Because a stream holds its connection across
6
+ many yields, it deliberately bypasses the retry wrapper — a partially consumed stream cannot be
7
+ transparently restarted.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import contextlib
13
+ import time
14
+ from typing import Any
15
+
16
+ from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
17
+
18
+ from .._core import result as result_mod
19
+ from .._core.errors import DatabaseQueryTimeoutError, classify
20
+ from .._core.query import Statement
21
+ from ..observability import metrics as m
22
+ from ..observability.metrics import MetricsSink
23
+ from ..observability.tracing import Tracer
24
+ from ._compat import stream_mappings
25
+
26
+
27
+ class AsyncResultStream:
28
+ """A bounded, connection-owning async row stream."""
29
+
30
+ def __init__(
31
+ self,
32
+ engine: AsyncEngine,
33
+ statement: Statement,
34
+ params: Any,
35
+ *,
36
+ batch_size: int,
37
+ map_to: Any,
38
+ database: str,
39
+ role: str,
40
+ query_name: str,
41
+ metrics: MetricsSink,
42
+ labels: dict[str, str],
43
+ max_duration: float | None = None,
44
+ tracer: Tracer | None = None,
45
+ shard_id: str | None = None,
46
+ ) -> None:
47
+ """Constructed by ``AsyncDatabase.stream(...)``; not intended to be built directly."""
48
+ self._engine = engine
49
+ self._statement = statement
50
+ self._params = params
51
+ self._batch_size = batch_size
52
+ self._mapper = result_mod.build_mapper(map_to)
53
+ self._database = database
54
+ self._role = role
55
+ self._query_name = query_name
56
+ self._metrics = metrics
57
+ self._labels = labels
58
+ self._max_duration = max_duration
59
+ self._tracer = tracer
60
+ self._shard_id = shard_id
61
+ self._conn: AsyncConnection | None = None
62
+ self._gen: Any = None
63
+ self._count = 0
64
+ self._started_at = 0.0
65
+ self._span_cm: Any = None
66
+ self._span: Any = None
67
+
68
+ async def __aenter__(self) -> AsyncResultStream:
69
+ """Open the underlying connection and server-side cursor."""
70
+ if self._tracer is not None:
71
+ self._span_cm = self._tracer.span(
72
+ "dbkit.stream",
73
+ operation_type="stream",
74
+ query_name=self._query_name,
75
+ database=self._database,
76
+ shard=self._shard_id,
77
+ role=self._role,
78
+ )
79
+ self._span = self._span_cm.__enter__()
80
+ try:
81
+ self._conn = await self._engine.connect()
82
+ self._gen = stream_mappings(self._conn, self._statement, self._params, self._batch_size)
83
+ except Exception as exc:
84
+ await self._cleanup()
85
+ raise classify(
86
+ exc, query_name=self._query_name, database_name=self._database, role=self._role
87
+ ) from exc
88
+ self._started_at = time.monotonic()
89
+ return self
90
+
91
+ def __aiter__(self) -> AsyncResultStream:
92
+ """Returns ``self`` — the stream is its own iterator."""
93
+ return self
94
+
95
+ async def __anext__(self) -> Any:
96
+ """The next mapped row, or raise :class:`StopAsyncIteration`/a timeout error."""
97
+ assert self._gen is not None
98
+ if (
99
+ self._max_duration is not None
100
+ and time.monotonic() - self._started_at > self._max_duration
101
+ ):
102
+ raise DatabaseQueryTimeoutError(
103
+ f"stream {self._query_name!r} exceeded max duration {self._max_duration}s",
104
+ query_name=self._query_name,
105
+ database_name=self._database,
106
+ role=self._role,
107
+ )
108
+ try:
109
+ row = await self._gen.__anext__()
110
+ except StopAsyncIteration:
111
+ self._metrics.observe(m.STREAM_ROWS, self._count, labels=self._labels)
112
+ raise
113
+ except Exception as exc:
114
+ raise classify(
115
+ exc, query_name=self._query_name, database_name=self._database, role=self._role
116
+ ) from exc
117
+ self._count += 1
118
+ return self._mapper(row)
119
+
120
+ async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool:
121
+ """Release the connection/cursor unconditionally; never suppresses the exception."""
122
+ await self._cleanup()
123
+ if self._span_cm is not None:
124
+ if self._span is not None:
125
+ self._span.set_attribute("db.rows_affected", self._count)
126
+ self._span_cm.__exit__(exc_type, exc_val, exc_tb)
127
+ return False
128
+
129
+ async def _cleanup(self) -> None:
130
+ if self._gen is not None:
131
+ with contextlib.suppress(Exception):
132
+ await self._gen.aclose()
133
+ self._gen = None
134
+ if self._conn is not None:
135
+ with contextlib.suppress(Exception):
136
+ await self._conn.close()
137
+ self._conn = None
@@ -0,0 +1,264 @@
1
+ """Explicit transactions, savepoints, commit-unknown, and cancellation cleanup (§11.3-11.6, §15).
2
+
3
+ A transaction owns a single connection for its lifetime. Commit failures where the outcome is
4
+ uncertain surface as :class:`DatabaseCommitUnknownError` (§15). On cancellation the transaction
5
+ is rolled back, the connection invalidated, and the cancellation re-raised — never swallowed
6
+ (§12.3).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import contextlib
12
+ import time
13
+ from collections.abc import AsyncIterator
14
+ from typing import Any
15
+
16
+ from sqlalchemy import text
17
+ from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
18
+
19
+ from .._core.errors import (
20
+ DatabaseCommitUnknownError,
21
+ classify,
22
+ is_connection_error,
23
+ )
24
+ from ..observability import logging as obslog
25
+ from ..observability import metrics as m
26
+ from ..observability.metrics import MetricsSink, NoopMetrics
27
+ from ._compat import is_cancellation, shield_from_cancellation
28
+ from .connection import AsyncConnectionScope, apply_statement_timeout_sql
29
+
30
+
31
+ class AsyncTransactionScope(AsyncConnectionScope):
32
+ """A connection scope wrapped in an explicit transaction, with savepoint support."""
33
+
34
+ def __init__(
35
+ self,
36
+ conn: AsyncConnection,
37
+ *,
38
+ is_postgres: bool,
39
+ default_timeout: float | None,
40
+ database: str,
41
+ shard_id: str,
42
+ role: str,
43
+ ) -> None:
44
+ """Wrap an already-``BEGIN``'d connection; not constructed directly by applications."""
45
+ super().__init__(
46
+ conn,
47
+ is_postgres=is_postgres,
48
+ default_timeout=default_timeout,
49
+ database=database,
50
+ shard_id=shard_id,
51
+ role=role,
52
+ )
53
+
54
+ @contextlib.asynccontextmanager
55
+ async def savepoint(self) -> AsyncIterator[AsyncTransactionScope]:
56
+ """A nested transaction (SAVEPOINT). Rolls back to the savepoint on error (§11.4)."""
57
+ nested = await self._conn.begin_nested()
58
+ try:
59
+ yield self
60
+ except BaseException:
61
+ await nested.rollback()
62
+ raise
63
+ else:
64
+ await nested.commit()
65
+
66
+
67
+ class _AsyncTransactionManager:
68
+ """Async context manager implementing the transaction lifecycle (§11.1, §11.3)."""
69
+
70
+ def __init__(
71
+ self,
72
+ engine: AsyncEngine,
73
+ *,
74
+ is_postgres: bool,
75
+ default_timeout: float | None,
76
+ database: str,
77
+ shard_id: str,
78
+ role: str,
79
+ isolation: str | None,
80
+ read_only: bool,
81
+ timeout: float | None,
82
+ lock_timeout: float | None,
83
+ query_name: str,
84
+ metrics: MetricsSink | None = None,
85
+ labels: dict[str, str] | None = None,
86
+ long_transaction_warning_seconds: float = 5.0,
87
+ span: Any = None,
88
+ deferrable: bool = False,
89
+ ) -> None:
90
+ """Constructed by ``AsyncDatabase.transaction(...)``; not built directly by applications."""
91
+ self._engine = engine
92
+ self._is_postgres = is_postgres
93
+ self._default_timeout = default_timeout
94
+ self._database = database
95
+ self._shard_id = shard_id
96
+ self._role = role
97
+ self._isolation = isolation
98
+ self._read_only = read_only
99
+ self._deferrable = deferrable
100
+ self._timeout = timeout if timeout is not None else default_timeout
101
+ self._lock_timeout = lock_timeout
102
+ self._query_name = query_name
103
+ self._metrics = metrics or NoopMetrics()
104
+ self._labels = labels or {"database": database, "shard": shard_id, "role": role}
105
+ self._long_transaction_warning = long_transaction_warning_seconds
106
+ self._span = span
107
+ self._conn: AsyncConnection | None = None
108
+ self._trans: Any = None
109
+ self._scope: AsyncTransactionScope | None = None
110
+ self._started_at: float = 0.0
111
+
112
+ async def __aenter__(self) -> AsyncTransactionScope:
113
+ """Acquire a connection, apply isolation/read-only/timeout settings, and ``BEGIN``."""
114
+ try:
115
+ self._conn = await self._engine.connect()
116
+ # Isolation level / read-only / deferrable must be set *before* BEGIN — SQLAlchemy's
117
+ # execution_options() applies them via the driver's native connection attributes
118
+ # where possible (e.g. psycopg's own .isolation_level/.read_only), not raw SQL
119
+ # (§11.5). isolation_level is dialect-portable; the postgresql_* options are PG-only.
120
+ opts = self._execution_options()
121
+ if opts:
122
+ self._conn = await self._conn.execution_options(**opts)
123
+ self._trans = await self._conn.begin()
124
+ await self._apply_settings(self._conn)
125
+ except BaseException as exc:
126
+ await self._cleanup_on_enter_failure()
127
+ if is_cancellation(exc):
128
+ raise
129
+ raise classify(
130
+ exc, query_name=self._query_name, database_name=self._database, role=self._role
131
+ ) from exc
132
+ self._started_at = time.monotonic()
133
+ self._scope = AsyncTransactionScope(
134
+ self._conn,
135
+ is_postgres=self._is_postgres,
136
+ default_timeout=self._timeout,
137
+ database=self._database,
138
+ shard_id=self._shard_id,
139
+ role=self._role,
140
+ )
141
+ return self._scope
142
+
143
+ def _execution_options(self) -> dict[str, Any]:
144
+ opts: dict[str, Any] = {}
145
+ if self._isolation:
146
+ opts["isolation_level"] = self._isolation.upper().replace("_", " ")
147
+ if self._is_postgres:
148
+ if self._read_only:
149
+ opts["postgresql_readonly"] = True
150
+ if self._deferrable:
151
+ opts["postgresql_deferrable"] = True
152
+ return opts
153
+
154
+ async def _apply_settings(self, conn: AsyncConnection) -> None:
155
+ # Statement/lock timeout are PostgreSQL GUCs with no portable execution_option — they
156
+ # have no driver-native equivalent, so SET LOCAL is the only option (§10, §12.1). Must
157
+ # run after BEGIN: SET LOCAL only takes effect inside an open transaction.
158
+ if not self._is_postgres:
159
+ return
160
+ stmt = apply_statement_timeout_sql(self._timeout)
161
+ if stmt is not None:
162
+ await conn.execute(text(stmt))
163
+ if self._lock_timeout is not None and self._lock_timeout > 0:
164
+ ms = max(int(self._lock_timeout * 1000), 1)
165
+ await conn.execute(text(f"SET LOCAL lock_timeout = {ms}"))
166
+
167
+ async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
168
+ """Commit on clean exit, else roll back; records metrics/logs/span for the outcome."""
169
+ assert self._conn is not None
170
+ outcome = "commit"
171
+ try:
172
+ if exc is not None:
173
+ outcome = "cancelled" if is_cancellation(exc) else "rollback"
174
+ await self._rollback(cancelled=is_cancellation(exc))
175
+ return False # propagate the original exception
176
+ try:
177
+ await self._commit()
178
+ except DatabaseCommitUnknownError:
179
+ outcome = "commit_unknown"
180
+ raise
181
+ except BaseException:
182
+ outcome = "rollback" # _commit() already attempted an internal rollback
183
+ raise
184
+ return False
185
+ finally:
186
+ duration = time.monotonic() - self._started_at
187
+ self._metrics.incr(m.TX_TOTAL, labels=self._labels)
188
+ self._metrics.observe(m.TX_DURATION, duration, labels=self._labels)
189
+ if outcome in ("rollback", "cancelled"):
190
+ self._metrics.incr(m.TX_ROLLBACK, labels=self._labels)
191
+ elif outcome == "commit_unknown":
192
+ self._metrics.incr(m.COMMIT_UNKNOWN, labels=self._labels)
193
+ if self._span is not None:
194
+ self._span.set_attribute("db.transaction.duration_ms", round(duration * 1000, 3))
195
+ if duration >= self._long_transaction_warning:
196
+ obslog.long_transaction_warning(
197
+ duration_ms=duration * 1000,
198
+ threshold_ms=self._long_transaction_warning * 1000,
199
+ database=self._database,
200
+ role=self._role,
201
+ outcome=outcome,
202
+ )
203
+ await self._release()
204
+
205
+ async def _commit(self) -> None:
206
+ assert self._trans is not None
207
+ try:
208
+ await self._trans.commit()
209
+ except BaseException as commit_exc:
210
+ # A failure during COMMIT may mean the transaction committed anyway (§15).
211
+ if is_connection_error(commit_exc):
212
+ raise DatabaseCommitUnknownError(
213
+ "connection failed during COMMIT; transaction outcome is unknown",
214
+ original=commit_exc,
215
+ database_name=self._database,
216
+ shard_id=self._shard_id,
217
+ role=self._role,
218
+ query_name=self._query_name,
219
+ ) from commit_exc
220
+ # Otherwise attempt rollback and classify normally.
221
+ with contextlib.suppress(Exception):
222
+ await self._trans.rollback()
223
+ raise classify(
224
+ commit_exc,
225
+ query_name=self._query_name,
226
+ database_name=self._database,
227
+ role=self._role,
228
+ ) from commit_exc
229
+
230
+ async def _rollback(self, *, cancelled: bool) -> None:
231
+ if self._trans is None:
232
+ return
233
+
234
+ async def _do_rollback() -> None:
235
+ try:
236
+ await self._trans.rollback()
237
+ except Exception:
238
+ # Rollback failed — the connection state is uncertain; invalidate it.
239
+ with contextlib.suppress(Exception):
240
+ await self._conn.invalidate() # type: ignore[union-attr]
241
+
242
+ await shield_from_cancellation(_do_rollback())
243
+ if cancelled:
244
+ # State after cancellation is uncertain; drop the connection from the pool.
245
+ with contextlib.suppress(Exception):
246
+ await self._conn.invalidate() # type: ignore[union-attr]
247
+
248
+ async def _release(self) -> None:
249
+ if self._conn is None:
250
+ return
251
+
252
+ async def _do_release() -> None:
253
+ with contextlib.suppress(Exception):
254
+ await self._conn.close() # type: ignore[union-attr]
255
+
256
+ await shield_from_cancellation(_do_release())
257
+ self._conn = None
258
+ self._trans = None
259
+
260
+ async def _cleanup_on_enter_failure(self) -> None:
261
+ if self._trans is not None:
262
+ with contextlib.suppress(Exception):
263
+ await self._trans.rollback()
264
+ await self._release()
@@ -0,0 +1 @@
1
+ """Shared, I/O-free kernel used by both the async and sync frontends."""
dbkit/_core/bulk.py ADDED
@@ -0,0 +1,91 @@
1
+ """Pure batch-sizing logic for bulk operations (§19.1).
2
+
3
+ Sizing is bounded by three limits so a single batch can't blow up memory or hit PostgreSQL's
4
+ 65535 bind-parameter ceiling: max rows, max total bind parameters, and (optionally) an
5
+ estimated payload-byte budget. This module is I/O-free and shared by both frontends.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Iterator, Mapping, Sequence
11
+ from dataclasses import dataclass
12
+ from typing import Any, Literal
13
+
14
+ # PostgreSQL caps a single statement at 65535 bind parameters.
15
+ PG_MAX_BIND_PARAMS = 65535
16
+
17
+ FailureMode = Literal["atomic", "best_effort", "split_on_failure"]
18
+ #: "execute_many" binds N params/row (SQLAlchemy's own multi-row translation, any dialect).
19
+ #: "unnest" binds one array param per column regardless of row count (PostgreSQL only, §19).
20
+ InsertStrategy = Literal["execute_many", "unnest"]
21
+
22
+
23
+ @dataclass(frozen=True, slots=True)
24
+ class BulkLimits:
25
+ """The three independent caps a bulk batch is sized against (§19.1)."""
26
+
27
+ max_rows: int = 1000
28
+ max_params: int = PG_MAX_BIND_PARAMS
29
+ max_payload_bytes: int | None = None
30
+
31
+
32
+ def estimate_row_bytes(row: Mapping[str, Any]) -> int:
33
+ """Cheap upper-bound estimate of one row's serialized payload size, in bytes.
34
+
35
+ Not exact (actual bind-parameter wire encoding varies by driver/type), but good enough to
36
+ bound a batch's total payload size to roughly ``max_payload_bytes`` rather than not bounding
37
+ it at all — the field existed on :class:`BulkLimits`/``BulkConfig`` but ``resolve_batch_rows``
38
+ never read it, so a batch of many small-column, wide-value (e.g. large ``text``/``bytea``)
39
+ rows had no actual byte-size ceiling despite the config implying one (performance review §9).
40
+ """
41
+ total = 0
42
+ for value in row.values():
43
+ if value is None:
44
+ continue
45
+ if isinstance(value, (bytes, bytearray)):
46
+ total += len(value)
47
+ else:
48
+ total += len(str(value).encode("utf-8", errors="ignore"))
49
+ return total
50
+
51
+
52
+ def resolve_batch_rows(
53
+ n_columns: int,
54
+ requested: int | None,
55
+ limits: BulkLimits,
56
+ *,
57
+ sample_row: Mapping[str, Any] | None = None,
58
+ ) -> int:
59
+ """Largest safe batch size in rows, given per-row column count and the limits.
60
+
61
+ When ``limits.max_payload_bytes`` is set and a ``sample_row`` is supplied, the ceiling is
62
+ also bounded by an estimated per-row byte size (:func:`estimate_row_bytes`) — a single
63
+ representative row is enough since batches are homogeneous (same columns, similar value
64
+ sizes), and re-estimating every row would cost more than the batching itself saves.
65
+ """
66
+ if n_columns <= 0:
67
+ return max(requested or limits.max_rows, 1)
68
+ by_params = max(limits.max_params // n_columns, 1)
69
+ ceiling = min(requested or limits.max_rows, limits.max_rows, by_params)
70
+ if limits.max_payload_bytes is not None and sample_row is not None:
71
+ row_bytes = estimate_row_bytes(sample_row)
72
+ if row_bytes > 0:
73
+ by_bytes = max(limits.max_payload_bytes // row_bytes, 1)
74
+ ceiling = min(ceiling, by_bytes)
75
+ return max(ceiling, 1)
76
+
77
+
78
+ def iter_batches(rows: Sequence[Any], batch_size: int) -> Iterator[Sequence[Any]]:
79
+ """Yield contiguous ``batch_size`` slices of ``rows``."""
80
+ if batch_size <= 0:
81
+ batch_size = 1
82
+ for start in range(0, len(rows), batch_size):
83
+ yield rows[start : start + batch_size]
84
+
85
+
86
+ def column_names(rows: Sequence[Mapping[str, Any]]) -> list[str]:
87
+ """Ordered column names inferred from the first row; every row must share them."""
88
+ if not rows:
89
+ return []
90
+ first = rows[0]
91
+ return list(first.keys())