dbkit-sql 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbkit/__init__.py +77 -0
- dbkit/_async/__init__.py +17 -0
- dbkit/_async/_compat.py +111 -0
- dbkit/_async/connection.py +328 -0
- dbkit/_async/database.py +884 -0
- dbkit/_async/engine.py +232 -0
- dbkit/_async/executor.py +329 -0
- dbkit/_async/health.py +53 -0
- dbkit/_async/resilience.py +148 -0
- dbkit/_async/streaming.py +137 -0
- dbkit/_async/transaction.py +264 -0
- dbkit/_core/__init__.py +1 -0
- dbkit/_core/bulk.py +91 -0
- dbkit/_core/circuit.py +130 -0
- dbkit/_core/config.py +613 -0
- dbkit/_core/errors/__init__.py +74 -0
- dbkit/_core/errors/base.py +344 -0
- dbkit/_core/errors/classifier.py +138 -0
- dbkit/_core/errors/redaction.py +96 -0
- dbkit/_core/errors/sqlstate.py +74 -0
- dbkit/_core/idempotency_lint.py +57 -0
- dbkit/_core/keys.py +29 -0
- dbkit/_core/policies.py +105 -0
- dbkit/_core/query.py +142 -0
- dbkit/_core/result.py +109 -0
- dbkit/_core/routing.py +248 -0
- dbkit/_pool.py +202 -0
- dbkit/_sync/__init__.py +20 -0
- dbkit/_sync/_compat.py +82 -0
- dbkit/_sync/connection.py +331 -0
- dbkit/_sync/database.py +887 -0
- dbkit/_sync/engine.py +235 -0
- dbkit/_sync/executor.py +332 -0
- dbkit/_sync/health.py +56 -0
- dbkit/_sync/resilience.py +151 -0
- dbkit/_sync/streaming.py +140 -0
- dbkit/_sync/transaction.py +267 -0
- dbkit/cli/__init__.py +1 -0
- dbkit/cli/main.py +256 -0
- dbkit/errors.py +6 -0
- dbkit/integrations/__init__.py +21 -0
- dbkit/integrations/batch.py +81 -0
- dbkit/integrations/inbox.py +153 -0
- dbkit/observability/__init__.py +21 -0
- dbkit/observability/logging.py +154 -0
- dbkit/observability/metrics.py +110 -0
- dbkit/observability/otel_metrics.py +69 -0
- dbkit/observability/prometheus.py +78 -0
- dbkit/observability/tracing.py +128 -0
- dbkit/postgres/__init__.py +1 -0
- dbkit/postgres/copy.py +77 -0
- dbkit/postgres/pipeline.py +48 -0
- dbkit/postgres/unnest.py +73 -0
- dbkit/testing/__init__.py +1 -0
- dbkit_sql-0.1.0.dist-info/METADATA +555 -0
- dbkit_sql-0.1.0.dist-info/RECORD +59 -0
- dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
- dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
- dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Retry executor and concurrency limiter (§14, §17).
|
|
2
|
+
|
|
3
|
+
The retry *decisions* (idempotency gating, backoff shape) live in the pure ``_core.policies``
|
|
4
|
+
module; this is the async loop that applies them, respects the caller deadline, and drives the
|
|
5
|
+
circuit breaker. The concurrency limiter bounds how many operations may use the database at
|
|
6
|
+
once, independently of pool size — and is acquired *before* pool checkout so queueing happens
|
|
7
|
+
in cheap asyncio waiters, not held connections (§17).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import contextlib
|
|
13
|
+
import random
|
|
14
|
+
import time
|
|
15
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
16
|
+
from typing import TypeVar
|
|
17
|
+
|
|
18
|
+
from .._core.circuit import CircuitBreaker, CircuitState, counts_as_failure
|
|
19
|
+
from .._core.config import RetryConfig
|
|
20
|
+
from .._core.errors import DatabaseCircuitOpenError, DatabaseError, DatabaseOverloadedError
|
|
21
|
+
from .._core.policies import backoff_delay_ms, should_retry
|
|
22
|
+
from .._core.query import Query
|
|
23
|
+
from ..observability import metrics as m
|
|
24
|
+
from ..observability.metrics import MetricsSink
|
|
25
|
+
from ._compat import semaphore_acquire, sleep
|
|
26
|
+
|
|
27
|
+
T = TypeVar("T")
|
|
28
|
+
|
|
29
|
+
#: Numeric value of each circuit-breaker state, for the ``CIRCUIT_STATE`` gauge (§16).
|
|
30
|
+
_CIRCUIT_STATE_VALUE = {
|
|
31
|
+
CircuitState.CLOSED: 0.0,
|
|
32
|
+
CircuitState.HALF_OPEN: 1.0,
|
|
33
|
+
CircuitState.OPEN: 2.0,
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _emit_circuit_state(
|
|
38
|
+
breaker: CircuitBreaker, now: float, metrics: MetricsSink, labels: dict[str, str]
|
|
39
|
+
) -> None:
|
|
40
|
+
metrics.gauge(m.CIRCUIT_STATE, _CIRCUIT_STATE_VALUE[breaker.state(now)], labels=labels)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
async def run_with_retries(
|
|
44
|
+
operation: Callable[[], Awaitable[T]],
|
|
45
|
+
*,
|
|
46
|
+
query: Query | None,
|
|
47
|
+
retry: RetryConfig,
|
|
48
|
+
breaker: CircuitBreaker | None,
|
|
49
|
+
metrics: MetricsSink,
|
|
50
|
+
labels: dict[str, str],
|
|
51
|
+
idempotent_override: bool | None = None,
|
|
52
|
+
deadline: float | None = None,
|
|
53
|
+
) -> T:
|
|
54
|
+
"""Run ``operation`` with conservative, idempotency-gated retries (§14).
|
|
55
|
+
|
|
56
|
+
Each attempt runs a *fresh* operation (new connection/transaction) — the caller supplies a
|
|
57
|
+
thunk that re-does the work. Non-retryable errors, exhausted budgets, unknown-commit
|
|
58
|
+
outcomes, and non-idempotent writes propagate immediately.
|
|
59
|
+
|
|
60
|
+
Total time spent retrying (backoff delays only, not attempt execution time) is bounded by
|
|
61
|
+
``retry.maximum_total_ms`` regardless of ``attempts``/``deadline`` — without this, a caller
|
|
62
|
+
with a generous ``attempts`` count and no explicit ``deadline`` has no bound on how long a
|
|
63
|
+
single logical call can spend retrying (performance review §5/§7).
|
|
64
|
+
"""
|
|
65
|
+
attempt = 0
|
|
66
|
+
started = time.monotonic()
|
|
67
|
+
while True:
|
|
68
|
+
attempt += 1
|
|
69
|
+
now = time.monotonic()
|
|
70
|
+
if breaker is not None:
|
|
71
|
+
allowed = breaker.allow(now)
|
|
72
|
+
_emit_circuit_state(breaker, now, metrics, labels)
|
|
73
|
+
if not allowed:
|
|
74
|
+
raise DatabaseCircuitOpenError(
|
|
75
|
+
"circuit is open for this target",
|
|
76
|
+
database_name=labels.get("database"),
|
|
77
|
+
role=labels.get("role"),
|
|
78
|
+
query_name=labels.get("query_name"),
|
|
79
|
+
)
|
|
80
|
+
try:
|
|
81
|
+
result = await operation()
|
|
82
|
+
except DatabaseError as err:
|
|
83
|
+
if breaker is not None:
|
|
84
|
+
if counts_as_failure(err):
|
|
85
|
+
breaker.on_failure(time.monotonic())
|
|
86
|
+
_emit_circuit_state(breaker, time.monotonic(), metrics, labels)
|
|
87
|
+
if not should_retry(
|
|
88
|
+
err,
|
|
89
|
+
query=query,
|
|
90
|
+
config=retry,
|
|
91
|
+
attempt=attempt,
|
|
92
|
+
idempotent_override=idempotent_override,
|
|
93
|
+
):
|
|
94
|
+
raise
|
|
95
|
+
delay = backoff_delay_ms(attempt, retry, rand=random.random()) / 1000.0
|
|
96
|
+
# never sleep past the caller deadline
|
|
97
|
+
if deadline is not None and time.monotonic() + delay >= deadline:
|
|
98
|
+
raise
|
|
99
|
+
# never sleep past the retry budget, independent of any caller deadline
|
|
100
|
+
elapsed_ms = (time.monotonic() - started) * 1000.0
|
|
101
|
+
if elapsed_ms + delay * 1000.0 >= retry.maximum_total_ms:
|
|
102
|
+
raise
|
|
103
|
+
metrics.incr(m.OP_RETRIES, labels={**labels, "retry_attempt": str(attempt)})
|
|
104
|
+
await sleep(delay)
|
|
105
|
+
continue
|
|
106
|
+
else:
|
|
107
|
+
if breaker is not None:
|
|
108
|
+
breaker.on_success(time.monotonic())
|
|
109
|
+
_emit_circuit_state(breaker, time.monotonic(), metrics, labels)
|
|
110
|
+
return result
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class ConcurrencyLimiter:
|
|
114
|
+
"""Per-tier semaphores bounding concurrent database use (§17).
|
|
115
|
+
|
|
116
|
+
Tiers: ``"database"`` (overall), ``"reads"``, ``"writes"``, ``"bulk"``. A tier with no
|
|
117
|
+
configured limit is unbounded (a no-op context).
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
def __init__(self, limits: dict[str, int | None]) -> None:
|
|
121
|
+
"""``limits`` maps tier name to its cap; a missing/``None``/``<=0`` limit is unbounded."""
|
|
122
|
+
# asyncio.Semaphore in the async build; threading.Semaphore after unasync.
|
|
123
|
+
import asyncio
|
|
124
|
+
|
|
125
|
+
self._sems: dict[str, asyncio.Semaphore] = {
|
|
126
|
+
tier: asyncio.Semaphore(n) for tier, n in limits.items() if n and n > 0
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
@contextlib.asynccontextmanager
|
|
130
|
+
async def acquire(self, tier: str, *, timeout: float | None = None) -> AsyncIterator[None]:
|
|
131
|
+
"""Hold one slot of ``tier`` for the block's duration (no-op if unbounded).
|
|
132
|
+
|
|
133
|
+
Raises :class:`DatabaseOverloadedError` if no slot frees up within ``timeout`` —
|
|
134
|
+
without this bound, a saturated tier would queue callers indefinitely, invisibly to
|
|
135
|
+
dbkit's own timeout/deadline machinery (§17).
|
|
136
|
+
"""
|
|
137
|
+
sem = self._sems.get(tier)
|
|
138
|
+
if sem is None:
|
|
139
|
+
yield
|
|
140
|
+
return
|
|
141
|
+
if not await semaphore_acquire(sem, timeout):
|
|
142
|
+
raise DatabaseOverloadedError(
|
|
143
|
+
f"concurrency limit reached for tier {tier!r}; no slot freed up within {timeout}s"
|
|
144
|
+
)
|
|
145
|
+
try:
|
|
146
|
+
yield
|
|
147
|
+
finally:
|
|
148
|
+
sem.release()
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Streaming large result sets without loading them into memory (§20).
|
|
2
|
+
|
|
3
|
+
``AsyncResultStream`` holds one connection for the lifetime of the stream and pulls rows from a
|
|
4
|
+
server-side cursor in bounded batches. It is both an async context manager (guaranteeing the
|
|
5
|
+
connection is released) and an async iterator. Because a stream holds its connection across
|
|
6
|
+
many yields, it deliberately bypasses the retry wrapper — a partially consumed stream cannot be
|
|
7
|
+
transparently restarted.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import contextlib
|
|
13
|
+
import time
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
|
|
17
|
+
|
|
18
|
+
from .._core import result as result_mod
|
|
19
|
+
from .._core.errors import DatabaseQueryTimeoutError, classify
|
|
20
|
+
from .._core.query import Statement
|
|
21
|
+
from ..observability import metrics as m
|
|
22
|
+
from ..observability.metrics import MetricsSink
|
|
23
|
+
from ..observability.tracing import Tracer
|
|
24
|
+
from ._compat import stream_mappings
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class AsyncResultStream:
|
|
28
|
+
"""A bounded, connection-owning async row stream."""
|
|
29
|
+
|
|
30
|
+
def __init__(
|
|
31
|
+
self,
|
|
32
|
+
engine: AsyncEngine,
|
|
33
|
+
statement: Statement,
|
|
34
|
+
params: Any,
|
|
35
|
+
*,
|
|
36
|
+
batch_size: int,
|
|
37
|
+
map_to: Any,
|
|
38
|
+
database: str,
|
|
39
|
+
role: str,
|
|
40
|
+
query_name: str,
|
|
41
|
+
metrics: MetricsSink,
|
|
42
|
+
labels: dict[str, str],
|
|
43
|
+
max_duration: float | None = None,
|
|
44
|
+
tracer: Tracer | None = None,
|
|
45
|
+
shard_id: str | None = None,
|
|
46
|
+
) -> None:
|
|
47
|
+
"""Constructed by ``AsyncDatabase.stream(...)``; not intended to be built directly."""
|
|
48
|
+
self._engine = engine
|
|
49
|
+
self._statement = statement
|
|
50
|
+
self._params = params
|
|
51
|
+
self._batch_size = batch_size
|
|
52
|
+
self._mapper = result_mod.build_mapper(map_to)
|
|
53
|
+
self._database = database
|
|
54
|
+
self._role = role
|
|
55
|
+
self._query_name = query_name
|
|
56
|
+
self._metrics = metrics
|
|
57
|
+
self._labels = labels
|
|
58
|
+
self._max_duration = max_duration
|
|
59
|
+
self._tracer = tracer
|
|
60
|
+
self._shard_id = shard_id
|
|
61
|
+
self._conn: AsyncConnection | None = None
|
|
62
|
+
self._gen: Any = None
|
|
63
|
+
self._count = 0
|
|
64
|
+
self._started_at = 0.0
|
|
65
|
+
self._span_cm: Any = None
|
|
66
|
+
self._span: Any = None
|
|
67
|
+
|
|
68
|
+
async def __aenter__(self) -> AsyncResultStream:
|
|
69
|
+
"""Open the underlying connection and server-side cursor."""
|
|
70
|
+
if self._tracer is not None:
|
|
71
|
+
self._span_cm = self._tracer.span(
|
|
72
|
+
"dbkit.stream",
|
|
73
|
+
operation_type="stream",
|
|
74
|
+
query_name=self._query_name,
|
|
75
|
+
database=self._database,
|
|
76
|
+
shard=self._shard_id,
|
|
77
|
+
role=self._role,
|
|
78
|
+
)
|
|
79
|
+
self._span = self._span_cm.__enter__()
|
|
80
|
+
try:
|
|
81
|
+
self._conn = await self._engine.connect()
|
|
82
|
+
self._gen = stream_mappings(self._conn, self._statement, self._params, self._batch_size)
|
|
83
|
+
except Exception as exc:
|
|
84
|
+
await self._cleanup()
|
|
85
|
+
raise classify(
|
|
86
|
+
exc, query_name=self._query_name, database_name=self._database, role=self._role
|
|
87
|
+
) from exc
|
|
88
|
+
self._started_at = time.monotonic()
|
|
89
|
+
return self
|
|
90
|
+
|
|
91
|
+
def __aiter__(self) -> AsyncResultStream:
|
|
92
|
+
"""Returns ``self`` — the stream is its own iterator."""
|
|
93
|
+
return self
|
|
94
|
+
|
|
95
|
+
async def __anext__(self) -> Any:
|
|
96
|
+
"""The next mapped row, or raise :class:`StopAsyncIteration`/a timeout error."""
|
|
97
|
+
assert self._gen is not None
|
|
98
|
+
if (
|
|
99
|
+
self._max_duration is not None
|
|
100
|
+
and time.monotonic() - self._started_at > self._max_duration
|
|
101
|
+
):
|
|
102
|
+
raise DatabaseQueryTimeoutError(
|
|
103
|
+
f"stream {self._query_name!r} exceeded max duration {self._max_duration}s",
|
|
104
|
+
query_name=self._query_name,
|
|
105
|
+
database_name=self._database,
|
|
106
|
+
role=self._role,
|
|
107
|
+
)
|
|
108
|
+
try:
|
|
109
|
+
row = await self._gen.__anext__()
|
|
110
|
+
except StopAsyncIteration:
|
|
111
|
+
self._metrics.observe(m.STREAM_ROWS, self._count, labels=self._labels)
|
|
112
|
+
raise
|
|
113
|
+
except Exception as exc:
|
|
114
|
+
raise classify(
|
|
115
|
+
exc, query_name=self._query_name, database_name=self._database, role=self._role
|
|
116
|
+
) from exc
|
|
117
|
+
self._count += 1
|
|
118
|
+
return self._mapper(row)
|
|
119
|
+
|
|
120
|
+
async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> bool:
|
|
121
|
+
"""Release the connection/cursor unconditionally; never suppresses the exception."""
|
|
122
|
+
await self._cleanup()
|
|
123
|
+
if self._span_cm is not None:
|
|
124
|
+
if self._span is not None:
|
|
125
|
+
self._span.set_attribute("db.rows_affected", self._count)
|
|
126
|
+
self._span_cm.__exit__(exc_type, exc_val, exc_tb)
|
|
127
|
+
return False
|
|
128
|
+
|
|
129
|
+
async def _cleanup(self) -> None:
|
|
130
|
+
if self._gen is not None:
|
|
131
|
+
with contextlib.suppress(Exception):
|
|
132
|
+
await self._gen.aclose()
|
|
133
|
+
self._gen = None
|
|
134
|
+
if self._conn is not None:
|
|
135
|
+
with contextlib.suppress(Exception):
|
|
136
|
+
await self._conn.close()
|
|
137
|
+
self._conn = None
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"""Explicit transactions, savepoints, commit-unknown, and cancellation cleanup (§11.3-11.6, §15).
|
|
2
|
+
|
|
3
|
+
A transaction owns a single connection for its lifetime. Commit failures where the outcome is
|
|
4
|
+
uncertain surface as :class:`DatabaseCommitUnknownError` (§15). On cancellation the transaction
|
|
5
|
+
is rolled back, the connection invalidated, and the cancellation re-raised — never swallowed
|
|
6
|
+
(§12.3).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import time
|
|
13
|
+
from collections.abc import AsyncIterator
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from sqlalchemy import text
|
|
17
|
+
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
|
|
18
|
+
|
|
19
|
+
from .._core.errors import (
|
|
20
|
+
DatabaseCommitUnknownError,
|
|
21
|
+
classify,
|
|
22
|
+
is_connection_error,
|
|
23
|
+
)
|
|
24
|
+
from ..observability import logging as obslog
|
|
25
|
+
from ..observability import metrics as m
|
|
26
|
+
from ..observability.metrics import MetricsSink, NoopMetrics
|
|
27
|
+
from ._compat import is_cancellation, shield_from_cancellation
|
|
28
|
+
from .connection import AsyncConnectionScope, apply_statement_timeout_sql
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class AsyncTransactionScope(AsyncConnectionScope):
|
|
32
|
+
"""A connection scope wrapped in an explicit transaction, with savepoint support."""
|
|
33
|
+
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
conn: AsyncConnection,
|
|
37
|
+
*,
|
|
38
|
+
is_postgres: bool,
|
|
39
|
+
default_timeout: float | None,
|
|
40
|
+
database: str,
|
|
41
|
+
shard_id: str,
|
|
42
|
+
role: str,
|
|
43
|
+
) -> None:
|
|
44
|
+
"""Wrap an already-``BEGIN``'d connection; not constructed directly by applications."""
|
|
45
|
+
super().__init__(
|
|
46
|
+
conn,
|
|
47
|
+
is_postgres=is_postgres,
|
|
48
|
+
default_timeout=default_timeout,
|
|
49
|
+
database=database,
|
|
50
|
+
shard_id=shard_id,
|
|
51
|
+
role=role,
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
@contextlib.asynccontextmanager
|
|
55
|
+
async def savepoint(self) -> AsyncIterator[AsyncTransactionScope]:
|
|
56
|
+
"""A nested transaction (SAVEPOINT). Rolls back to the savepoint on error (§11.4)."""
|
|
57
|
+
nested = await self._conn.begin_nested()
|
|
58
|
+
try:
|
|
59
|
+
yield self
|
|
60
|
+
except BaseException:
|
|
61
|
+
await nested.rollback()
|
|
62
|
+
raise
|
|
63
|
+
else:
|
|
64
|
+
await nested.commit()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class _AsyncTransactionManager:
|
|
68
|
+
"""Async context manager implementing the transaction lifecycle (§11.1, §11.3)."""
|
|
69
|
+
|
|
70
|
+
def __init__(
|
|
71
|
+
self,
|
|
72
|
+
engine: AsyncEngine,
|
|
73
|
+
*,
|
|
74
|
+
is_postgres: bool,
|
|
75
|
+
default_timeout: float | None,
|
|
76
|
+
database: str,
|
|
77
|
+
shard_id: str,
|
|
78
|
+
role: str,
|
|
79
|
+
isolation: str | None,
|
|
80
|
+
read_only: bool,
|
|
81
|
+
timeout: float | None,
|
|
82
|
+
lock_timeout: float | None,
|
|
83
|
+
query_name: str,
|
|
84
|
+
metrics: MetricsSink | None = None,
|
|
85
|
+
labels: dict[str, str] | None = None,
|
|
86
|
+
long_transaction_warning_seconds: float = 5.0,
|
|
87
|
+
span: Any = None,
|
|
88
|
+
deferrable: bool = False,
|
|
89
|
+
) -> None:
|
|
90
|
+
"""Constructed by ``AsyncDatabase.transaction(...)``; not built directly by applications."""
|
|
91
|
+
self._engine = engine
|
|
92
|
+
self._is_postgres = is_postgres
|
|
93
|
+
self._default_timeout = default_timeout
|
|
94
|
+
self._database = database
|
|
95
|
+
self._shard_id = shard_id
|
|
96
|
+
self._role = role
|
|
97
|
+
self._isolation = isolation
|
|
98
|
+
self._read_only = read_only
|
|
99
|
+
self._deferrable = deferrable
|
|
100
|
+
self._timeout = timeout if timeout is not None else default_timeout
|
|
101
|
+
self._lock_timeout = lock_timeout
|
|
102
|
+
self._query_name = query_name
|
|
103
|
+
self._metrics = metrics or NoopMetrics()
|
|
104
|
+
self._labels = labels or {"database": database, "shard": shard_id, "role": role}
|
|
105
|
+
self._long_transaction_warning = long_transaction_warning_seconds
|
|
106
|
+
self._span = span
|
|
107
|
+
self._conn: AsyncConnection | None = None
|
|
108
|
+
self._trans: Any = None
|
|
109
|
+
self._scope: AsyncTransactionScope | None = None
|
|
110
|
+
self._started_at: float = 0.0
|
|
111
|
+
|
|
112
|
+
async def __aenter__(self) -> AsyncTransactionScope:
|
|
113
|
+
"""Acquire a connection, apply isolation/read-only/timeout settings, and ``BEGIN``."""
|
|
114
|
+
try:
|
|
115
|
+
self._conn = await self._engine.connect()
|
|
116
|
+
# Isolation level / read-only / deferrable must be set *before* BEGIN — SQLAlchemy's
|
|
117
|
+
# execution_options() applies them via the driver's native connection attributes
|
|
118
|
+
# where possible (e.g. psycopg's own .isolation_level/.read_only), not raw SQL
|
|
119
|
+
# (§11.5). isolation_level is dialect-portable; the postgresql_* options are PG-only.
|
|
120
|
+
opts = self._execution_options()
|
|
121
|
+
if opts:
|
|
122
|
+
self._conn = await self._conn.execution_options(**opts)
|
|
123
|
+
self._trans = await self._conn.begin()
|
|
124
|
+
await self._apply_settings(self._conn)
|
|
125
|
+
except BaseException as exc:
|
|
126
|
+
await self._cleanup_on_enter_failure()
|
|
127
|
+
if is_cancellation(exc):
|
|
128
|
+
raise
|
|
129
|
+
raise classify(
|
|
130
|
+
exc, query_name=self._query_name, database_name=self._database, role=self._role
|
|
131
|
+
) from exc
|
|
132
|
+
self._started_at = time.monotonic()
|
|
133
|
+
self._scope = AsyncTransactionScope(
|
|
134
|
+
self._conn,
|
|
135
|
+
is_postgres=self._is_postgres,
|
|
136
|
+
default_timeout=self._timeout,
|
|
137
|
+
database=self._database,
|
|
138
|
+
shard_id=self._shard_id,
|
|
139
|
+
role=self._role,
|
|
140
|
+
)
|
|
141
|
+
return self._scope
|
|
142
|
+
|
|
143
|
+
def _execution_options(self) -> dict[str, Any]:
|
|
144
|
+
opts: dict[str, Any] = {}
|
|
145
|
+
if self._isolation:
|
|
146
|
+
opts["isolation_level"] = self._isolation.upper().replace("_", " ")
|
|
147
|
+
if self._is_postgres:
|
|
148
|
+
if self._read_only:
|
|
149
|
+
opts["postgresql_readonly"] = True
|
|
150
|
+
if self._deferrable:
|
|
151
|
+
opts["postgresql_deferrable"] = True
|
|
152
|
+
return opts
|
|
153
|
+
|
|
154
|
+
async def _apply_settings(self, conn: AsyncConnection) -> None:
|
|
155
|
+
# Statement/lock timeout are PostgreSQL GUCs with no portable execution_option — they
|
|
156
|
+
# have no driver-native equivalent, so SET LOCAL is the only option (§10, §12.1). Must
|
|
157
|
+
# run after BEGIN: SET LOCAL only takes effect inside an open transaction.
|
|
158
|
+
if not self._is_postgres:
|
|
159
|
+
return
|
|
160
|
+
stmt = apply_statement_timeout_sql(self._timeout)
|
|
161
|
+
if stmt is not None:
|
|
162
|
+
await conn.execute(text(stmt))
|
|
163
|
+
if self._lock_timeout is not None and self._lock_timeout > 0:
|
|
164
|
+
ms = max(int(self._lock_timeout * 1000), 1)
|
|
165
|
+
await conn.execute(text(f"SET LOCAL lock_timeout = {ms}"))
|
|
166
|
+
|
|
167
|
+
async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> bool:
|
|
168
|
+
"""Commit on clean exit, else roll back; records metrics/logs/span for the outcome."""
|
|
169
|
+
assert self._conn is not None
|
|
170
|
+
outcome = "commit"
|
|
171
|
+
try:
|
|
172
|
+
if exc is not None:
|
|
173
|
+
outcome = "cancelled" if is_cancellation(exc) else "rollback"
|
|
174
|
+
await self._rollback(cancelled=is_cancellation(exc))
|
|
175
|
+
return False # propagate the original exception
|
|
176
|
+
try:
|
|
177
|
+
await self._commit()
|
|
178
|
+
except DatabaseCommitUnknownError:
|
|
179
|
+
outcome = "commit_unknown"
|
|
180
|
+
raise
|
|
181
|
+
except BaseException:
|
|
182
|
+
outcome = "rollback" # _commit() already attempted an internal rollback
|
|
183
|
+
raise
|
|
184
|
+
return False
|
|
185
|
+
finally:
|
|
186
|
+
duration = time.monotonic() - self._started_at
|
|
187
|
+
self._metrics.incr(m.TX_TOTAL, labels=self._labels)
|
|
188
|
+
self._metrics.observe(m.TX_DURATION, duration, labels=self._labels)
|
|
189
|
+
if outcome in ("rollback", "cancelled"):
|
|
190
|
+
self._metrics.incr(m.TX_ROLLBACK, labels=self._labels)
|
|
191
|
+
elif outcome == "commit_unknown":
|
|
192
|
+
self._metrics.incr(m.COMMIT_UNKNOWN, labels=self._labels)
|
|
193
|
+
if self._span is not None:
|
|
194
|
+
self._span.set_attribute("db.transaction.duration_ms", round(duration * 1000, 3))
|
|
195
|
+
if duration >= self._long_transaction_warning:
|
|
196
|
+
obslog.long_transaction_warning(
|
|
197
|
+
duration_ms=duration * 1000,
|
|
198
|
+
threshold_ms=self._long_transaction_warning * 1000,
|
|
199
|
+
database=self._database,
|
|
200
|
+
role=self._role,
|
|
201
|
+
outcome=outcome,
|
|
202
|
+
)
|
|
203
|
+
await self._release()
|
|
204
|
+
|
|
205
|
+
async def _commit(self) -> None:
|
|
206
|
+
assert self._trans is not None
|
|
207
|
+
try:
|
|
208
|
+
await self._trans.commit()
|
|
209
|
+
except BaseException as commit_exc:
|
|
210
|
+
# A failure during COMMIT may mean the transaction committed anyway (§15).
|
|
211
|
+
if is_connection_error(commit_exc):
|
|
212
|
+
raise DatabaseCommitUnknownError(
|
|
213
|
+
"connection failed during COMMIT; transaction outcome is unknown",
|
|
214
|
+
original=commit_exc,
|
|
215
|
+
database_name=self._database,
|
|
216
|
+
shard_id=self._shard_id,
|
|
217
|
+
role=self._role,
|
|
218
|
+
query_name=self._query_name,
|
|
219
|
+
) from commit_exc
|
|
220
|
+
# Otherwise attempt rollback and classify normally.
|
|
221
|
+
with contextlib.suppress(Exception):
|
|
222
|
+
await self._trans.rollback()
|
|
223
|
+
raise classify(
|
|
224
|
+
commit_exc,
|
|
225
|
+
query_name=self._query_name,
|
|
226
|
+
database_name=self._database,
|
|
227
|
+
role=self._role,
|
|
228
|
+
) from commit_exc
|
|
229
|
+
|
|
230
|
+
async def _rollback(self, *, cancelled: bool) -> None:
|
|
231
|
+
if self._trans is None:
|
|
232
|
+
return
|
|
233
|
+
|
|
234
|
+
async def _do_rollback() -> None:
|
|
235
|
+
try:
|
|
236
|
+
await self._trans.rollback()
|
|
237
|
+
except Exception:
|
|
238
|
+
# Rollback failed — the connection state is uncertain; invalidate it.
|
|
239
|
+
with contextlib.suppress(Exception):
|
|
240
|
+
await self._conn.invalidate() # type: ignore[union-attr]
|
|
241
|
+
|
|
242
|
+
await shield_from_cancellation(_do_rollback())
|
|
243
|
+
if cancelled:
|
|
244
|
+
# State after cancellation is uncertain; drop the connection from the pool.
|
|
245
|
+
with contextlib.suppress(Exception):
|
|
246
|
+
await self._conn.invalidate() # type: ignore[union-attr]
|
|
247
|
+
|
|
248
|
+
async def _release(self) -> None:
|
|
249
|
+
if self._conn is None:
|
|
250
|
+
return
|
|
251
|
+
|
|
252
|
+
async def _do_release() -> None:
|
|
253
|
+
with contextlib.suppress(Exception):
|
|
254
|
+
await self._conn.close() # type: ignore[union-attr]
|
|
255
|
+
|
|
256
|
+
await shield_from_cancellation(_do_release())
|
|
257
|
+
self._conn = None
|
|
258
|
+
self._trans = None
|
|
259
|
+
|
|
260
|
+
async def _cleanup_on_enter_failure(self) -> None:
|
|
261
|
+
if self._trans is not None:
|
|
262
|
+
with contextlib.suppress(Exception):
|
|
263
|
+
await self._trans.rollback()
|
|
264
|
+
await self._release()
|
dbkit/_core/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Shared, I/O-free kernel used by both the async and sync frontends."""
|
dbkit/_core/bulk.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Pure batch-sizing logic for bulk operations (§19.1).
|
|
2
|
+
|
|
3
|
+
Sizing is bounded by three limits so a single batch can't blow up memory or hit PostgreSQL's
|
|
4
|
+
65535 bind-parameter ceiling: max rows, max total bind parameters, and (optionally) an
|
|
5
|
+
estimated payload-byte budget. This module is I/O-free and shared by both frontends.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Iterator, Mapping, Sequence
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any, Literal
|
|
13
|
+
|
|
14
|
+
# PostgreSQL caps a single statement at 65535 bind parameters.
|
|
15
|
+
PG_MAX_BIND_PARAMS = 65535
|
|
16
|
+
|
|
17
|
+
FailureMode = Literal["atomic", "best_effort", "split_on_failure"]
|
|
18
|
+
#: "execute_many" binds N params/row (SQLAlchemy's own multi-row translation, any dialect).
|
|
19
|
+
#: "unnest" binds one array param per column regardless of row count (PostgreSQL only, §19).
|
|
20
|
+
InsertStrategy = Literal["execute_many", "unnest"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class BulkLimits:
|
|
25
|
+
"""The three independent caps a bulk batch is sized against (§19.1)."""
|
|
26
|
+
|
|
27
|
+
max_rows: int = 1000
|
|
28
|
+
max_params: int = PG_MAX_BIND_PARAMS
|
|
29
|
+
max_payload_bytes: int | None = None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def estimate_row_bytes(row: Mapping[str, Any]) -> int:
|
|
33
|
+
"""Cheap upper-bound estimate of one row's serialized payload size, in bytes.
|
|
34
|
+
|
|
35
|
+
Not exact (actual bind-parameter wire encoding varies by driver/type), but good enough to
|
|
36
|
+
bound a batch's total payload size to roughly ``max_payload_bytes`` rather than not bounding
|
|
37
|
+
it at all — the field existed on :class:`BulkLimits`/``BulkConfig`` but ``resolve_batch_rows``
|
|
38
|
+
never read it, so a batch of many small-column, wide-value (e.g. large ``text``/``bytea``)
|
|
39
|
+
rows had no actual byte-size ceiling despite the config implying one (performance review §9).
|
|
40
|
+
"""
|
|
41
|
+
total = 0
|
|
42
|
+
for value in row.values():
|
|
43
|
+
if value is None:
|
|
44
|
+
continue
|
|
45
|
+
if isinstance(value, (bytes, bytearray)):
|
|
46
|
+
total += len(value)
|
|
47
|
+
else:
|
|
48
|
+
total += len(str(value).encode("utf-8", errors="ignore"))
|
|
49
|
+
return total
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def resolve_batch_rows(
|
|
53
|
+
n_columns: int,
|
|
54
|
+
requested: int | None,
|
|
55
|
+
limits: BulkLimits,
|
|
56
|
+
*,
|
|
57
|
+
sample_row: Mapping[str, Any] | None = None,
|
|
58
|
+
) -> int:
|
|
59
|
+
"""Largest safe batch size in rows, given per-row column count and the limits.
|
|
60
|
+
|
|
61
|
+
When ``limits.max_payload_bytes`` is set and a ``sample_row`` is supplied, the ceiling is
|
|
62
|
+
also bounded by an estimated per-row byte size (:func:`estimate_row_bytes`) — a single
|
|
63
|
+
representative row is enough since batches are homogeneous (same columns, similar value
|
|
64
|
+
sizes), and re-estimating every row would cost more than the batching itself saves.
|
|
65
|
+
"""
|
|
66
|
+
if n_columns <= 0:
|
|
67
|
+
return max(requested or limits.max_rows, 1)
|
|
68
|
+
by_params = max(limits.max_params // n_columns, 1)
|
|
69
|
+
ceiling = min(requested or limits.max_rows, limits.max_rows, by_params)
|
|
70
|
+
if limits.max_payload_bytes is not None and sample_row is not None:
|
|
71
|
+
row_bytes = estimate_row_bytes(sample_row)
|
|
72
|
+
if row_bytes > 0:
|
|
73
|
+
by_bytes = max(limits.max_payload_bytes // row_bytes, 1)
|
|
74
|
+
ceiling = min(ceiling, by_bytes)
|
|
75
|
+
return max(ceiling, 1)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def iter_batches(rows: Sequence[Any], batch_size: int) -> Iterator[Sequence[Any]]:
|
|
79
|
+
"""Yield contiguous ``batch_size`` slices of ``rows``."""
|
|
80
|
+
if batch_size <= 0:
|
|
81
|
+
batch_size = 1
|
|
82
|
+
for start in range(0, len(rows), batch_size):
|
|
83
|
+
yield rows[start : start + batch_size]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def column_names(rows: Sequence[Mapping[str, Any]]) -> list[str]:
|
|
87
|
+
"""Ordered column names inferred from the first row; every row must share them."""
|
|
88
|
+
if not rows:
|
|
89
|
+
return []
|
|
90
|
+
first = rows[0]
|
|
91
|
+
return list(first.keys())
|