dbkit-sql 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbkit/__init__.py +77 -0
- dbkit/_async/__init__.py +17 -0
- dbkit/_async/_compat.py +111 -0
- dbkit/_async/connection.py +328 -0
- dbkit/_async/database.py +884 -0
- dbkit/_async/engine.py +232 -0
- dbkit/_async/executor.py +329 -0
- dbkit/_async/health.py +53 -0
- dbkit/_async/resilience.py +148 -0
- dbkit/_async/streaming.py +137 -0
- dbkit/_async/transaction.py +264 -0
- dbkit/_core/__init__.py +1 -0
- dbkit/_core/bulk.py +91 -0
- dbkit/_core/circuit.py +130 -0
- dbkit/_core/config.py +613 -0
- dbkit/_core/errors/__init__.py +74 -0
- dbkit/_core/errors/base.py +344 -0
- dbkit/_core/errors/classifier.py +138 -0
- dbkit/_core/errors/redaction.py +96 -0
- dbkit/_core/errors/sqlstate.py +74 -0
- dbkit/_core/idempotency_lint.py +57 -0
- dbkit/_core/keys.py +29 -0
- dbkit/_core/policies.py +105 -0
- dbkit/_core/query.py +142 -0
- dbkit/_core/result.py +109 -0
- dbkit/_core/routing.py +248 -0
- dbkit/_pool.py +202 -0
- dbkit/_sync/__init__.py +20 -0
- dbkit/_sync/_compat.py +82 -0
- dbkit/_sync/connection.py +331 -0
- dbkit/_sync/database.py +887 -0
- dbkit/_sync/engine.py +235 -0
- dbkit/_sync/executor.py +332 -0
- dbkit/_sync/health.py +56 -0
- dbkit/_sync/resilience.py +151 -0
- dbkit/_sync/streaming.py +140 -0
- dbkit/_sync/transaction.py +267 -0
- dbkit/cli/__init__.py +1 -0
- dbkit/cli/main.py +256 -0
- dbkit/errors.py +6 -0
- dbkit/integrations/__init__.py +21 -0
- dbkit/integrations/batch.py +81 -0
- dbkit/integrations/inbox.py +153 -0
- dbkit/observability/__init__.py +21 -0
- dbkit/observability/logging.py +154 -0
- dbkit/observability/metrics.py +110 -0
- dbkit/observability/otel_metrics.py +69 -0
- dbkit/observability/prometheus.py +78 -0
- dbkit/observability/tracing.py +128 -0
- dbkit/postgres/__init__.py +1 -0
- dbkit/postgres/copy.py +77 -0
- dbkit/postgres/pipeline.py +48 -0
- dbkit/postgres/unnest.py +73 -0
- dbkit/testing/__init__.py +1 -0
- dbkit_sql-0.1.0.dist-info/METADATA +555 -0
- dbkit_sql-0.1.0.dist-info/RECORD +59 -0
- dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
- dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
- dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
dbkit/cli/main.py
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
"""dbkit CLI — configuration validation, health, and pool diagnostics (§31).
|
|
2
|
+
|
|
3
|
+
Requires the ``cli`` extra: ``pip install dbkit-sql[cli]``. All output redacts secrets; commands
|
|
4
|
+
that touch the network report classified errors cleanly instead of raw tracebacks.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import asyncio
|
|
10
|
+
from collections.abc import Awaitable
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, TypeVar
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from .._async.database import AsyncDatabase
|
|
17
|
+
from .._core.config import DbkitConfig
|
|
18
|
+
from .._core.errors import DatabaseConfigurationError, DatabaseError
|
|
19
|
+
from .._core.idempotency_lint import looks_unsafe_to_retry
|
|
20
|
+
from .._core.query import default_registry
|
|
21
|
+
|
|
22
|
+
app = typer.Typer(add_completion=False, help="dbkit — configuration, health, and pool diagnostics.")
|
|
23
|
+
|
|
24
|
+
T = TypeVar("T")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _load_config(config_path: Path) -> DbkitConfig:
|
|
28
|
+
if not config_path.exists():
|
|
29
|
+
typer.echo(f"config file not found: {config_path}", err=True)
|
|
30
|
+
raise typer.Exit(code=1)
|
|
31
|
+
try:
|
|
32
|
+
return DbkitConfig.from_yaml(str(config_path))
|
|
33
|
+
except DatabaseConfigurationError as exc:
|
|
34
|
+
typer.echo(f"configuration error: {exc}", err=True)
|
|
35
|
+
raise typer.Exit(code=1) from exc
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _print_operational_warnings(config: DbkitConfig) -> None:
|
|
39
|
+
for warning in (*config.budget_enforcement_warnings(), *config.tls_warnings()):
|
|
40
|
+
typer.echo(f" [WARNING] {warning}")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _run(coro: Awaitable[T]) -> T:
|
|
44
|
+
try:
|
|
45
|
+
return asyncio.run(coro) # type: ignore[arg-type]
|
|
46
|
+
except DatabaseError as exc:
|
|
47
|
+
typer.echo(f"error: {exc}", err=True)
|
|
48
|
+
raise typer.Exit(code=1) from exc
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@app.command()
|
|
52
|
+
def check(
|
|
53
|
+
config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
|
|
54
|
+
) -> None:
|
|
55
|
+
"""Validate configuration, then run a readiness check against every required database."""
|
|
56
|
+
config = _load_config(config_path)
|
|
57
|
+
typer.echo(
|
|
58
|
+
f"configuration OK: {len(config.databases)} database(s), environment={config.environment!r}"
|
|
59
|
+
)
|
|
60
|
+
_print_operational_warnings(config)
|
|
61
|
+
|
|
62
|
+
async def _go() -> bool:
|
|
63
|
+
db = AsyncDatabase.from_config(config)
|
|
64
|
+
await db.start()
|
|
65
|
+
try:
|
|
66
|
+
report = await db.health()
|
|
67
|
+
for t in report.targets:
|
|
68
|
+
status = "OK" if t.healthy else f"FAILED: {t.error}"
|
|
69
|
+
typer.echo(f" {t.key}: {status}")
|
|
70
|
+
return report.ready
|
|
71
|
+
finally:
|
|
72
|
+
await db.close()
|
|
73
|
+
|
|
74
|
+
if not _run(_go()):
|
|
75
|
+
raise typer.Exit(code=1)
|
|
76
|
+
typer.echo("all required databases are ready")
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@app.command()
|
|
80
|
+
def health(
|
|
81
|
+
config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
|
|
82
|
+
database: str | None = typer.Option(None, help="Limit output to one named database."),
|
|
83
|
+
) -> None:
|
|
84
|
+
"""Run a readiness health check and print per-target status."""
|
|
85
|
+
config = _load_config(config_path)
|
|
86
|
+
|
|
87
|
+
async def _go() -> bool:
|
|
88
|
+
db = AsyncDatabase.from_config(config)
|
|
89
|
+
await db.start()
|
|
90
|
+
try:
|
|
91
|
+
report = await db.health()
|
|
92
|
+
for t in report.targets:
|
|
93
|
+
if database and not t.key.startswith(f"{database}."):
|
|
94
|
+
continue
|
|
95
|
+
status = "OK" if t.healthy else f"FAILED: {t.error}"
|
|
96
|
+
typer.echo(f"{t.key}: {status}")
|
|
97
|
+
return report.ready
|
|
98
|
+
finally:
|
|
99
|
+
await db.close()
|
|
100
|
+
|
|
101
|
+
if not _run(_go()):
|
|
102
|
+
raise typer.Exit(code=1)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@app.command()
|
|
106
|
+
def pools(
|
|
107
|
+
config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
|
|
108
|
+
) -> None:
|
|
109
|
+
"""Print current pool status for every configured database (after warming a connection)."""
|
|
110
|
+
config = _load_config(config_path)
|
|
111
|
+
|
|
112
|
+
async def _go() -> list[dict[str, Any]]:
|
|
113
|
+
db = AsyncDatabase.from_config(config)
|
|
114
|
+
await db.start(warm=True)
|
|
115
|
+
try:
|
|
116
|
+
return [s.to_dict() for s in db.pool_status()]
|
|
117
|
+
finally:
|
|
118
|
+
await db.close()
|
|
119
|
+
|
|
120
|
+
snaps = _run(_go())
|
|
121
|
+
if not snaps:
|
|
122
|
+
typer.echo("no engines created")
|
|
123
|
+
return
|
|
124
|
+
for s in snaps:
|
|
125
|
+
typer.echo(
|
|
126
|
+
f"{s['key']}: size={s['size']} checked_out={s['checked_out']} "
|
|
127
|
+
f"overflow={s['overflow']} utilization={s['utilization']:.0%} "
|
|
128
|
+
f"created={s['created']} closed={s['closed']} invalidations={s['invalidations']}"
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
@app.command()
|
|
133
|
+
def engines(
|
|
134
|
+
config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
|
|
135
|
+
) -> None:
|
|
136
|
+
"""List the configured database targets without connecting to any of them."""
|
|
137
|
+
config = _load_config(config_path)
|
|
138
|
+
for name, db in config.databases.items():
|
|
139
|
+
typer.echo(f"{name}.primary driver={db.primary.driver} required={db.primary.required}")
|
|
140
|
+
for r in db.replicas:
|
|
141
|
+
typer.echo(f"{name}.replica:{r.name} driver={r.driver} weight={r.weight}")
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@app.command("config-validate")
|
|
145
|
+
def config_validate(
|
|
146
|
+
config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
|
|
147
|
+
) -> None:
|
|
148
|
+
"""Validate a config file and print a secret-redacted summary (no network access)."""
|
|
149
|
+
config = _load_config(config_path)
|
|
150
|
+
typer.echo(f"environment: {config.environment}")
|
|
151
|
+
typer.echo(f"databases: {list(config.databases)}")
|
|
152
|
+
redacted = config.redacted()
|
|
153
|
+
for name, db in redacted.databases.items():
|
|
154
|
+
typer.echo(f" {name}.primary: {db.primary.url}")
|
|
155
|
+
for r in db.replicas:
|
|
156
|
+
typer.echo(f" {name}.replica:{r.name}: {r.url}")
|
|
157
|
+
typer.echo("configuration is valid")
|
|
158
|
+
_print_operational_warnings(config)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
@app.command("connection-budget")
|
|
162
|
+
def connection_budget(
|
|
163
|
+
config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
|
|
164
|
+
replicas: int = typer.Option(1, help="Number of application replicas/pods."),
|
|
165
|
+
) -> None:
|
|
166
|
+
"""Print the projected cluster-wide connection budget (no network access)."""
|
|
167
|
+
config = _load_config(config_path)
|
|
168
|
+
report = config.connection_budget_report(replicas=replicas)
|
|
169
|
+
typer.echo(f"per-process: {report['per_process']}")
|
|
170
|
+
typer.echo(f"app replicas: {report['app_replicas']}")
|
|
171
|
+
typer.echo(f"cluster total: {report['cluster_total']}")
|
|
172
|
+
limit = config.connection_budget.maximum_per_process
|
|
173
|
+
if limit is not None:
|
|
174
|
+
status = "OK" if report["per_process"] <= limit else "EXCEEDS budget"
|
|
175
|
+
typer.echo(f"configured per-process budget: {limit} ({status})")
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@app.command()
|
|
179
|
+
def metrics(
|
|
180
|
+
config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
|
|
181
|
+
) -> None:
|
|
182
|
+
"""Run one health probe and print the resulting metric values, Prometheus text format.
|
|
183
|
+
|
|
184
|
+
Requires the ``prometheus`` extra. This is a wiring smoke test, not a live incident-triage
|
|
185
|
+
snapshot: this command is a fresh process with its own metrics registry, so it can only show
|
|
186
|
+
what *this command's own probe* recorded (pool gauges, one operation/duration observation per
|
|
187
|
+
target) — it has no way to read an already-running application's accumulated counters. Point
|
|
188
|
+
your real Prometheus scraper at the running application's own metrics endpoint for that; use
|
|
189
|
+
this command to sanity-check metric names/labels before wiring up dashboards/alerts (see
|
|
190
|
+
``docs/observability.md``).
|
|
191
|
+
"""
|
|
192
|
+
try:
|
|
193
|
+
from prometheus_client import CollectorRegistry, generate_latest
|
|
194
|
+
except ImportError as exc:
|
|
195
|
+
typer.echo(
|
|
196
|
+
"the `prometheus` extra is required: pip install dbkit-sql[prometheus]", err=True
|
|
197
|
+
)
|
|
198
|
+
raise typer.Exit(code=1) from exc
|
|
199
|
+
|
|
200
|
+
from .._core.query import Query, sql
|
|
201
|
+
from .._core.routing import DatabaseTarget
|
|
202
|
+
from ..observability.prometheus import PrometheusMetrics
|
|
203
|
+
|
|
204
|
+
config = _load_config(config_path)
|
|
205
|
+
registry = CollectorRegistry()
|
|
206
|
+
sink = PrometheusMetrics(namespace="dbkit", registry=registry)
|
|
207
|
+
probe = Query(name="dbkit.cli.metrics_probe", statement=sql("SELECT 1"))
|
|
208
|
+
|
|
209
|
+
async def _go() -> None:
|
|
210
|
+
db = AsyncDatabase.from_config(config, metrics=sink)
|
|
211
|
+
await db.start(warm=True)
|
|
212
|
+
try:
|
|
213
|
+
for name, target_config in config.databases.items():
|
|
214
|
+
if target_config.primary.required:
|
|
215
|
+
await db.fetch_value(probe, target=DatabaseTarget(database=name, role="write"))
|
|
216
|
+
db.pool_status() # samples pool gauges — otherwise-lazy, see _pool.py
|
|
217
|
+
finally:
|
|
218
|
+
await db.close()
|
|
219
|
+
|
|
220
|
+
_run(_go())
|
|
221
|
+
typer.echo(generate_latest(registry).decode())
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@app.command("query-list")
|
|
225
|
+
def query_list() -> None:
|
|
226
|
+
"""List queries registered on the process-global default registry.
|
|
227
|
+
|
|
228
|
+
Only queries constructed in this process appear — import your application's query
|
|
229
|
+
modules first if you need them listed. Writes marked ``idempotent=True`` with no visible
|
|
230
|
+
``ON CONFLICT``-style guard in their SQL text get a best-effort warning — dbkit's retry
|
|
231
|
+
executor trusts ``idempotent=True`` as-is; it does not verify the SQL is actually safe to
|
|
232
|
+
run twice (§14).
|
|
233
|
+
"""
|
|
234
|
+
names = default_registry.names()
|
|
235
|
+
if not names:
|
|
236
|
+
typer.echo("no queries registered in the default registry")
|
|
237
|
+
return
|
|
238
|
+
for name in names:
|
|
239
|
+
q = default_registry.get(name)
|
|
240
|
+
assert q is not None
|
|
241
|
+
line = f"{name} operation={q.operation} idempotent={q.idempotent}"
|
|
242
|
+
if looks_unsafe_to_retry(q):
|
|
243
|
+
line += (
|
|
244
|
+
" [WARNING: idempotent=True but no ON CONFLICT/guard detected — "
|
|
245
|
+
"verify this write is actually safe to run twice]"
|
|
246
|
+
)
|
|
247
|
+
typer.echo(line)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def main() -> None:
|
|
251
|
+
"""Entry point for the ``dbkit`` console script."""
|
|
252
|
+
app()
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
if __name__ == "__main__":
|
|
256
|
+
main()
|
dbkit/errors.py
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Integration helpers for message-driven workloads (§28) and micro-batching (§17.1)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .batch import BatchCollector
|
|
6
|
+
from .inbox import (
|
|
7
|
+
ack_after_commit,
|
|
8
|
+
inbox_ddl,
|
|
9
|
+
inbox_month_partition_ddl,
|
|
10
|
+
partitioned_inbox_ddl,
|
|
11
|
+
process_once,
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"BatchCollector",
|
|
16
|
+
"ack_after_commit",
|
|
17
|
+
"inbox_ddl",
|
|
18
|
+
"inbox_month_partition_ddl",
|
|
19
|
+
"partitioned_inbox_ddl",
|
|
20
|
+
"process_once",
|
|
21
|
+
]
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Consumer micro-batching (§17.1 high-perf addition).
|
|
2
|
+
|
|
3
|
+
``BatchCollector`` aggregates items from many concurrent message handlers and flushes them as
|
|
4
|
+
one batched database write, either when the buffer reaches ``max_size`` or after
|
|
5
|
+
``max_delay_ms`` — whichever comes first. This is the primary throughput lever for
|
|
6
|
+
message-driven ingestion: N handlers each contribute a row, one COPY/executemany persists them.
|
|
7
|
+
|
|
8
|
+
Async-only (message consumers are async). The flush callback receives the buffered items and
|
|
9
|
+
should perform the batched write (e.g. ``db.copy_records`` / ``db.insert_many``).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import asyncio
|
|
15
|
+
from collections.abc import Awaitable, Callable, Sequence
|
|
16
|
+
from typing import Generic, TypeVar
|
|
17
|
+
|
|
18
|
+
T = TypeVar("T")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BatchCollector(Generic[T]):
|
|
22
|
+
"""Aggregates items across concurrent producers and flushes them as one batch (§17.1)."""
|
|
23
|
+
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
flush: Callable[[Sequence[T]], Awaitable[None]],
|
|
27
|
+
*,
|
|
28
|
+
max_size: int = 1000,
|
|
29
|
+
max_delay_ms: float = 50.0,
|
|
30
|
+
) -> None:
|
|
31
|
+
"""``flush`` is called with the buffered batch whenever ``max_size``/``max_delay_ms``
|
|
32
|
+
is reached; it should perform the batched write (e.g. ``db.insert_many``)."""
|
|
33
|
+
if max_size < 1:
|
|
34
|
+
raise ValueError("max_size must be >= 1")
|
|
35
|
+
self._flush_cb = flush
|
|
36
|
+
self._max_size = max_size
|
|
37
|
+
self._max_delay = max_delay_ms / 1000.0
|
|
38
|
+
self._buffer: list[T] = []
|
|
39
|
+
self._lock = asyncio.Lock()
|
|
40
|
+
self._timer: asyncio.Task[None] | None = None
|
|
41
|
+
self._closed = False
|
|
42
|
+
|
|
43
|
+
async def add(self, item: T) -> None:
|
|
44
|
+
"""Add an item; flush immediately if the buffer is full, else arm the flush timer."""
|
|
45
|
+
async with self._lock:
|
|
46
|
+
if self._closed:
|
|
47
|
+
raise RuntimeError("BatchCollector is closed")
|
|
48
|
+
self._buffer.append(item)
|
|
49
|
+
if len(self._buffer) >= self._max_size:
|
|
50
|
+
await self._flush_locked()
|
|
51
|
+
elif self._timer is None:
|
|
52
|
+
self._timer = asyncio.create_task(self._timer_flush())
|
|
53
|
+
|
|
54
|
+
async def flush(self) -> None:
|
|
55
|
+
"""Flush any buffered items now."""
|
|
56
|
+
async with self._lock:
|
|
57
|
+
await self._flush_locked()
|
|
58
|
+
|
|
59
|
+
async def close(self) -> None:
|
|
60
|
+
"""Flush remaining items and stop the timer."""
|
|
61
|
+
async with self._lock:
|
|
62
|
+
self._closed = True
|
|
63
|
+
await self._flush_locked()
|
|
64
|
+
|
|
65
|
+
async def _timer_flush(self) -> None:
|
|
66
|
+
try:
|
|
67
|
+
await asyncio.sleep(self._max_delay)
|
|
68
|
+
except asyncio.CancelledError:
|
|
69
|
+
return
|
|
70
|
+
async with self._lock:
|
|
71
|
+
await self._flush_locked()
|
|
72
|
+
|
|
73
|
+
async def _flush_locked(self) -> None:
|
|
74
|
+
if self._timer is not None:
|
|
75
|
+
self._timer.cancel()
|
|
76
|
+
self._timer = None
|
|
77
|
+
if not self._buffer:
|
|
78
|
+
return
|
|
79
|
+
items = self._buffer
|
|
80
|
+
self._buffer = []
|
|
81
|
+
await self._flush_cb(items)
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Inbox / idempotent-consume helpers for message-driven writes (§28).
|
|
2
|
+
|
|
3
|
+
These make RabbitMQ (or any broker) message processing safe against redelivery. The inbox row
|
|
4
|
+
and the business writes commit in the **same** transaction, so a message is processed
|
|
5
|
+
exactly-once even though delivery is at-least-once. Ack the broker message only after the
|
|
6
|
+
commit succeeds; on a commit-unknown outcome, do NOT ack — require idempotent replay (§15, §28).
|
|
7
|
+
|
|
8
|
+
Async helpers targeting :class:`AsyncDatabase`. The DDL is frontend-agnostic.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import contextlib
|
|
14
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from .._core.query import sql
|
|
18
|
+
from .._core.routing import DatabaseTarget
|
|
19
|
+
from ..errors import DatabaseCommitUnknownError, DatabaseError
|
|
20
|
+
|
|
21
|
+
DEFAULT_INBOX_TABLE = "consumed_messages"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def inbox_ddl(table: str = DEFAULT_INBOX_TABLE) -> str:
|
|
25
|
+
"""DDL for the inbox table (§28.3). Time-partition it in production for cheap pruning —
|
|
26
|
+
see :func:`partitioned_inbox_ddl`/:func:`inbox_month_partition_ddl` for that variant."""
|
|
27
|
+
return f"""
|
|
28
|
+
CREATE TABLE IF NOT EXISTS {table} (
|
|
29
|
+
consumer_name TEXT NOT NULL,
|
|
30
|
+
message_id TEXT NOT NULL,
|
|
31
|
+
processed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
32
|
+
PRIMARY KEY (consumer_name, message_id)
|
|
33
|
+
)
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def partitioned_inbox_ddl(table: str = DEFAULT_INBOX_TABLE) -> str:
|
|
38
|
+
"""DDL for a **partitioned** inbox table, range-partitioned by ``processed_at`` (§28.3).
|
|
39
|
+
|
|
40
|
+
A single unbounded ``consumed_messages`` table's dedup lookup gets slower and its storage
|
|
41
|
+
grows without bound as message volume accumulates. Range-partitioning by month makes
|
|
42
|
+
pruning old data a cheap ``DROP TABLE`` on a partition instead of a slow ``DELETE``. The
|
|
43
|
+
partition key must be part of the primary key for declarative partitioning, so this variant
|
|
44
|
+
keys on ``(consumer_name, message_id, processed_at)`` instead of just the first two.
|
|
45
|
+
|
|
46
|
+
Partitions themselves aren't created automatically — call
|
|
47
|
+
:func:`inbox_month_partition_ddl` for each month you need (typically from a scheduled job
|
|
48
|
+
that creates the next 1-2 months ahead of time; `pg_partman` or a cron-triggered `dbkit`
|
|
49
|
+
script both work). Older partitions can simply be dropped once no longer needed for
|
|
50
|
+
dedup/audit purposes — a message's redelivery window is bounded by the broker's own
|
|
51
|
+
retention, so a partition older than that window is always safe to drop.
|
|
52
|
+
"""
|
|
53
|
+
return f"""
|
|
54
|
+
CREATE TABLE IF NOT EXISTS {table} (
|
|
55
|
+
consumer_name TEXT NOT NULL,
|
|
56
|
+
message_id TEXT NOT NULL,
|
|
57
|
+
processed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
58
|
+
PRIMARY KEY (consumer_name, message_id, processed_at)
|
|
59
|
+
) PARTITION BY RANGE (processed_at)
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def inbox_month_partition_ddl(year: int, month: int, *, table: str = DEFAULT_INBOX_TABLE) -> str:
|
|
64
|
+
"""DDL for one calendar-month partition of :func:`partitioned_inbox_ddl`'s table.
|
|
65
|
+
|
|
66
|
+
``year``/``month`` are passed explicitly (not computed from the current date) so this stays
|
|
67
|
+
a pure function — call it with whatever month you actually need created, e.g. from a
|
|
68
|
+
scheduled job that creates the next month a few days before it starts.
|
|
69
|
+
"""
|
|
70
|
+
start = f"{year:04d}-{month:02d}-01"
|
|
71
|
+
next_year, next_month = (year + 1, 1) if month == 12 else (year, month + 1)
|
|
72
|
+
end = f"{next_year:04d}-{next_month:02d}-01"
|
|
73
|
+
partition_name = f"{table}_{year:04d}_{month:02d}"
|
|
74
|
+
return f"""
|
|
75
|
+
CREATE TABLE IF NOT EXISTS {partition_name} PARTITION OF {table}
|
|
76
|
+
FOR VALUES FROM ('{start}') TO ('{end}')
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
async def _claim(tx: Any, consumer: str, message_id: str, table: str) -> bool:
|
|
81
|
+
"""Insert the inbox row; return True if this is the first time we've seen the message."""
|
|
82
|
+
row = await tx.fetch_optional(
|
|
83
|
+
sql(
|
|
84
|
+
f"INSERT INTO {table} (consumer_name, message_id) VALUES (:c, :m) "
|
|
85
|
+
"ON CONFLICT (consumer_name, message_id) DO NOTHING RETURNING 1"
|
|
86
|
+
),
|
|
87
|
+
{"c": consumer, "m": message_id},
|
|
88
|
+
)
|
|
89
|
+
return row is not None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@contextlib.asynccontextmanager
|
|
93
|
+
async def process_once(
|
|
94
|
+
db: Any,
|
|
95
|
+
*,
|
|
96
|
+
consumer: str,
|
|
97
|
+
message_id: str,
|
|
98
|
+
target: DatabaseTarget,
|
|
99
|
+
inbox_table: str = DEFAULT_INBOX_TABLE,
|
|
100
|
+
) -> AsyncIterator[tuple[Any, bool]]:
|
|
101
|
+
"""Open a transaction, claim the message in the inbox, and yield ``(tx, first_time)``.
|
|
102
|
+
|
|
103
|
+
Do the business writes on ``tx`` only when ``first_time`` is True; a duplicate delivery
|
|
104
|
+
still commits (a harmless no-op) so the broker message can be acked::
|
|
105
|
+
|
|
106
|
+
async with process_once(db, consumer="c", message_id=mid, target=t) as (tx, first):
|
|
107
|
+
if first:
|
|
108
|
+
await tx.execute(INSERT_ORDER, params)
|
|
109
|
+
"""
|
|
110
|
+
async with db.transaction(target=target) as tx:
|
|
111
|
+
first = await _claim(tx, consumer, message_id, inbox_table)
|
|
112
|
+
yield tx, first
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
async def ack_after_commit(
|
|
116
|
+
db: Any,
|
|
117
|
+
*,
|
|
118
|
+
consumer: str,
|
|
119
|
+
message_id: str,
|
|
120
|
+
target: DatabaseTarget,
|
|
121
|
+
work: Callable[[Any], Awaitable[None]],
|
|
122
|
+
ack: Callable[[], Awaitable[None]],
|
|
123
|
+
dead_letter: Callable[[DatabaseError], Awaitable[None]] | None = None,
|
|
124
|
+
retry: Callable[[DatabaseError], Awaitable[None]] | None = None,
|
|
125
|
+
inbox_table: str = DEFAULT_INBOX_TABLE,
|
|
126
|
+
) -> bool:
|
|
127
|
+
"""Run ``work`` idempotently, then ack — the full §28 consume flow.
|
|
128
|
+
|
|
129
|
+
Returns True if the message was processed (or was a duplicate) and acked. Behavior on
|
|
130
|
+
failure:
|
|
131
|
+
|
|
132
|
+
* commit-unknown -> re-raised; the message is **not** acked (idempotent replay will dedupe).
|
|
133
|
+
* retryable error -> ``retry`` callback if given, else re-raised (nack/requeue).
|
|
134
|
+
* permanent error -> ``dead_letter`` callback if given, else re-raised (route to DLQ).
|
|
135
|
+
"""
|
|
136
|
+
try:
|
|
137
|
+
async with process_once(
|
|
138
|
+
db, consumer=consumer, message_id=message_id, target=target, inbox_table=inbox_table
|
|
139
|
+
) as (tx, first):
|
|
140
|
+
if first:
|
|
141
|
+
await work(tx)
|
|
142
|
+
except DatabaseCommitUnknownError:
|
|
143
|
+
raise # never ack on unknown commit outcome
|
|
144
|
+
except DatabaseError as exc:
|
|
145
|
+
if exc.retryable and retry is not None:
|
|
146
|
+
await retry(exc)
|
|
147
|
+
return False
|
|
148
|
+
if not exc.retryable and dead_letter is not None:
|
|
149
|
+
await dead_letter(exc)
|
|
150
|
+
return False
|
|
151
|
+
raise
|
|
152
|
+
await ack()
|
|
153
|
+
return True
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Observability: structured logging, metrics, and OpenTelemetry tracing (§25)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .logging import log_event, logger, long_transaction_warning, slow_query_warning
|
|
6
|
+
from .metrics import MetricsSink, NoopMetrics, try_otel_metrics_sink, try_prometheus_sink
|
|
7
|
+
from .tracing import SpanHandle, Tracer, make_tracer
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"MetricsSink",
|
|
11
|
+
"NoopMetrics",
|
|
12
|
+
"SpanHandle",
|
|
13
|
+
"Tracer",
|
|
14
|
+
"log_event",
|
|
15
|
+
"logger",
|
|
16
|
+
"long_transaction_warning",
|
|
17
|
+
"make_tracer",
|
|
18
|
+
"slow_query_warning",
|
|
19
|
+
"try_otel_metrics_sink",
|
|
20
|
+
"try_prometheus_sink",
|
|
21
|
+
]
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Structured logging helpers (§25.3).
|
|
2
|
+
|
|
3
|
+
dbkit logs through the stdlib ``logging`` module under the ``dbkit`` logger, passing a
|
|
4
|
+
structured payload in ``extra={"dbkit": {...}}`` so a JSON formatter can render it. Bound
|
|
5
|
+
parameters are never logged unless explicitly enabled, and sensitive params are always
|
|
6
|
+
redacted.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
from collections.abc import Mapping
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger("dbkit")
|
|
16
|
+
|
|
17
|
+
try:
|
|
18
|
+
from opentelemetry import trace as _otel_trace
|
|
19
|
+
|
|
20
|
+
_OTEL_AVAILABLE = True
|
|
21
|
+
except ImportError: # pragma: no cover - exercised by the no-otel test lane
|
|
22
|
+
_OTEL_AVAILABLE = False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _trace_context() -> dict[str, str]:
|
|
26
|
+
"""``trace_id``/``span_id`` of the current OTel span, for trace/log correlation (§25.2/§25.3).
|
|
27
|
+
|
|
28
|
+
Empty if OTel isn't installed or no span is currently recording. The availability check is
|
|
29
|
+
done once at import time (not per call) so this stays cheap on the hot logging path even
|
|
30
|
+
when OTel isn't installed — a per-call ``import opentelemetry`` would otherwise re-walk
|
|
31
|
+
``sys.path`` on every log event, since failed imports aren't cached in ``sys.modules``.
|
|
32
|
+
"""
|
|
33
|
+
if not _OTEL_AVAILABLE:
|
|
34
|
+
return {}
|
|
35
|
+
ctx = _otel_trace.get_current_span().get_span_context()
|
|
36
|
+
if not ctx.is_valid:
|
|
37
|
+
return {}
|
|
38
|
+
return {"trace_id": format(ctx.trace_id, "032x"), "span_id": format(ctx.span_id, "016x")}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def log_event(
|
|
42
|
+
level: int,
|
|
43
|
+
event: str,
|
|
44
|
+
*,
|
|
45
|
+
query_name: str | None = None,
|
|
46
|
+
database: str | None = None,
|
|
47
|
+
shard: str | None = None,
|
|
48
|
+
role: str | None = None,
|
|
49
|
+
duration_ms: float | None = None,
|
|
50
|
+
pool_wait_ms: float | None = None,
|
|
51
|
+
rows: int | None = None,
|
|
52
|
+
retry_attempt: int | None = None,
|
|
53
|
+
error_code: str | None = None,
|
|
54
|
+
**extra: Any,
|
|
55
|
+
) -> None:
|
|
56
|
+
"""Emit one structured lifecycle event (§25.3)."""
|
|
57
|
+
if not logger.isEnabledFor(level):
|
|
58
|
+
return
|
|
59
|
+
payload: dict[str, Any] = {"event": event}
|
|
60
|
+
for key, value in (
|
|
61
|
+
("query_name", query_name),
|
|
62
|
+
("database", database),
|
|
63
|
+
("shard", shard),
|
|
64
|
+
("role", role),
|
|
65
|
+
("duration_ms", duration_ms),
|
|
66
|
+
("pool_wait_ms", pool_wait_ms),
|
|
67
|
+
("rows", rows),
|
|
68
|
+
("retry_attempt", retry_attempt),
|
|
69
|
+
("error_code", error_code),
|
|
70
|
+
):
|
|
71
|
+
if value is not None:
|
|
72
|
+
payload[key] = value
|
|
73
|
+
payload.update(_trace_context())
|
|
74
|
+
payload.update(extra)
|
|
75
|
+
logger.log(level, event, extra={"dbkit": payload})
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def slow_query_warning(
|
|
79
|
+
*,
|
|
80
|
+
query_name: str,
|
|
81
|
+
duration_ms: float,
|
|
82
|
+
threshold_ms: float,
|
|
83
|
+
database: str | None = None,
|
|
84
|
+
pool_wait_ms: float | None = None,
|
|
85
|
+
rows: int | None = None,
|
|
86
|
+
) -> None:
|
|
87
|
+
"""Emit a ``database.query.slow`` warning (caller decides whether the threshold was hit)."""
|
|
88
|
+
log_event(
|
|
89
|
+
logging.WARNING,
|
|
90
|
+
"database.query.slow",
|
|
91
|
+
query_name=query_name,
|
|
92
|
+
database=database,
|
|
93
|
+
duration_ms=round(duration_ms, 3),
|
|
94
|
+
pool_wait_ms=None if pool_wait_ms is None else round(pool_wait_ms, 3),
|
|
95
|
+
rows=rows,
|
|
96
|
+
threshold_ms=threshold_ms,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def long_transaction_warning(
|
|
101
|
+
*,
|
|
102
|
+
duration_ms: float,
|
|
103
|
+
threshold_ms: float,
|
|
104
|
+
database: str | None = None,
|
|
105
|
+
role: str | None = None,
|
|
106
|
+
outcome: str | None = None,
|
|
107
|
+
) -> None:
|
|
108
|
+
"""Warn when an explicit transaction was held open longer than the configured threshold —
|
|
109
|
+
the transaction analog of the pool's long-connection-hold warning (§10.5, §16)."""
|
|
110
|
+
log_event(
|
|
111
|
+
logging.WARNING,
|
|
112
|
+
"database.transaction.long_running",
|
|
113
|
+
database=database,
|
|
114
|
+
role=role,
|
|
115
|
+
duration_ms=round(duration_ms, 3),
|
|
116
|
+
threshold_ms=threshold_ms,
|
|
117
|
+
outcome=outcome,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def bulk_batch_dropped_warning(
|
|
122
|
+
*,
|
|
123
|
+
query_name: str,
|
|
124
|
+
database: str | None = None,
|
|
125
|
+
mode: str,
|
|
126
|
+
rows_dropped: int,
|
|
127
|
+
error_category: str | None = None,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""Warn that a bulk batch (``best_effort``) or row (``split_on_failure``, after its batch
|
|
130
|
+
already failed) was dropped rather than written — this is the only signal a caller gets for
|
|
131
|
+
silently-lost rows under either failure mode (§19.3, performance review §9)."""
|
|
132
|
+
log_event(
|
|
133
|
+
logging.WARNING,
|
|
134
|
+
"database.bulk.rows_dropped",
|
|
135
|
+
query_name=query_name,
|
|
136
|
+
database=database,
|
|
137
|
+
mode=mode,
|
|
138
|
+
rows_dropped=rows_dropped,
|
|
139
|
+
error_category=error_category,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def redacted_params_for_log(
|
|
144
|
+
params: Mapping[str, Any] | None,
|
|
145
|
+
*,
|
|
146
|
+
log_parameters: bool,
|
|
147
|
+
sensitive: set[str] | None,
|
|
148
|
+
) -> dict[str, Any] | None:
|
|
149
|
+
"""Return params suitable for logging, or ``None`` when parameter logging is disabled."""
|
|
150
|
+
if not log_parameters or not params:
|
|
151
|
+
return None
|
|
152
|
+
from .._core.errors.redaction import redact_params
|
|
153
|
+
|
|
154
|
+
return redact_params(params, sensitive=sensitive)
|