dbkit-sql 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. dbkit/__init__.py +77 -0
  2. dbkit/_async/__init__.py +17 -0
  3. dbkit/_async/_compat.py +111 -0
  4. dbkit/_async/connection.py +328 -0
  5. dbkit/_async/database.py +884 -0
  6. dbkit/_async/engine.py +232 -0
  7. dbkit/_async/executor.py +329 -0
  8. dbkit/_async/health.py +53 -0
  9. dbkit/_async/resilience.py +148 -0
  10. dbkit/_async/streaming.py +137 -0
  11. dbkit/_async/transaction.py +264 -0
  12. dbkit/_core/__init__.py +1 -0
  13. dbkit/_core/bulk.py +91 -0
  14. dbkit/_core/circuit.py +130 -0
  15. dbkit/_core/config.py +613 -0
  16. dbkit/_core/errors/__init__.py +74 -0
  17. dbkit/_core/errors/base.py +344 -0
  18. dbkit/_core/errors/classifier.py +138 -0
  19. dbkit/_core/errors/redaction.py +96 -0
  20. dbkit/_core/errors/sqlstate.py +74 -0
  21. dbkit/_core/idempotency_lint.py +57 -0
  22. dbkit/_core/keys.py +29 -0
  23. dbkit/_core/policies.py +105 -0
  24. dbkit/_core/query.py +142 -0
  25. dbkit/_core/result.py +109 -0
  26. dbkit/_core/routing.py +248 -0
  27. dbkit/_pool.py +202 -0
  28. dbkit/_sync/__init__.py +20 -0
  29. dbkit/_sync/_compat.py +82 -0
  30. dbkit/_sync/connection.py +331 -0
  31. dbkit/_sync/database.py +887 -0
  32. dbkit/_sync/engine.py +235 -0
  33. dbkit/_sync/executor.py +332 -0
  34. dbkit/_sync/health.py +56 -0
  35. dbkit/_sync/resilience.py +151 -0
  36. dbkit/_sync/streaming.py +140 -0
  37. dbkit/_sync/transaction.py +267 -0
  38. dbkit/cli/__init__.py +1 -0
  39. dbkit/cli/main.py +256 -0
  40. dbkit/errors.py +6 -0
  41. dbkit/integrations/__init__.py +21 -0
  42. dbkit/integrations/batch.py +81 -0
  43. dbkit/integrations/inbox.py +153 -0
  44. dbkit/observability/__init__.py +21 -0
  45. dbkit/observability/logging.py +154 -0
  46. dbkit/observability/metrics.py +110 -0
  47. dbkit/observability/otel_metrics.py +69 -0
  48. dbkit/observability/prometheus.py +78 -0
  49. dbkit/observability/tracing.py +128 -0
  50. dbkit/postgres/__init__.py +1 -0
  51. dbkit/postgres/copy.py +77 -0
  52. dbkit/postgres/pipeline.py +48 -0
  53. dbkit/postgres/unnest.py +73 -0
  54. dbkit/testing/__init__.py +1 -0
  55. dbkit_sql-0.1.0.dist-info/METADATA +555 -0
  56. dbkit_sql-0.1.0.dist-info/RECORD +59 -0
  57. dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
  58. dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
  59. dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
dbkit/cli/main.py ADDED
@@ -0,0 +1,256 @@
1
+ """dbkit CLI — configuration validation, health, and pool diagnostics (§31).
2
+
3
+ Requires the ``cli`` extra: ``pip install dbkit-sql[cli]``. All output redacts secrets; commands
4
+ that touch the network report classified errors cleanly instead of raw tracebacks.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ from collections.abc import Awaitable
11
+ from pathlib import Path
12
+ from typing import Any, TypeVar
13
+
14
+ import typer
15
+
16
+ from .._async.database import AsyncDatabase
17
+ from .._core.config import DbkitConfig
18
+ from .._core.errors import DatabaseConfigurationError, DatabaseError
19
+ from .._core.idempotency_lint import looks_unsafe_to_retry
20
+ from .._core.query import default_registry
21
+
22
+ app = typer.Typer(add_completion=False, help="dbkit — configuration, health, and pool diagnostics.")
23
+
24
+ T = TypeVar("T")
25
+
26
+
27
+ def _load_config(config_path: Path) -> DbkitConfig:
28
+ if not config_path.exists():
29
+ typer.echo(f"config file not found: {config_path}", err=True)
30
+ raise typer.Exit(code=1)
31
+ try:
32
+ return DbkitConfig.from_yaml(str(config_path))
33
+ except DatabaseConfigurationError as exc:
34
+ typer.echo(f"configuration error: {exc}", err=True)
35
+ raise typer.Exit(code=1) from exc
36
+
37
+
38
+ def _print_operational_warnings(config: DbkitConfig) -> None:
39
+ for warning in (*config.budget_enforcement_warnings(), *config.tls_warnings()):
40
+ typer.echo(f" [WARNING] {warning}")
41
+
42
+
43
+ def _run(coro: Awaitable[T]) -> T:
44
+ try:
45
+ return asyncio.run(coro) # type: ignore[arg-type]
46
+ except DatabaseError as exc:
47
+ typer.echo(f"error: {exc}", err=True)
48
+ raise typer.Exit(code=1) from exc
49
+
50
+
51
+ @app.command()
52
+ def check(
53
+ config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
54
+ ) -> None:
55
+ """Validate configuration, then run a readiness check against every required database."""
56
+ config = _load_config(config_path)
57
+ typer.echo(
58
+ f"configuration OK: {len(config.databases)} database(s), environment={config.environment!r}"
59
+ )
60
+ _print_operational_warnings(config)
61
+
62
+ async def _go() -> bool:
63
+ db = AsyncDatabase.from_config(config)
64
+ await db.start()
65
+ try:
66
+ report = await db.health()
67
+ for t in report.targets:
68
+ status = "OK" if t.healthy else f"FAILED: {t.error}"
69
+ typer.echo(f" {t.key}: {status}")
70
+ return report.ready
71
+ finally:
72
+ await db.close()
73
+
74
+ if not _run(_go()):
75
+ raise typer.Exit(code=1)
76
+ typer.echo("all required databases are ready")
77
+
78
+
79
+ @app.command()
80
+ def health(
81
+ config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
82
+ database: str | None = typer.Option(None, help="Limit output to one named database."),
83
+ ) -> None:
84
+ """Run a readiness health check and print per-target status."""
85
+ config = _load_config(config_path)
86
+
87
+ async def _go() -> bool:
88
+ db = AsyncDatabase.from_config(config)
89
+ await db.start()
90
+ try:
91
+ report = await db.health()
92
+ for t in report.targets:
93
+ if database and not t.key.startswith(f"{database}."):
94
+ continue
95
+ status = "OK" if t.healthy else f"FAILED: {t.error}"
96
+ typer.echo(f"{t.key}: {status}")
97
+ return report.ready
98
+ finally:
99
+ await db.close()
100
+
101
+ if not _run(_go()):
102
+ raise typer.Exit(code=1)
103
+
104
+
105
+ @app.command()
106
+ def pools(
107
+ config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
108
+ ) -> None:
109
+ """Print current pool status for every configured database (after warming a connection)."""
110
+ config = _load_config(config_path)
111
+
112
+ async def _go() -> list[dict[str, Any]]:
113
+ db = AsyncDatabase.from_config(config)
114
+ await db.start(warm=True)
115
+ try:
116
+ return [s.to_dict() for s in db.pool_status()]
117
+ finally:
118
+ await db.close()
119
+
120
+ snaps = _run(_go())
121
+ if not snaps:
122
+ typer.echo("no engines created")
123
+ return
124
+ for s in snaps:
125
+ typer.echo(
126
+ f"{s['key']}: size={s['size']} checked_out={s['checked_out']} "
127
+ f"overflow={s['overflow']} utilization={s['utilization']:.0%} "
128
+ f"created={s['created']} closed={s['closed']} invalidations={s['invalidations']}"
129
+ )
130
+
131
+
132
+ @app.command()
133
+ def engines(
134
+ config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
135
+ ) -> None:
136
+ """List the configured database targets without connecting to any of them."""
137
+ config = _load_config(config_path)
138
+ for name, db in config.databases.items():
139
+ typer.echo(f"{name}.primary driver={db.primary.driver} required={db.primary.required}")
140
+ for r in db.replicas:
141
+ typer.echo(f"{name}.replica:{r.name} driver={r.driver} weight={r.weight}")
142
+
143
+
144
+ @app.command("config-validate")
145
+ def config_validate(
146
+ config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
147
+ ) -> None:
148
+ """Validate a config file and print a secret-redacted summary (no network access)."""
149
+ config = _load_config(config_path)
150
+ typer.echo(f"environment: {config.environment}")
151
+ typer.echo(f"databases: {list(config.databases)}")
152
+ redacted = config.redacted()
153
+ for name, db in redacted.databases.items():
154
+ typer.echo(f" {name}.primary: {db.primary.url}")
155
+ for r in db.replicas:
156
+ typer.echo(f" {name}.replica:{r.name}: {r.url}")
157
+ typer.echo("configuration is valid")
158
+ _print_operational_warnings(config)
159
+
160
+
161
+ @app.command("connection-budget")
162
+ def connection_budget(
163
+ config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
164
+ replicas: int = typer.Option(1, help="Number of application replicas/pods."),
165
+ ) -> None:
166
+ """Print the projected cluster-wide connection budget (no network access)."""
167
+ config = _load_config(config_path)
168
+ report = config.connection_budget_report(replicas=replicas)
169
+ typer.echo(f"per-process: {report['per_process']}")
170
+ typer.echo(f"app replicas: {report['app_replicas']}")
171
+ typer.echo(f"cluster total: {report['cluster_total']}")
172
+ limit = config.connection_budget.maximum_per_process
173
+ if limit is not None:
174
+ status = "OK" if report["per_process"] <= limit else "EXCEEDS budget"
175
+ typer.echo(f"configured per-process budget: {limit} ({status})")
176
+
177
+
178
+ @app.command()
179
+ def metrics(
180
+ config_path: Path = typer.Argument(..., help="Path to a dbkit YAML config file."),
181
+ ) -> None:
182
+ """Run one health probe and print the resulting metric values, Prometheus text format.
183
+
184
+ Requires the ``prometheus`` extra. This is a wiring smoke test, not a live incident-triage
185
+ snapshot: this command is a fresh process with its own metrics registry, so it can only show
186
+ what *this command's own probe* recorded (pool gauges, one operation/duration observation per
187
+ target) — it has no way to read an already-running application's accumulated counters. Point
188
+ your real Prometheus scraper at the running application's own metrics endpoint for that; use
189
+ this command to sanity-check metric names/labels before wiring up dashboards/alerts (see
190
+ ``docs/observability.md``).
191
+ """
192
+ try:
193
+ from prometheus_client import CollectorRegistry, generate_latest
194
+ except ImportError as exc:
195
+ typer.echo(
196
+ "the `prometheus` extra is required: pip install dbkit-sql[prometheus]", err=True
197
+ )
198
+ raise typer.Exit(code=1) from exc
199
+
200
+ from .._core.query import Query, sql
201
+ from .._core.routing import DatabaseTarget
202
+ from ..observability.prometheus import PrometheusMetrics
203
+
204
+ config = _load_config(config_path)
205
+ registry = CollectorRegistry()
206
+ sink = PrometheusMetrics(namespace="dbkit", registry=registry)
207
+ probe = Query(name="dbkit.cli.metrics_probe", statement=sql("SELECT 1"))
208
+
209
+ async def _go() -> None:
210
+ db = AsyncDatabase.from_config(config, metrics=sink)
211
+ await db.start(warm=True)
212
+ try:
213
+ for name, target_config in config.databases.items():
214
+ if target_config.primary.required:
215
+ await db.fetch_value(probe, target=DatabaseTarget(database=name, role="write"))
216
+ db.pool_status() # samples pool gauges — otherwise-lazy, see _pool.py
217
+ finally:
218
+ await db.close()
219
+
220
+ _run(_go())
221
+ typer.echo(generate_latest(registry).decode())
222
+
223
+
224
+ @app.command("query-list")
225
+ def query_list() -> None:
226
+ """List queries registered on the process-global default registry.
227
+
228
+ Only queries constructed in this process appear — import your application's query
229
+ modules first if you need them listed. Writes marked ``idempotent=True`` with no visible
230
+ ``ON CONFLICT``-style guard in their SQL text get a best-effort warning — dbkit's retry
231
+ executor trusts ``idempotent=True`` as-is; it does not verify the SQL is actually safe to
232
+ run twice (§14).
233
+ """
234
+ names = default_registry.names()
235
+ if not names:
236
+ typer.echo("no queries registered in the default registry")
237
+ return
238
+ for name in names:
239
+ q = default_registry.get(name)
240
+ assert q is not None
241
+ line = f"{name} operation={q.operation} idempotent={q.idempotent}"
242
+ if looks_unsafe_to_retry(q):
243
+ line += (
244
+ " [WARNING: idempotent=True but no ON CONFLICT/guard detected — "
245
+ "verify this write is actually safe to run twice]"
246
+ )
247
+ typer.echo(line)
248
+
249
+
250
+ def main() -> None:
251
+ """Entry point for the ``dbkit`` console script."""
252
+ app()
253
+
254
+
255
+ if __name__ == "__main__":
256
+ main()
dbkit/errors.py ADDED
@@ -0,0 +1,6 @@
1
+ """Public re-export of the dbkit error model. See :mod:`dbkit._core.errors`."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from ._core.errors import * # noqa: F403
6
+ from ._core.errors import __all__ # noqa: F401
@@ -0,0 +1,21 @@
1
+ """Integration helpers for message-driven workloads (§28) and micro-batching (§17.1)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .batch import BatchCollector
6
+ from .inbox import (
7
+ ack_after_commit,
8
+ inbox_ddl,
9
+ inbox_month_partition_ddl,
10
+ partitioned_inbox_ddl,
11
+ process_once,
12
+ )
13
+
14
+ __all__ = [
15
+ "BatchCollector",
16
+ "ack_after_commit",
17
+ "inbox_ddl",
18
+ "inbox_month_partition_ddl",
19
+ "partitioned_inbox_ddl",
20
+ "process_once",
21
+ ]
@@ -0,0 +1,81 @@
1
+ """Consumer micro-batching (§17.1 high-perf addition).
2
+
3
+ ``BatchCollector`` aggregates items from many concurrent message handlers and flushes them as
4
+ one batched database write, either when the buffer reaches ``max_size`` or after
5
+ ``max_delay_ms`` — whichever comes first. This is the primary throughput lever for
6
+ message-driven ingestion: N handlers each contribute a row, one COPY/executemany persists them.
7
+
8
+ Async-only (message consumers are async). The flush callback receives the buffered items and
9
+ should perform the batched write (e.g. ``db.copy_records`` / ``db.insert_many``).
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import asyncio
15
+ from collections.abc import Awaitable, Callable, Sequence
16
+ from typing import Generic, TypeVar
17
+
18
+ T = TypeVar("T")
19
+
20
+
21
+ class BatchCollector(Generic[T]):
22
+ """Aggregates items across concurrent producers and flushes them as one batch (§17.1)."""
23
+
24
+ def __init__(
25
+ self,
26
+ flush: Callable[[Sequence[T]], Awaitable[None]],
27
+ *,
28
+ max_size: int = 1000,
29
+ max_delay_ms: float = 50.0,
30
+ ) -> None:
31
+ """``flush`` is called with the buffered batch whenever ``max_size``/``max_delay_ms``
32
+ is reached; it should perform the batched write (e.g. ``db.insert_many``)."""
33
+ if max_size < 1:
34
+ raise ValueError("max_size must be >= 1")
35
+ self._flush_cb = flush
36
+ self._max_size = max_size
37
+ self._max_delay = max_delay_ms / 1000.0
38
+ self._buffer: list[T] = []
39
+ self._lock = asyncio.Lock()
40
+ self._timer: asyncio.Task[None] | None = None
41
+ self._closed = False
42
+
43
+ async def add(self, item: T) -> None:
44
+ """Add an item; flush immediately if the buffer is full, else arm the flush timer."""
45
+ async with self._lock:
46
+ if self._closed:
47
+ raise RuntimeError("BatchCollector is closed")
48
+ self._buffer.append(item)
49
+ if len(self._buffer) >= self._max_size:
50
+ await self._flush_locked()
51
+ elif self._timer is None:
52
+ self._timer = asyncio.create_task(self._timer_flush())
53
+
54
+ async def flush(self) -> None:
55
+ """Flush any buffered items now."""
56
+ async with self._lock:
57
+ await self._flush_locked()
58
+
59
+ async def close(self) -> None:
60
+ """Flush remaining items and stop the timer."""
61
+ async with self._lock:
62
+ self._closed = True
63
+ await self._flush_locked()
64
+
65
+ async def _timer_flush(self) -> None:
66
+ try:
67
+ await asyncio.sleep(self._max_delay)
68
+ except asyncio.CancelledError:
69
+ return
70
+ async with self._lock:
71
+ await self._flush_locked()
72
+
73
+ async def _flush_locked(self) -> None:
74
+ if self._timer is not None:
75
+ self._timer.cancel()
76
+ self._timer = None
77
+ if not self._buffer:
78
+ return
79
+ items = self._buffer
80
+ self._buffer = []
81
+ await self._flush_cb(items)
@@ -0,0 +1,153 @@
1
+ """Inbox / idempotent-consume helpers for message-driven writes (§28).
2
+
3
+ These make RabbitMQ (or any broker) message processing safe against redelivery. The inbox row
4
+ and the business writes commit in the **same** transaction, so a message is processed
5
+ exactly-once even though delivery is at-least-once. Ack the broker message only after the
6
+ commit succeeds; on a commit-unknown outcome, do NOT ack — require idempotent replay (§15, §28).
7
+
8
+ Async helpers targeting :class:`AsyncDatabase`. The DDL is frontend-agnostic.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import contextlib
14
+ from collections.abc import AsyncIterator, Awaitable, Callable
15
+ from typing import Any
16
+
17
+ from .._core.query import sql
18
+ from .._core.routing import DatabaseTarget
19
+ from ..errors import DatabaseCommitUnknownError, DatabaseError
20
+
21
+ DEFAULT_INBOX_TABLE = "consumed_messages"
22
+
23
+
24
+ def inbox_ddl(table: str = DEFAULT_INBOX_TABLE) -> str:
25
+ """DDL for the inbox table (§28.3). Time-partition it in production for cheap pruning —
26
+ see :func:`partitioned_inbox_ddl`/:func:`inbox_month_partition_ddl` for that variant."""
27
+ return f"""
28
+ CREATE TABLE IF NOT EXISTS {table} (
29
+ consumer_name TEXT NOT NULL,
30
+ message_id TEXT NOT NULL,
31
+ processed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
32
+ PRIMARY KEY (consumer_name, message_id)
33
+ )
34
+ """
35
+
36
+
37
+ def partitioned_inbox_ddl(table: str = DEFAULT_INBOX_TABLE) -> str:
38
+ """DDL for a **partitioned** inbox table, range-partitioned by ``processed_at`` (§28.3).
39
+
40
+ A single unbounded ``consumed_messages`` table's dedup lookup gets slower and its storage
41
+ grows without bound as message volume accumulates. Range-partitioning by month makes
42
+ pruning old data a cheap ``DROP TABLE`` on a partition instead of a slow ``DELETE``. The
43
+ partition key must be part of the primary key for declarative partitioning, so this variant
44
+ keys on ``(consumer_name, message_id, processed_at)`` instead of just the first two.
45
+
46
+ Partitions themselves aren't created automatically — call
47
+ :func:`inbox_month_partition_ddl` for each month you need (typically from a scheduled job
48
+ that creates the next 1-2 months ahead of time; `pg_partman` or a cron-triggered `dbkit`
49
+ script both work). Older partitions can simply be dropped once no longer needed for
50
+ dedup/audit purposes — a message's redelivery window is bounded by the broker's own
51
+ retention, so a partition older than that window is always safe to drop.
52
+ """
53
+ return f"""
54
+ CREATE TABLE IF NOT EXISTS {table} (
55
+ consumer_name TEXT NOT NULL,
56
+ message_id TEXT NOT NULL,
57
+ processed_at TIMESTAMPTZ NOT NULL DEFAULT now(),
58
+ PRIMARY KEY (consumer_name, message_id, processed_at)
59
+ ) PARTITION BY RANGE (processed_at)
60
+ """
61
+
62
+
63
+ def inbox_month_partition_ddl(year: int, month: int, *, table: str = DEFAULT_INBOX_TABLE) -> str:
64
+ """DDL for one calendar-month partition of :func:`partitioned_inbox_ddl`'s table.
65
+
66
+ ``year``/``month`` are passed explicitly (not computed from the current date) so this stays
67
+ a pure function — call it with whatever month you actually need created, e.g. from a
68
+ scheduled job that creates the next month a few days before it starts.
69
+ """
70
+ start = f"{year:04d}-{month:02d}-01"
71
+ next_year, next_month = (year + 1, 1) if month == 12 else (year, month + 1)
72
+ end = f"{next_year:04d}-{next_month:02d}-01"
73
+ partition_name = f"{table}_{year:04d}_{month:02d}"
74
+ return f"""
75
+ CREATE TABLE IF NOT EXISTS {partition_name} PARTITION OF {table}
76
+ FOR VALUES FROM ('{start}') TO ('{end}')
77
+ """
78
+
79
+
80
+ async def _claim(tx: Any, consumer: str, message_id: str, table: str) -> bool:
81
+ """Insert the inbox row; return True if this is the first time we've seen the message."""
82
+ row = await tx.fetch_optional(
83
+ sql(
84
+ f"INSERT INTO {table} (consumer_name, message_id) VALUES (:c, :m) "
85
+ "ON CONFLICT (consumer_name, message_id) DO NOTHING RETURNING 1"
86
+ ),
87
+ {"c": consumer, "m": message_id},
88
+ )
89
+ return row is not None
90
+
91
+
92
+ @contextlib.asynccontextmanager
93
+ async def process_once(
94
+ db: Any,
95
+ *,
96
+ consumer: str,
97
+ message_id: str,
98
+ target: DatabaseTarget,
99
+ inbox_table: str = DEFAULT_INBOX_TABLE,
100
+ ) -> AsyncIterator[tuple[Any, bool]]:
101
+ """Open a transaction, claim the message in the inbox, and yield ``(tx, first_time)``.
102
+
103
+ Do the business writes on ``tx`` only when ``first_time`` is True; a duplicate delivery
104
+ still commits (a harmless no-op) so the broker message can be acked::
105
+
106
+ async with process_once(db, consumer="c", message_id=mid, target=t) as (tx, first):
107
+ if first:
108
+ await tx.execute(INSERT_ORDER, params)
109
+ """
110
+ async with db.transaction(target=target) as tx:
111
+ first = await _claim(tx, consumer, message_id, inbox_table)
112
+ yield tx, first
113
+
114
+
115
+ async def ack_after_commit(
116
+ db: Any,
117
+ *,
118
+ consumer: str,
119
+ message_id: str,
120
+ target: DatabaseTarget,
121
+ work: Callable[[Any], Awaitable[None]],
122
+ ack: Callable[[], Awaitable[None]],
123
+ dead_letter: Callable[[DatabaseError], Awaitable[None]] | None = None,
124
+ retry: Callable[[DatabaseError], Awaitable[None]] | None = None,
125
+ inbox_table: str = DEFAULT_INBOX_TABLE,
126
+ ) -> bool:
127
+ """Run ``work`` idempotently, then ack — the full §28 consume flow.
128
+
129
+ Returns True if the message was processed (or was a duplicate) and acked. Behavior on
130
+ failure:
131
+
132
+ * commit-unknown -> re-raised; the message is **not** acked (idempotent replay will dedupe).
133
+ * retryable error -> ``retry`` callback if given, else re-raised (nack/requeue).
134
+ * permanent error -> ``dead_letter`` callback if given, else re-raised (route to DLQ).
135
+ """
136
+ try:
137
+ async with process_once(
138
+ db, consumer=consumer, message_id=message_id, target=target, inbox_table=inbox_table
139
+ ) as (tx, first):
140
+ if first:
141
+ await work(tx)
142
+ except DatabaseCommitUnknownError:
143
+ raise # never ack on unknown commit outcome
144
+ except DatabaseError as exc:
145
+ if exc.retryable and retry is not None:
146
+ await retry(exc)
147
+ return False
148
+ if not exc.retryable and dead_letter is not None:
149
+ await dead_letter(exc)
150
+ return False
151
+ raise
152
+ await ack()
153
+ return True
@@ -0,0 +1,21 @@
1
+ """Observability: structured logging, metrics, and OpenTelemetry tracing (§25)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from .logging import log_event, logger, long_transaction_warning, slow_query_warning
6
+ from .metrics import MetricsSink, NoopMetrics, try_otel_metrics_sink, try_prometheus_sink
7
+ from .tracing import SpanHandle, Tracer, make_tracer
8
+
9
+ __all__ = [
10
+ "MetricsSink",
11
+ "NoopMetrics",
12
+ "SpanHandle",
13
+ "Tracer",
14
+ "log_event",
15
+ "logger",
16
+ "long_transaction_warning",
17
+ "make_tracer",
18
+ "slow_query_warning",
19
+ "try_otel_metrics_sink",
20
+ "try_prometheus_sink",
21
+ ]
@@ -0,0 +1,154 @@
1
+ """Structured logging helpers (§25.3).
2
+
3
+ dbkit logs through the stdlib ``logging`` module under the ``dbkit`` logger, passing a
4
+ structured payload in ``extra={"dbkit": {...}}`` so a JSON formatter can render it. Bound
5
+ parameters are never logged unless explicitly enabled, and sensitive params are always
6
+ redacted.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import logging
12
+ from collections.abc import Mapping
13
+ from typing import Any
14
+
15
+ logger = logging.getLogger("dbkit")
16
+
17
+ try:
18
+ from opentelemetry import trace as _otel_trace
19
+
20
+ _OTEL_AVAILABLE = True
21
+ except ImportError: # pragma: no cover - exercised by the no-otel test lane
22
+ _OTEL_AVAILABLE = False
23
+
24
+
25
+ def _trace_context() -> dict[str, str]:
26
+ """``trace_id``/``span_id`` of the current OTel span, for trace/log correlation (§25.2/§25.3).
27
+
28
+ Empty if OTel isn't installed or no span is currently recording. The availability check is
29
+ done once at import time (not per call) so this stays cheap on the hot logging path even
30
+ when OTel isn't installed — a per-call ``import opentelemetry`` would otherwise re-walk
31
+ ``sys.path`` on every log event, since failed imports aren't cached in ``sys.modules``.
32
+ """
33
+ if not _OTEL_AVAILABLE:
34
+ return {}
35
+ ctx = _otel_trace.get_current_span().get_span_context()
36
+ if not ctx.is_valid:
37
+ return {}
38
+ return {"trace_id": format(ctx.trace_id, "032x"), "span_id": format(ctx.span_id, "016x")}
39
+
40
+
41
+ def log_event(
42
+ level: int,
43
+ event: str,
44
+ *,
45
+ query_name: str | None = None,
46
+ database: str | None = None,
47
+ shard: str | None = None,
48
+ role: str | None = None,
49
+ duration_ms: float | None = None,
50
+ pool_wait_ms: float | None = None,
51
+ rows: int | None = None,
52
+ retry_attempt: int | None = None,
53
+ error_code: str | None = None,
54
+ **extra: Any,
55
+ ) -> None:
56
+ """Emit one structured lifecycle event (§25.3)."""
57
+ if not logger.isEnabledFor(level):
58
+ return
59
+ payload: dict[str, Any] = {"event": event}
60
+ for key, value in (
61
+ ("query_name", query_name),
62
+ ("database", database),
63
+ ("shard", shard),
64
+ ("role", role),
65
+ ("duration_ms", duration_ms),
66
+ ("pool_wait_ms", pool_wait_ms),
67
+ ("rows", rows),
68
+ ("retry_attempt", retry_attempt),
69
+ ("error_code", error_code),
70
+ ):
71
+ if value is not None:
72
+ payload[key] = value
73
+ payload.update(_trace_context())
74
+ payload.update(extra)
75
+ logger.log(level, event, extra={"dbkit": payload})
76
+
77
+
78
+ def slow_query_warning(
79
+ *,
80
+ query_name: str,
81
+ duration_ms: float,
82
+ threshold_ms: float,
83
+ database: str | None = None,
84
+ pool_wait_ms: float | None = None,
85
+ rows: int | None = None,
86
+ ) -> None:
87
+ """Emit a ``database.query.slow`` warning (caller decides whether the threshold was hit)."""
88
+ log_event(
89
+ logging.WARNING,
90
+ "database.query.slow",
91
+ query_name=query_name,
92
+ database=database,
93
+ duration_ms=round(duration_ms, 3),
94
+ pool_wait_ms=None if pool_wait_ms is None else round(pool_wait_ms, 3),
95
+ rows=rows,
96
+ threshold_ms=threshold_ms,
97
+ )
98
+
99
+
100
+ def long_transaction_warning(
101
+ *,
102
+ duration_ms: float,
103
+ threshold_ms: float,
104
+ database: str | None = None,
105
+ role: str | None = None,
106
+ outcome: str | None = None,
107
+ ) -> None:
108
+ """Warn when an explicit transaction was held open longer than the configured threshold —
109
+ the transaction analog of the pool's long-connection-hold warning (§10.5, §16)."""
110
+ log_event(
111
+ logging.WARNING,
112
+ "database.transaction.long_running",
113
+ database=database,
114
+ role=role,
115
+ duration_ms=round(duration_ms, 3),
116
+ threshold_ms=threshold_ms,
117
+ outcome=outcome,
118
+ )
119
+
120
+
121
+ def bulk_batch_dropped_warning(
122
+ *,
123
+ query_name: str,
124
+ database: str | None = None,
125
+ mode: str,
126
+ rows_dropped: int,
127
+ error_category: str | None = None,
128
+ ) -> None:
129
+ """Warn that a bulk batch (``best_effort``) or row (``split_on_failure``, after its batch
130
+ already failed) was dropped rather than written — this is the only signal a caller gets for
131
+ silently-lost rows under either failure mode (§19.3, performance review §9)."""
132
+ log_event(
133
+ logging.WARNING,
134
+ "database.bulk.rows_dropped",
135
+ query_name=query_name,
136
+ database=database,
137
+ mode=mode,
138
+ rows_dropped=rows_dropped,
139
+ error_category=error_category,
140
+ )
141
+
142
+
143
+ def redacted_params_for_log(
144
+ params: Mapping[str, Any] | None,
145
+ *,
146
+ log_parameters: bool,
147
+ sensitive: set[str] | None,
148
+ ) -> dict[str, Any] | None:
149
+ """Return params suitable for logging, or ``None`` when parameter logging is disabled."""
150
+ if not log_parameters or not params:
151
+ return None
152
+ from .._core.errors.redaction import redact_params
153
+
154
+ return redact_params(params, sensitive=sensitive)