dbkit-sql 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. dbkit/__init__.py +77 -0
  2. dbkit/_async/__init__.py +17 -0
  3. dbkit/_async/_compat.py +111 -0
  4. dbkit/_async/connection.py +328 -0
  5. dbkit/_async/database.py +884 -0
  6. dbkit/_async/engine.py +232 -0
  7. dbkit/_async/executor.py +329 -0
  8. dbkit/_async/health.py +53 -0
  9. dbkit/_async/resilience.py +148 -0
  10. dbkit/_async/streaming.py +137 -0
  11. dbkit/_async/transaction.py +264 -0
  12. dbkit/_core/__init__.py +1 -0
  13. dbkit/_core/bulk.py +91 -0
  14. dbkit/_core/circuit.py +130 -0
  15. dbkit/_core/config.py +613 -0
  16. dbkit/_core/errors/__init__.py +74 -0
  17. dbkit/_core/errors/base.py +344 -0
  18. dbkit/_core/errors/classifier.py +138 -0
  19. dbkit/_core/errors/redaction.py +96 -0
  20. dbkit/_core/errors/sqlstate.py +74 -0
  21. dbkit/_core/idempotency_lint.py +57 -0
  22. dbkit/_core/keys.py +29 -0
  23. dbkit/_core/policies.py +105 -0
  24. dbkit/_core/query.py +142 -0
  25. dbkit/_core/result.py +109 -0
  26. dbkit/_core/routing.py +248 -0
  27. dbkit/_pool.py +202 -0
  28. dbkit/_sync/__init__.py +20 -0
  29. dbkit/_sync/_compat.py +82 -0
  30. dbkit/_sync/connection.py +331 -0
  31. dbkit/_sync/database.py +887 -0
  32. dbkit/_sync/engine.py +235 -0
  33. dbkit/_sync/executor.py +332 -0
  34. dbkit/_sync/health.py +56 -0
  35. dbkit/_sync/resilience.py +151 -0
  36. dbkit/_sync/streaming.py +140 -0
  37. dbkit/_sync/transaction.py +267 -0
  38. dbkit/cli/__init__.py +1 -0
  39. dbkit/cli/main.py +256 -0
  40. dbkit/errors.py +6 -0
  41. dbkit/integrations/__init__.py +21 -0
  42. dbkit/integrations/batch.py +81 -0
  43. dbkit/integrations/inbox.py +153 -0
  44. dbkit/observability/__init__.py +21 -0
  45. dbkit/observability/logging.py +154 -0
  46. dbkit/observability/metrics.py +110 -0
  47. dbkit/observability/otel_metrics.py +69 -0
  48. dbkit/observability/prometheus.py +78 -0
  49. dbkit/observability/tracing.py +128 -0
  50. dbkit/postgres/__init__.py +1 -0
  51. dbkit/postgres/copy.py +77 -0
  52. dbkit/postgres/pipeline.py +48 -0
  53. dbkit/postgres/unnest.py +73 -0
  54. dbkit/testing/__init__.py +1 -0
  55. dbkit_sql-0.1.0.dist-info/METADATA +555 -0
  56. dbkit_sql-0.1.0.dist-info/RECORD +59 -0
  57. dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
  58. dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
  59. dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,555 @@
1
+ Metadata-Version: 2.4
2
+ Name: dbkit-sql
3
+ Version: 0.1.0
4
+ Summary: A thin, high-throughput SQL toolkit over SQLAlchemy Core — sync and async, ORM-free.
5
+ Project-URL: Homepage, https://github.com/talaatmagdyx/dbkit-sql
6
+ Project-URL: Source, https://github.com/talaatmagdyx/dbkit-sql
7
+ Project-URL: Issues, https://github.com/talaatmagdyx/dbkit-sql/issues
8
+ Author: dbkit contributors
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: async,connection-pool,database,postgresql,sql,sqlalchemy
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Database
19
+ Classifier: Typing :: Typed
20
+ Requires-Python: >=3.11
21
+ Requires-Dist: sqlalchemy[asyncio]>=2.0.30
22
+ Provides-Extra: asyncpg
23
+ Requires-Dist: asyncpg>=0.29; extra == 'asyncpg'
24
+ Provides-Extra: bench
25
+ Requires-Dist: psutil>=5.9; extra == 'bench'
26
+ Provides-Extra: cli
27
+ Requires-Dist: typer>=0.12; extra == 'cli'
28
+ Provides-Extra: dev
29
+ Requires-Dist: asyncpg>=0.29; extra == 'dev'
30
+ Requires-Dist: build>=1.2; extra == 'dev'
31
+ Requires-Dist: docker>=7.0; extra == 'dev'
32
+ Requires-Dist: hypothesis>=6.100; extra == 'dev'
33
+ Requires-Dist: mkdocs-material>=9.5; extra == 'dev'
34
+ Requires-Dist: mkdocs>=1.6; extra == 'dev'
35
+ Requires-Dist: mkdocstrings[python]>=0.26; extra == 'dev'
36
+ Requires-Dist: mypy>=1.10; extra == 'dev'
37
+ Requires-Dist: opentelemetry-api>=1.26; extra == 'dev'
38
+ Requires-Dist: opentelemetry-sdk>=1.26; extra == 'dev'
39
+ Requires-Dist: prometheus-client>=0.20; extra == 'dev'
40
+ Requires-Dist: psutil>=5.9; extra == 'dev'
41
+ Requires-Dist: psycopg[binary]>=3.2; extra == 'dev'
42
+ Requires-Dist: pydantic>=2.7; extra == 'dev'
43
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
44
+ Requires-Dist: pytest-cov>=5.0; extra == 'dev'
45
+ Requires-Dist: pytest>=8.2; extra == 'dev'
46
+ Requires-Dist: pyyaml>=6.0; extra == 'dev'
47
+ Requires-Dist: ruff>=0.5; extra == 'dev'
48
+ Requires-Dist: testcontainers[postgres]>=4.5; extra == 'dev'
49
+ Requires-Dist: twine>=5.1; extra == 'dev'
50
+ Requires-Dist: typer>=0.12; extra == 'dev'
51
+ Provides-Extra: orjson
52
+ Requires-Dist: orjson>=3.10; extra == 'orjson'
53
+ Provides-Extra: otel
54
+ Requires-Dist: opentelemetry-api>=1.26; extra == 'otel'
55
+ Provides-Extra: prometheus
56
+ Requires-Dist: prometheus-client>=0.20; extra == 'prometheus'
57
+ Provides-Extra: psycopg
58
+ Requires-Dist: psycopg[binary]>=3.2; extra == 'psycopg'
59
+ Provides-Extra: pydantic
60
+ Requires-Dist: pydantic>=2.7; extra == 'pydantic'
61
+ Provides-Extra: yaml
62
+ Requires-Dist: pyyaml>=6.0; extra == 'yaml'
63
+ Description-Content-Type: text/markdown
64
+
65
+ <p align="center"><img src="https://raw.githubusercontent.com/talaatmagdyx/dbkit-sql/main/docs/assets/logo.svg" alt="dbkit" width="120"></p>
66
+
67
+ # dbkit
68
+
69
+ **SQLAlchemy Core made enjoyable — less pool/retry plumbing, more SQL that matters.**
70
+
71
+ [![PyPI](https://img.shields.io/pypi/v/dbkit-sql)](https://pypi.org/project/dbkit-sql/)
72
+ [![CI](https://github.com/talaatmagdyx/dbkit-sql/actions/workflows/ci.yml/badge.svg)](https://github.com/talaatmagdyx/dbkit-sql/actions/workflows/ci.yml)
73
+ [![Python](https://img.shields.io/badge/python-3.11%20%7C%203.12%20%7C%203.13-blue)](https://github.com/talaatmagdyx/dbkit-sql/blob/main/pyproject.toml)
74
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](https://github.com/talaatmagdyx/dbkit-sql/blob/main/LICENSE)
75
+ [![Typed](https://img.shields.io/badge/types-mypy%20--strict-blue)](https://github.com/talaatmagdyx/dbkit-sql/blob/main/pyproject.toml)
76
+ [![Style](https://img.shields.io/badge/style-ruff-261230)](https://github.com/talaatmagdyx/dbkit-sql/blob/main/pyproject.toml)
77
+
78
+ dbkit is a **SQL-first toolkit for Python services built on top of SQLAlchemy Core**. It gives
79
+ you one API for sync and async, bounded connection pools, idempotency-gated retries, a circuit
80
+ breaker, sharding and replica routing, bounded-memory streaming, adaptive-batch bulk writes, and
81
+ structured observability — so your team can focus on what each query *does*, not on rebuilding
82
+ pool sizing, retry loops, and error classification in every service.
83
+
84
+ SQLAlchemy Core is powerful and deliberately unopinionated about all of that. dbkit adds the
85
+ missing layer **without hiding SQLAlchemy from you** — no ORM, no sessions, no query builder
86
+ magic standing between you and the SQL you wrote.
87
+
88
+ ```python
89
+ from dbkit import AsyncDatabase, DatabaseTarget, Query, sql
90
+
91
+ db = AsyncDatabase.from_config({
92
+ "databases": {"app": {"primary": {"url": "postgresql+psycopg://localhost/app"}}},
93
+ })
94
+
95
+ GET_USER = Query(
96
+ name="users.get_by_id",
97
+ statement=sql("SELECT id, email FROM users WHERE id = :id"),
98
+ operation="read", idempotent=True, timeout=1.0,
99
+ )
100
+
101
+ async def main() -> None:
102
+ await db.start()
103
+ user = await db.fetch_optional(GET_USER, {"id": 1}, target=DatabaseTarget(database="app"))
104
+ ```
105
+
106
+ That should feel like application code. The pool sizing, retry/backoff policy, error
107
+ classification, transaction lifecycle, and test infrastructure should not be rewritten in every
108
+ service that talks to Postgres. **That is what dbkit is for.**
109
+
110
+ **Contents:**
111
+ - [What dbkit believes](#what-dbkit-believes)
112
+ - [Why dbkit exists](#why-dbkit-exists)
113
+ - [Install](#install)
114
+ - [Quick start](#quick-start)
115
+ - [Consistency & retry model](#consistency--retry-model)
116
+ - [What happens when things fail](#what-happens-when-things-fail)
117
+ - [Retry & concurrency policy knobs](#retry--concurrency-policy-knobs)
118
+ - [Production profile](#production-profile)
119
+ - [Observability](#observability)
120
+ - [Sharding & replica routing](#sharding--replica-routing)
121
+ - [Building blocks](#building-blocks-batteries-included)
122
+ - [Operate it from the terminal](#operate-it-from-the-terminal)
123
+ - [Where dbkit fits](#where-dbkit-fits)
124
+ - [Performance](#performance)
125
+ - [Migrating from raw SQLAlchemy Core](#migrating-from-raw-sqlalchemy-core)
126
+ - [Examples](#examples)
127
+ - [Architecture](#architecture)
128
+ - [Compatibility](#compatibility)
129
+ - [Documentation](#documentation)
130
+ - [Contributing & security](#contributing--security)
131
+ - [License](#license)
132
+
133
+ ---
134
+
135
+ ## What dbkit believes
136
+
137
+ Most services that talk to PostgreSQL need the same things:
138
+
139
+ - one API whether the service is sync or async
140
+ - a connection pool sized on purpose, not by accident
141
+ - retries that only ever touch operations known to be safe to repeat
142
+ - errors classified by what actually happened, not a raw driver exception
143
+ - a circuit breaker so a downed database doesn't get hammered
144
+ - streaming and bulk paths that don't buffer a million rows in memory
145
+ - structured logs/metrics/traces with secrets redacted by default
146
+ - tests that run against a real database, because SQL correctness is not
147
+ something an in-memory fake can honestly stand in for
148
+
149
+ dbkit packages those concerns into one focused toolkit. The philosophy:
150
+
151
+ > **Make SQLAlchemy Core pleasant for developers and predictable for operators.**
152
+
153
+ Developers get a clean, typed query/config model. Operators get classified errors, bounded
154
+ resources, and visibility into what the pool and the retry policy are actually doing.
155
+
156
+ ## Why dbkit exists
157
+
158
+ Starting with SQLAlchemy Core is easy: `engine.connect()`, `conn.execute(text(...))`. Then
159
+ production asks better questions:
160
+
161
+ - What happens if the client gives up mid-query — does the server keep running it anyway?
162
+ - Is that retry actually safe, or did it just duplicate a write?
163
+ - Did the connection die *during* commit — did the write happen or not?
164
+ - Can 500 concurrent requests open 500 connections and take Postgres down with them?
165
+ - Can a shard-routing config balloon connection count in ways nobody sized for?
166
+ - Can CI catch a broken transaction rollback without a live Postgres container?
167
+
168
+ dbkit exists for those questions. Its goal is not to replace SQLAlchemy — it's to make direct
169
+ SQLAlchemy Core usage feel like good application code: bounded pools, explicit transactions,
170
+ classified failures, real tests, production-ready lifecycle.
171
+
172
+ **dbkit is:**
173
+
174
+ - a SQL-first toolkit over SQLAlchemy Core
175
+ - a resilience layer (retries, circuit breaker, concurrency limits)
176
+ - a sharding/replica-routing layer for multi-database deployments
177
+ - a high-throughput layer (streaming, bulk insert/upsert, PostgreSQL COPY)
178
+ - an observability layer (structured logs, Prometheus, OpenTelemetry)
179
+
180
+ **dbkit is not:**
181
+
182
+ - an ORM — no models, sessions, or relationship loading
183
+ - a schema-migration tool — pair it with Alembic
184
+ - a message broker client — the transactional-inbox helpers are primitives your
185
+ own consumer loop wires in, not a broker integration
186
+ - an exactly-once delivery system — retries are at-least-once, gated by `idempotent=True`
187
+
188
+ ---
189
+
190
+ ## Install
191
+
192
+ Available on PyPI: **[pypi.org/project/dbkit-sql](https://pypi.org/project/dbkit-sql/)**
193
+
194
+ ```bash
195
+ pip install "dbkit-sql[psycopg]" # async + sync PostgreSQL
196
+ pip install "dbkit-sql[psycopg,yaml,prometheus,otel,cli]"
197
+ ```
198
+
199
+ Requires Python ≥ 3.11. The distribution is `dbkit-sql` (the name `dbkit` was already taken on
200
+ PyPI) — the import stays `import dbkit` regardless of which extras you install.
201
+
202
+ ## Quick start
203
+
204
+ ### 1. Connect and query
205
+
206
+ ```python
207
+ from dbkit import AsyncDatabase, DatabaseTarget, Query, sql
208
+
209
+ GET_USER = Query(
210
+ name="users.get_by_id",
211
+ statement=sql("SELECT id, email FROM users WHERE id = :id"),
212
+ operation="read",
213
+ idempotent=True,
214
+ timeout=1.0,
215
+ )
216
+
217
+ db = AsyncDatabase.from_config({
218
+ "databases": {"app": {"primary": {"url": "postgresql+psycopg://localhost/app"}}},
219
+ })
220
+
221
+ async def main():
222
+ await db.start()
223
+ try:
224
+ user = await db.fetch_optional(GET_USER, {"id": 1},
225
+ target=DatabaseTarget(database="app"))
226
+ finally:
227
+ await db.close()
228
+ ```
229
+
230
+ ### 2. Explicit transactions with real isolation control
231
+
232
+ ```python
233
+ async with db.transaction(target=DatabaseTarget(database="app"), isolation="serializable") as tx:
234
+ await tx.execute(sql("UPDATE accounts SET balance = balance - :amt WHERE id = :from_id"),
235
+ {"amt": 100, "from_id": 1})
236
+ await tx.execute(sql("UPDATE accounts SET balance = balance + :amt WHERE id = :to_id"),
237
+ {"amt": 100, "to_id": 2})
238
+ ```
239
+
240
+ A deadlock or serialization failure here surfaces as a classified, `retryable`
241
+ `DatabaseDeadlockError`/`DatabaseSerializationError` — dbkit never auto-retries an explicit
242
+ transaction body (that would mean silently re-running your code), so you catch and retry at the
243
+ call site, same as any other classified error.
244
+
245
+ ### 3. Retries and a circuit breaker, configured once
246
+
247
+ ```python
248
+ db = AsyncDatabase.from_config({
249
+ "databases": {"app": {"primary": {"url": "postgresql+psycopg://localhost/app"}}},
250
+ "defaults": {
251
+ "retry": {"attempts": 3, "retry_reads": True, "maximum_total_ms": 750},
252
+ "circuit_breaker": {"enabled": True, "failure_threshold": 5, "open_seconds": 30},
253
+ },
254
+ })
255
+ ```
256
+
257
+ Reads marked `idempotent=True` retry transparently on transient failures (serialization
258
+ failures, deadlocks, connection drops). Writes only retry if you explicitly opt in with
259
+ `retry_writes=True` *and* mark the query `idempotent=True` — dbkit will not guess.
260
+
261
+ ### 4. Test against a real, disposable PostgreSQL
262
+
263
+ dbkit doesn't ship an in-memory fake database — SQL correctness (isolation levels, deadlocks,
264
+ constraint violations, `SELECT ... FOR UPDATE`) is genuinely a property of the real engine, not
265
+ something worth faking. Instead, point tests at a disposable container:
266
+
267
+ ```python
268
+ import pytest
269
+ from dbkit import AsyncDatabase
270
+
271
+ @pytest.fixture
272
+ async def db(pg_dsn):
273
+ database = AsyncDatabase.from_config({"databases": {"app": {"primary": {"url": pg_dsn}}}})
274
+ await database.start()
275
+ yield database
276
+ await database.close()
277
+ ```
278
+
279
+ `docs/testing.md` covers the `testcontainers`-based fixture this project's own suite uses.
280
+
281
+ ### 5. Wire into any framework's lifespan
282
+
283
+ ```python
284
+ from contextlib import asynccontextmanager
285
+ from fastapi import FastAPI
286
+
287
+ @asynccontextmanager
288
+ async def lifespan(app: FastAPI):
289
+ await db.start()
290
+ try:
291
+ yield
292
+ finally:
293
+ await db.close()
294
+
295
+ app = FastAPI(lifespan=lifespan)
296
+ ```
297
+
298
+ ### Sync example
299
+
300
+ ```python
301
+ from dbkit import Database
302
+ db = Database.from_config(...)
303
+ db.start()
304
+ user = db.fetch_optional(GET_USER, {"id": 1}, target=DatabaseTarget(database="app"))
305
+ db.close()
306
+ ```
307
+
308
+ The sync API is generated from the async one (`tools/run_unasync.py`), so the two never drift —
309
+ fits simple workers, scripts, and teams that don't want an asyncio runtime.
310
+
311
+ ---
312
+
313
+ ## Consistency & retry model
314
+
315
+ dbkit is an **at-least-once** toolkit for retried operations: a network blip right at commit
316
+ means the outcome is genuinely unknown, not silently "it worked." dbkit never guesses in that
317
+ case — it raises a distinct `DatabaseCommitUnknownError` instead of reporting success or
318
+ silently retrying a write that may have already landed.
319
+
320
+ For anything where a duplicate write would be a real problem: mark the query `idempotent=True`
321
+ only when it actually is (`ON CONFLICT`, `WHERE NOT EXISTS`, `MERGE` — the idempotency lint
322
+ flags `INSERT`s missing an obvious guard), and keep `retry_writes` off for anything that isn't.
323
+ The rule is simple:
324
+
325
+ > dbkit can help you retry safely. Your write must still be safe to retry.
326
+
327
+ ## What happens when things fail?
328
+
329
+ | Failure mode | dbkit behavior |
330
+ |---|---|
331
+ | Client times out mid-query | The driver sends a real server-side cancel — the abandoned query does not keep running (verified against both psycopg3 and asyncpg) |
332
+ | Deadlock (`40P01`) | Classified `DatabaseDeadlockError`, `retryable=True`; explicit transactions are not auto-retried — you catch and retry |
333
+ | Serialization failure (`40001`) | Classified `DatabaseSerializationError`, transparently retried for idempotent reads |
334
+ | Connection dies mid-commit | `DatabaseCommitUnknownError` — never reported as success, never silently retried |
335
+ | Pool exhausted | `DatabasePoolTimeoutError` within `timeout_seconds` — a queued wait, not a hang |
336
+ | Concurrency-limiter tier saturated | `DatabaseOverloadedError` — fails fast instead of queueing forever |
337
+ | Sustained backend failures | Circuit breaker opens; fails fast with `DatabaseCircuitOpenError` instead of hammering a downed database |
338
+ | Unique/FK/check constraint violated | `DatabaseUniqueViolationError` / `DatabaseForeignKeyViolationError` / `DatabaseCheckViolationError` — distinct, catchable types |
339
+ | Bulk write partially fails (`best_effort`) | Dropped rows/batches are logged and counted (`db_bulk_rows_dropped_total`), never silently discarded |
340
+ | High concurrency vs. a small shard-engine cache | Real connection exhaustion, but always the classified `DatabaseConnectionError` — never a hang |
341
+
342
+ ## Retry & concurrency policy knobs
343
+
344
+ | Knob | What it controls |
345
+ |---|---|
346
+ | `retry.retry_reads` / `retry.retry_writes` | Whether reads/writes are eligible for automatic retry at all |
347
+ | `Query(idempotent=True)` | The gate that actually authorizes a write to be retried — trust-based, lint-assisted |
348
+ | `retry.attempts` / `retry.maximum_total_ms` | A hard ceiling on retry attempts *and* total elapsed retry time, whichever hits first |
349
+ | `circuit_breaker.failure_threshold` / `open_seconds` | When the breaker trips, and how long it stays open before a trial request |
350
+ | `concurrency.*` tiers (`reads`/`writes`/`bulk`/`expensive`) | Independent semaphores acquired before pool checkout — an expensive report query can't starve ordinary reads |
351
+ | `pool.size` / `max_overflow` / `timeout_seconds` | The connection budget per database×shard×role, and how long a caller waits for one |
352
+
353
+ ## Production profile
354
+
355
+ The recommended baseline: `connection_budget.enforce_at_startup=True` so a misconfigured pool
356
+ fails at boot instead of exhausting Postgres under load; `max_engines` sized to comfortably
357
+ exceed your expected *concurrent* distinct-shard fan-out (not just total shard count — see
358
+ `benchmarks/bench_shard_cardinality.py` for why those are different numbers) with
359
+ `evict_lru=True` for dynamic per-tenant topologies; `retry_writes` scoped only to genuinely
360
+ idempotent queries; TLS/`sslmode` checked via `dbkit check` before every environment promotion.
361
+ See [`docs/security.md`](docs/security.md) and [`docs/troubleshooting.md`](docs/troubleshooting.md)
362
+ for the full posture and common failure modes.
363
+
364
+ ## Observability
365
+
366
+ Structured logs carry operation context (query name, database/shard/role, duration, outcome,
367
+ retry attempt) with secret redaction on by default. Metrics cover retries, circuit-breaker
368
+ state, pool utilization, transaction outcomes (including the distinct commit-unknown case), and
369
+ bulk-write drops — both a Prometheus and an OpenTelemetry metrics sink are supported, at a
370
+ measured, not assumed, cost (see [Performance](#performance)). Tracing is standard
371
+ OpenTelemetry (`pip install dbkit-sql[otel]`), one continuous span tree per operation.
372
+
373
+ ## Sharding & replica routing
374
+
375
+ ```python
376
+ from dbkit import AsyncDatabase, DatabaseTarget, HashShardResolver
377
+
378
+ db = AsyncDatabase.from_config(
379
+ {
380
+ "databases": {"app": {"primary": {"url": "..."}, "replicas": [{"name": "r1", "url": "..."}]}},
381
+ "max_engines": 200,
382
+ "evict_lru_engines": True,
383
+ },
384
+ shard_resolver=HashShardResolver(num_shards=64),
385
+ )
386
+
387
+ target = DatabaseTarget(database="app", role="read", shard_key=f"tenant-{tenant_id}")
388
+ ```
389
+
390
+ Pluggable resolvers (hash / range / directory / callable), replica routing with a
391
+ read-your-writes override (pins reads to the primary for the scope — not lag-aware replica
392
+ tracking), and LRU engine eviction bounded by `max_engines` so a dynamic or unbounded shard-key
393
+ space still keeps a bounded number of live connections. No cross-shard transactions — pair with
394
+ an outbox/saga pattern for multi-shard writes.
395
+
396
+ ## Building blocks, batteries included
397
+
398
+ | Building block | Job |
399
+ |---|---|
400
+ | `RetryConfig` | Idempotency-gated retries with exponential backoff + jitter and a hard total-time ceiling |
401
+ | `CircuitBreaker` | Per database×shard×role blast-radius containment against sustained infra failures |
402
+ | `ConcurrencyLimiter` | Independent semaphore tiers (reads/writes/bulk/expensive) acquired before pool checkout |
403
+ | `db.stream()` | Server-side-cursor streaming — bounded memory regardless of result-set size (measured, not assumed) |
404
+ | `insert_many` / `upsert_many` | Adaptive-batch bulk writes, `max_payload_bytes`-aware |
405
+ | `unnest()` bulk strategy | Array-parameter bulk insert/upsert — ~29× a naive `execute_many` loop |
406
+ | `db.copy_records` | PostgreSQL COPY for the largest ingest jobs |
407
+ | Transactional inbox | Effectively-once consumer processing — dedup and business write share one transaction |
408
+ | PgBouncer-compatible pooling | A mode for transaction-pooling PgBouncer deployments |
409
+ | psycopg pipeline mode | A raw escape hatch for pipelined round-trips, opt-in |
410
+ | `PrometheusMetrics` / OTel metrics & tracing | Structured observability, cost measured per adapter |
411
+
412
+ ## Operate it from the terminal
413
+
414
+ ```bash
415
+ pip install "dbkit-sql[cli]"
416
+ dbkit config-validate config.yaml # schema + connection-budget validation
417
+ dbkit check config.yaml # validate + full readiness check (TLS, budget, connectivity)
418
+ dbkit pools config.yaml # warm a connection, print pool status
419
+ dbkit engines config.yaml # live engine registry snapshot
420
+ dbkit connection-budget config.yaml # projected cluster-wide connection budget
421
+ dbkit metrics config.yaml # one-shot Prometheus text-format snapshot
422
+ ```
423
+
424
+ See [`docs/cli.md`](docs/cli.md) for the full command reference.
425
+
426
+ ## Where dbkit fits
427
+
428
+ dbkit sits *above* SQLAlchemy Core — a resilience and ergonomics layer, not a replacement; drop
429
+ to `.raw`/the underlying `Connection` any time. It's for teams that use SQLAlchemy Core (or raw
430
+ SQL) directly and want production-safe database access without rebuilding pooling policy,
431
+ retries, sharding, and observability in every service.
432
+
433
+ **A good fit when:**
434
+
435
+ - PostgreSQL is your primary database and you don't want an ORM
436
+ - a network blip mid-write must never turn into an ambiguous outcome
437
+ - retry and circuit-breaker behavior must be explicit, not implicit in a driver
438
+ - you run more than one database, shard, or read replica
439
+ - operators need visibility into pool/retry/circuit-breaker state
440
+
441
+ **Do NOT use dbkit when:**
442
+
443
+ - **You need an ORM.** Relationship loading, unit-of-work sessions, and a model layer are
444
+ explicitly out of scope — pair a real ORM with dbkit's config/resilience layer instead, or
445
+ use the ORM alone.
446
+ - **You need schema migrations.** Pair dbkit with Alembic or your migration tool of choice.
447
+ - **You need a message-broker client.** The inbox/batching helpers are primitives your own
448
+ consumer loop wires in — dbkit does not talk to RabbitMQ, Kafka, or SQS.
449
+ - **You need exactly-once delivery.** Nothing over PostgreSQL gives you that, including dbkit;
450
+ the idempotency gate gives idempotent *processing*, a different (weaker, honest) guarantee.
451
+ - **You need dialects beyond PostgreSQL/psycopg/asyncpg covered by CI.** Other
452
+ SQLAlchemy-supported dialects may work, but aren't tested here.
453
+
454
+ ## Performance
455
+
456
+ Measured against real PostgreSQL, not estimated — see the benchmark scripts under
457
+ `benchmarks/` for methodology and to reproduce:
458
+
459
+ | Metric | Result | Script |
460
+ |---|---|---|
461
+ | Concurrency scaling (simple indexed read) | Plateaus ~3.6–3.8k ops/s at concurrency ≥ 10, regardless of pool capacity (15 vs. 100) | `bench_concurrency_scaling.py` |
462
+ | Overhead vs. raw SQLAlchemy Core | ~19–22% (session-measured), gated at 40% in CI | `bench_overhead.py`, `check_regression.py` |
463
+ | `unnest()` bulk insert vs. `execute_many` | ~29× at steady state, 20,000 rows | `bench_unnest.py` |
464
+ | OTel tracing / Prometheus / OTel metrics overhead | +3.2% / +5.6% / +0.5% p50 vs. no observability | `bench_observability_overhead.py` |
465
+ | Streaming 1M–5M rows, narrow + wide | RSS growth bounded to +3.6MB regardless of row count | `bench_streaming_scale.py` |
466
+ | Connection-budget formula vs. real processes | 4 real OS processes at full pool utilization matched the predicted connection count exactly | `test_multiprocess_connection_budget.py` |
467
+
468
+ Absolute numbers vary by machine — the ratios and the "bounded, not unbounded" shape are the
469
+ stable part. What isn't yet measured: a real multi-node production topology and multi-day
470
+ sustained load — every number above comes from a single-node test instance.
471
+
472
+ ## Migrating from raw SQLAlchemy Core
473
+
474
+ dbkit wraps `create_async_engine`/`create_engine`, so migration is incremental, not a rewrite:
475
+
476
+ - `engine.connect()` + hand-rolled retry loop → `AsyncDatabase.fetch_*`/`execute` with
477
+ `RetryConfig` — the retry loop, backoff, and idempotency gate disappear from your code.
478
+ - Manual `try/except` on driver exceptions → classified `DatabaseError` subclasses
479
+ (`DatabaseDeadlockError`, `DatabaseUniqueViolationError`, …) — catch what actually happened.
480
+ - A single hard-coded `engine` per database → named databases, shard resolvers, and replica
481
+ routing via one `DbkitConfig`.
482
+ - Raw `conn.execute(text(...))` still works via `.raw` — dbkit is additive, not a cage.
483
+
484
+ ## Examples
485
+
486
+ **[`examples/`](examples/)** — 16 runnable, idempotent scripts covering every feature, plus a
487
+ `run_all.py` that runs them all safely in sequence:
488
+
489
+ | Want to… | Example |
490
+ |---|---|
491
+ | See the smallest working consumer | [`quickstart_async.py`](examples/quickstart_async.py) · [`quickstart_sync.py`](examples/quickstart_sync.py) |
492
+ | Use explicit transactions & savepoints | [`transactions_savepoints.py`](examples/transactions_savepoints.py) |
493
+ | Handle classified errors | [`error_handling.py`](examples/error_handling.py) |
494
+ | Wire retries + the circuit breaker | [`retries_and_circuit_breaker.py`](examples/retries_and_circuit_breaker.py) |
495
+ | Stream millions of rows | [`streaming.py`](examples/streaming.py) · [`streaming_checkpoint_resume.py`](examples/streaming_checkpoint_resume.py) |
496
+ | Bulk insert/upsert and COPY | [`bulk_insert_upsert.py`](examples/bulk_insert_upsert.py) · [`copy_ingest.py`](examples/copy_ingest.py) |
497
+ | Effectively-once consumer processing | [`inbox_idempotent_consumer.py`](examples/inbox_idempotent_consumer.py) · [`batch_collector.py`](examples/batch_collector.py) |
498
+ | Check health/pool state | [`health_and_pool.py`](examples/health_and_pool.py) |
499
+ | Wire OpenTelemetry | [`opentelemetry_observability.py`](examples/opentelemetry_observability.py) |
500
+ | Confirm sync/async parity | [`sync_feature_parity.py`](examples/sync_feature_parity.py) |
501
+
502
+ ```bash
503
+ export DBKIT_DSN=postgresql+psycopg://localhost/postgres
504
+ python examples/quickstart_async.py
505
+ python examples/run_all.py # runs every example, safe to repeat
506
+ ```
507
+
508
+ ## Architecture
509
+
510
+ ```
511
+ dbkit/
512
+ _core/ # config, query/routing, errors, policies, bulk, circuit — pure logic, no I/O
513
+ _async/ # hand-written async runtime (database, connection, transaction, engine, resilience)
514
+ _sync/ # generated from _async/ via tools/run_unasync.py — never edited by hand
515
+ observability/ # structured logging, Prometheus, OpenTelemetry metrics/tracing
516
+ postgres/ # COPY, pipeline mode, unnest() bulk strategy — Postgres-only escape hatches
517
+ integrations/ # transactional inbox (dedup consumer), BatchCollector micro-batching
518
+ cli/ # config-validate, check, pools, engines, connection-budget, metrics
519
+ ```
520
+
521
+ The shared `_core/` has **zero** I/O — error classification, config, routing, and policy
522
+ decisions are pure and shared by both the sync and async frontends.
523
+
524
+ ## Compatibility
525
+
526
+ Python 3.11+ (tested: 3.11 / 3.12 / 3.13) · PostgreSQL 16 in CI · SQLAlchemy 2.0.30+ ·
527
+ `psycopg[binary] >= 3.2` (default driver, the only one with a sync API — COPY and pipeline mode
528
+ also require it) · `asyncpg >= 0.29` (async-only, CI-covered for reads/writes/transactions,
529
+ resilience/chaos, sharding/replicas, and the CLI; no COPY/pipeline mode).
530
+
531
+ ## Documentation
532
+
533
+ **📚 Full rendered docs: [talaatmagdyx.github.io/dbkit-sql](https://talaatmagdyx.github.io/dbkit-sql/)**
534
+
535
+ - [API reference](docs/api/) — `Database`/`AsyncDatabase`, config, query/routing, errors
536
+ - [CLI command reference](docs/cli.md)
537
+ - [Dashboards & alert rules](docs/observability.md) (Grafana/Prometheus)
538
+ - [Security posture](docs/security.md) — redaction, TLS, connection budgets, idempotency lint
539
+ - [Troubleshooting guide](docs/troubleshooting.md)
540
+ - [Testing, chaos, and benchmark commands](docs/testing.md)
541
+ - [Full spec](docs/requirements.md) and [phased delivery roadmap](docs/roadmap.md)
542
+ - [Versioning & deprecation policy](docs/versioning.md)
543
+
544
+ ## Contributing & security
545
+
546
+ See [`CONTRIBUTING.md`](CONTRIBUTING.md) for the ground rules (no ORM, never hand-edit
547
+ `_sync/`, pure logic stays in `_core/`) and [`docs/testing.md`](docs/testing.md) for the local
548
+ quality gates (`ruff`, `mypy --strict`, the full test/integration suite) to run before opening a
549
+ PR. Found a real vulnerability? **Do not open a public issue** — follow
550
+ [`SECURITY.md`](SECURITY.md) to report it privately. See [`docs/security.md`](docs/security.md)
551
+ for the documented security posture.
552
+
553
+ ## License
554
+
555
+ [MIT](LICENSE)
@@ -0,0 +1,59 @@
1
+ dbkit/__init__.py,sha256=hBRiVQzdWxTQY3vtzNsMqpchZYRyrbIZI7KDoeXp3Io,1728
2
+ dbkit/_pool.py,sha256=EiOMtxIsMvucUmPU0wGZQfJxgAxK-zdX382RmBLKSuA,7673
3
+ dbkit/errors.py,sha256=Gu2s0fT52TTq1WMW2m5Uxcej_AoQTmzVfrSMv9s7Nhs,207
4
+ dbkit/_async/__init__.py,sha256=HKzdMZhGVZ9wdQFaNCa-llfjwMAwis66OYNRE7Xyc5c,471
5
+ dbkit/_async/_compat.py,sha256=bvJWXvhKe-92g4ke4v8kepQ8QykZT8EHGsvsRXs0LLM,4472
6
+ dbkit/_async/connection.py,sha256=tDcPbmhoqkrDGF2hwfYYPao08MRwn7Y_y4TAIBisOYo,13507
7
+ dbkit/_async/database.py,sha256=LKcv2dEKD6Io2lm7pMiqFl2DI0oL9nz3CLFaxBSEl8o,35524
8
+ dbkit/_async/engine.py,sha256=AhpRwYvpGkxFMvdoWi2OaRHzSV8DsfIccLvLu0ABSCM,9809
9
+ dbkit/_async/executor.py,sha256=e1VMBxhefMZo7gKzMseBQdka3Tu3XZAa_791w1857-E,14339
10
+ dbkit/_async/health.py,sha256=KTjGJ_f2RLLmnopUGTUpfKdq3_KIlYUtmM6fkTXenKQ,1533
11
+ dbkit/_async/resilience.py,sha256=Rti3BkV0sSue8hbTsaiJbx6YA3PXJ34CCd8ImCJZcxA,5987
12
+ dbkit/_async/streaming.py,sha256=EjanXAUOCRhXgwohXHrjd7noXr19ZKjNoZFwmIOEJ2w,5093
13
+ dbkit/_async/transaction.py,sha256=O5sLoW1kbQTTX_PizU4sChlXr2u8LSGypF4k302DHe8,10642
14
+ dbkit/_core/__init__.py,sha256=OQD-7nAVm6u0rxe-wzKLrA6qmLQqtfAAWCEtIe-eJHk,73
15
+ dbkit/_core/bulk.py,sha256=5MIHLycFDz0cOCrSv6rA4vkRBMejHV47v6Nyjszeq1s,3681
16
+ dbkit/_core/circuit.py,sha256=R6WecYMhn0WvT9yTKJyIqaJcqvRtsSVEf6SfBqaRb6o,4599
17
+ dbkit/_core/config.py,sha256=FcDuZZesCQezcZPt3i84-WAIYNE5UL4pIFU9fgQ4UIE,25839
18
+ dbkit/_core/idempotency_lint.py,sha256=PaHOsTqL4foI_dVAkawfrrCt8B3C2KjcaU4ztSsHnH0,2830
19
+ dbkit/_core/keys.py,sha256=4NAHi4gG4IGNAPgM5ggmiXBzstzuZ6kSsDObv3ync-o,872
20
+ dbkit/_core/policies.py,sha256=Xn5t5a0En0vsnmGwC3AFzwKRNP8QtcWx_Oaz5N50pfA,3546
21
+ dbkit/_core/query.py,sha256=1hp8l7HkCaZom86BYri8yN6GkMnYrx5Q7-lk_BBLfoo,6049
22
+ dbkit/_core/result.py,sha256=qMxmNweX8WIocC-vhmGV-xyHtIuLaus-hb32ycFRrC8,3724
23
+ dbkit/_core/routing.py,sha256=932OyNhYu_swl4X1W7m1vx6toR0PvunyZ29BJHc8SLQ,9954
24
+ dbkit/_core/errors/__init__.py,sha256=ROPwrJl9zFRIfxJLVgD2eVvVVinQJ1cc31G33UCL1wY,2161
25
+ dbkit/_core/errors/base.py,sha256=J1AZ01-R1YWigbTF3AgS0QxOjJwQx_DPD93HpQBRCh4,10887
26
+ dbkit/_core/errors/classifier.py,sha256=-I3wIuMBKXC3ANGxlVdOV1fSpksUoHcDSEVa2izG-ts,5178
27
+ dbkit/_core/errors/redaction.py,sha256=I_DnwWtuy666fPZ0YVk2ZbkiqnHthMSn7EiyzLYSF6g,3158
28
+ dbkit/_core/errors/sqlstate.py,sha256=Zjyssgc73tm8lisMAbjWhZjW4oVg3ysXYTFxjbo-slA,3349
29
+ dbkit/_sync/__init__.py,sha256=cYtbdf8nN5YbpcARV-FaEEPVkiYe2Vk2t6AM7TEmDJ0,584
30
+ dbkit/_sync/_compat.py,sha256=4TYwH8gVdw3j7xb9mvAphVO6LH_djZ8mhP4Mv444Jks,3115
31
+ dbkit/_sync/connection.py,sha256=SWu9Ehwrat7ccvrgV8sa2aCcVoECiUerwiE3geD6k5M,13333
32
+ dbkit/_sync/database.py,sha256=04mHcwACoJf1M7JSFh5dhNdRIzp_OCuikeP00BHxCqU,35158
33
+ dbkit/_sync/engine.py,sha256=Kroa6fMkxaWxJ7NyTWlvfW1GhQT696po8_avZVy7yW4,9839
34
+ dbkit/_sync/executor.py,sha256=nLH1TsNdUFFtWyNQW5bfFewdQyrj2WB725MjodemDWQ,14249
35
+ dbkit/_sync/health.py,sha256=Y99Zd_kYNhRkIo9slq0Z0c-QdbMyH4suJAlikmBX1Yc,1639
36
+ dbkit/_sync/resilience.py,sha256=NctTcZf0SWIGMgjErUy3XZ2dk8pvFyjIh_1Ut6JqzWs,6105
37
+ dbkit/_sync/streaming.py,sha256=fHo44OOEpExRM7wfVU-apJa0MHCnrrgIW3x0b5V9wQ8,5114
38
+ dbkit/_sync/transaction.py,sha256=33VfeYdwhnz6eBCiFuGhpNzLuwSSStsSZE1_8LkezbI,10496
39
+ dbkit/cli/__init__.py,sha256=uxJm6uDiwk-B0ENvXlvWTutiIV2HwBsB73fTKeDrShs,56
40
+ dbkit/cli/main.py,sha256=PcNkm1zTZIeOYk5FtTJNC0OaJKXyYzhxCdkfzSlY8iY,9616
41
+ dbkit/integrations/__init__.py,sha256=2UorbYW-C3fqDB1nHtXC8mDMi8yyuT8Y9PxQ394zsJ4,458
42
+ dbkit/integrations/batch.py,sha256=oGqlrdgRxSPMM3QYPiFqv2pdqcjpgwnpy8x0Ra6YEg4,2958
43
+ dbkit/integrations/inbox.py,sha256=HSXoyqMC1ez08pZkrTWY7LZ-1fh7lAhWYUtQS7WKfBA,6406
44
+ dbkit/observability/__init__.py,sha256=g3mV9HE0z36mhx1Ybuw-SJINvk8hQLyoVH-aFFN7DoU,598
45
+ dbkit/observability/logging.py,sha256=EUf1CgmiQ02ggKWgyOEE5UxZLlslVc2yevmSkr4QZCs,4848
46
+ dbkit/observability/metrics.py,sha256=3-A2MOV1qOCuj0vyR9FrRNuHu2k2DFNd7D2c0MUS5Uo,4084
47
+ dbkit/observability/otel_metrics.py,sha256=TElQhbSjlwCV7c0OKFdxdcwguuDvHfGIgb_vUguAit8,2969
48
+ dbkit/observability/prometheus.py,sha256=5Kts-Lsvo1vT_go1-19lumuT5FZemjIQxFzh06mwr-I,3594
49
+ dbkit/observability/tracing.py,sha256=X-TFrA2vwjigxOa-gMB4EXEONf87xa8PSwgxDUE7-rI,4532
50
+ dbkit/postgres/__init__.py,sha256=NbIaKW7r-IZoVPoyzgFsLOGma7_EazsJbD7i8SfoGTw,61
51
+ dbkit/postgres/copy.py,sha256=uqoQhB_a4rHc7ahB2tlX_7fdUtJGgebwZCWX-PhQU40,2947
52
+ dbkit/postgres/pipeline.py,sha256=Ym4roIt5Tv4RiKDKVxfyDjE8An86VAZCUInHwgz39Gk,2000
53
+ dbkit/postgres/unnest.py,sha256=lSViVkKQ8K-NoxEl2UtFSau697wnMFdBJDBKWbE6ioM,3572
54
+ dbkit/testing/__init__.py,sha256=G8jieUpxIT0OPcmhPCaGMW_7aRJvw8sS4K7Z2o-x0Co,60
55
+ dbkit_sql-0.1.0.dist-info/METADATA,sha256=L2jMhy-OgmbgvvvU6-zn29CoXJJO_CRMx_o0zR2KAk0,27180
56
+ dbkit_sql-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
57
+ dbkit_sql-0.1.0.dist-info/entry_points.txt,sha256=lkcS_gAba07Xq6Tm4xuEGSvEqQEnVUG_1sfsO9L9O6A,46
58
+ dbkit_sql-0.1.0.dist-info/licenses/LICENSE,sha256=HPDJs06q_YVI0ZFnpF12thVUr-dC5pxTow19W6z-YXQ,1075
59
+ dbkit_sql-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.31.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ dbkit = dbkit.cli.main:main
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 dbkit contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.