dbkit-sql 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. dbkit/__init__.py +77 -0
  2. dbkit/_async/__init__.py +17 -0
  3. dbkit/_async/_compat.py +111 -0
  4. dbkit/_async/connection.py +328 -0
  5. dbkit/_async/database.py +884 -0
  6. dbkit/_async/engine.py +232 -0
  7. dbkit/_async/executor.py +329 -0
  8. dbkit/_async/health.py +53 -0
  9. dbkit/_async/resilience.py +148 -0
  10. dbkit/_async/streaming.py +137 -0
  11. dbkit/_async/transaction.py +264 -0
  12. dbkit/_core/__init__.py +1 -0
  13. dbkit/_core/bulk.py +91 -0
  14. dbkit/_core/circuit.py +130 -0
  15. dbkit/_core/config.py +613 -0
  16. dbkit/_core/errors/__init__.py +74 -0
  17. dbkit/_core/errors/base.py +344 -0
  18. dbkit/_core/errors/classifier.py +138 -0
  19. dbkit/_core/errors/redaction.py +96 -0
  20. dbkit/_core/errors/sqlstate.py +74 -0
  21. dbkit/_core/idempotency_lint.py +57 -0
  22. dbkit/_core/keys.py +29 -0
  23. dbkit/_core/policies.py +105 -0
  24. dbkit/_core/query.py +142 -0
  25. dbkit/_core/result.py +109 -0
  26. dbkit/_core/routing.py +248 -0
  27. dbkit/_pool.py +202 -0
  28. dbkit/_sync/__init__.py +20 -0
  29. dbkit/_sync/_compat.py +82 -0
  30. dbkit/_sync/connection.py +331 -0
  31. dbkit/_sync/database.py +887 -0
  32. dbkit/_sync/engine.py +235 -0
  33. dbkit/_sync/executor.py +332 -0
  34. dbkit/_sync/health.py +56 -0
  35. dbkit/_sync/resilience.py +151 -0
  36. dbkit/_sync/streaming.py +140 -0
  37. dbkit/_sync/transaction.py +267 -0
  38. dbkit/cli/__init__.py +1 -0
  39. dbkit/cli/main.py +256 -0
  40. dbkit/errors.py +6 -0
  41. dbkit/integrations/__init__.py +21 -0
  42. dbkit/integrations/batch.py +81 -0
  43. dbkit/integrations/inbox.py +153 -0
  44. dbkit/observability/__init__.py +21 -0
  45. dbkit/observability/logging.py +154 -0
  46. dbkit/observability/metrics.py +110 -0
  47. dbkit/observability/otel_metrics.py +69 -0
  48. dbkit/observability/prometheus.py +78 -0
  49. dbkit/observability/tracing.py +128 -0
  50. dbkit/postgres/__init__.py +1 -0
  51. dbkit/postgres/copy.py +77 -0
  52. dbkit/postgres/pipeline.py +48 -0
  53. dbkit/postgres/unnest.py +73 -0
  54. dbkit/testing/__init__.py +1 -0
  55. dbkit_sql-0.1.0.dist-info/METADATA +555 -0
  56. dbkit_sql-0.1.0.dist-info/RECORD +59 -0
  57. dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
  58. dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
  59. dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,110 @@
1
+ """Metrics protocol and a no-op default; a Prometheus adapter lives behind the extra (§25.1).
2
+
3
+ The core depends only on the :class:`MetricsSink` protocol, so applications can plug in any
4
+ backend. Label sets are restricted to the low-cardinality allowlist in §25.1 — never raw SQL,
5
+ user/tenant/message/request ids, or exception strings.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from collections.abc import Mapping
11
+ from typing import Any, Protocol, runtime_checkable
12
+
13
+ Labels = Mapping[str, str]
14
+
15
+
16
+ @runtime_checkable
17
+ class MetricsSink(Protocol):
18
+ """Minimal metrics surface dbkit emits to."""
19
+
20
+ def incr(self, name: str, value: float = 1.0, labels: Labels | None = None) -> None:
21
+ """Increment a counter metric by ``value``."""
22
+ ...
23
+
24
+ def observe(self, name: str, value: float, labels: Labels | None = None) -> None:
25
+ """Record one observation of a histogram/summary metric."""
26
+ ...
27
+
28
+ def gauge(self, name: str, value: float, labels: Labels | None = None) -> None:
29
+ """Set a gauge metric to its current ``value``."""
30
+ ...
31
+
32
+
33
+ class NoopMetrics:
34
+ """Default sink — does nothing, allocates nothing meaningful on the hot path."""
35
+
36
+ def incr(self, name: str, value: float = 1.0, labels: Labels | None = None) -> None:
37
+ """Does nothing."""
38
+
39
+ def observe(self, name: str, value: float, labels: Labels | None = None) -> None:
40
+ """Does nothing."""
41
+
42
+ def gauge(self, name: str, value: float, labels: Labels | None = None) -> None:
43
+ """Does nothing."""
44
+
45
+
46
+ # Metric names (§25.1). Kept as constants so labels/names stay consistent across the codebase.
47
+ OP_TOTAL = "db_operation_total"
48
+ OP_DURATION = "db_operation_duration_seconds"
49
+ OP_ERRORS = "db_operation_errors_total"
50
+ OP_RETRIES = "db_operation_retries_total"
51
+ OP_TIMEOUTS = "db_operation_timeouts_total"
52
+ OP_CANCELLED = "db_operation_cancelled_total"
53
+
54
+ POOL_SIZE = "db_pool_size"
55
+ POOL_CHECKED_OUT = "db_pool_checked_out"
56
+ POOL_OVERFLOW = "db_pool_overflow"
57
+ POOL_WAIT_SECONDS = "db_pool_wait_seconds"
58
+ POOL_TIMEOUT_TOTAL = "db_pool_timeout_total"
59
+ POOL_INVALIDATIONS = "db_pool_invalidations_total"
60
+ CONN_CREATED = "db_connections_created_total"
61
+ CONN_CLOSED = "db_connections_closed_total"
62
+ CONN_CHECKOUT_DURATION = "db_connection_checkout_duration_seconds"
63
+ CONN_HOLD_DURATION = "db_connection_hold_duration_seconds"
64
+
65
+ TX_TOTAL = "db_transaction_total"
66
+ TX_DURATION = "db_transaction_duration_seconds"
67
+ TX_ROLLBACK = "db_transaction_rollback_total"
68
+ COMMIT_UNKNOWN = "db_commit_unknown_total"
69
+
70
+ #: Current circuit breaker state per db+shard+role: 0=closed, 1=half_open, 2=open (§16).
71
+ CIRCUIT_STATE = "db_circuit_breaker_state"
72
+
73
+ STREAM_ROWS = "db_stream_rows_total"
74
+ STREAM_BYTES = "db_stream_bytes_total"
75
+ BULK_ROWS = "db_bulk_rows_total"
76
+ BULK_BATCH_SIZE = "db_bulk_batch_size"
77
+ #: Rows silently dropped by a best_effort/split_on_failure bulk write (never retried) — the
78
+ #: only metric signal for otherwise-silent data loss under those failure modes (§19.3).
79
+ BULK_ROWS_DROPPED = "db_bulk_rows_dropped_total"
80
+
81
+ #: Labels dbkit is allowed to attach (§25.1). Adapters should reject anything outside this set.
82
+ ALLOWED_LABELS = frozenset(
83
+ {
84
+ "service",
85
+ "environment",
86
+ "database",
87
+ "shard",
88
+ "role",
89
+ "query_name",
90
+ "operation",
91
+ "error_category",
92
+ "retry_attempt",
93
+ "api", # "sync" | "async" — distinguishes the two frontends
94
+ }
95
+ )
96
+
97
+
98
+ def try_prometheus_sink(namespace: str = "dbkit") -> MetricsSink:
99
+ """Construct a Prometheus-backed sink, or raise if ``prometheus_client`` is missing."""
100
+ from .prometheus import PrometheusMetrics
101
+
102
+ return PrometheusMetrics(namespace=namespace)
103
+
104
+
105
+ def try_otel_metrics_sink(meter_name: str = "dbkit", meter_provider: Any = None) -> MetricsSink:
106
+ """Construct an OpenTelemetry Metrics-backed sink, or raise if ``opentelemetry-api`` is
107
+ missing. An alternative to :func:`try_prometheus_sink` — pick one per deployment."""
108
+ from .otel_metrics import OTelMetrics
109
+
110
+ return OTelMetrics(meter_name=meter_name, meter_provider=meter_provider)
@@ -0,0 +1,69 @@
1
+ """OpenTelemetry Metrics adapter for :class:`MetricsSink` (behind the ``otel`` extra).
2
+
3
+ An alternative to the Prometheus adapter: routes the same counters/histograms/gauges through
4
+ ``opentelemetry.metrics`` instead, so they flow to whatever exporter the application's
5
+ ``MeterProvider`` is configured with (OTLP, Prometheus-via-OTel, console, etc.).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any
11
+
12
+ from .metrics import ALLOWED_LABELS, Labels
13
+
14
+
15
+ class OTelMetrics:
16
+ """Lazily create one OTel instrument per metric name via ``opentelemetry.metrics``.
17
+
18
+ Unlike the Prometheus adapter, OTel attributes don't need a fixed pre-declared label set —
19
+ each measurement carries only the labels it was given (filtered against
20
+ :data:`ALLOWED_LABELS`).
21
+ """
22
+
23
+ def __init__(self, *, meter_provider: Any = None, meter_name: str = "dbkit") -> None:
24
+ """Raises ``RuntimeError`` if ``opentelemetry-api`` isn't installed."""
25
+ try:
26
+ from opentelemetry import metrics
27
+ except ImportError as exc: # pragma: no cover
28
+ raise RuntimeError(
29
+ "OTelMetrics requires opentelemetry-api — install dbkit[otel]"
30
+ ) from exc
31
+ self._meter = metrics.get_meter(meter_name, meter_provider=meter_provider)
32
+ self._counters: dict[str, Any] = {}
33
+ self._histograms: dict[str, Any] = {}
34
+ self._gauges: dict[str, Any] = {}
35
+
36
+ @staticmethod
37
+ def _attributes(labels: Labels | None) -> dict[str, str]:
38
+ provided = labels or {}
39
+ bad = set(provided) - ALLOWED_LABELS
40
+ if bad:
41
+ raise ValueError(f"disallowed metric labels: {sorted(bad)}")
42
+ return {k: str(v) for k, v in provided.items()}
43
+
44
+ def _counter(self, name: str) -> Any:
45
+ if name not in self._counters:
46
+ self._counters[name] = self._meter.create_counter(name)
47
+ return self._counters[name]
48
+
49
+ def _histogram(self, name: str) -> Any:
50
+ if name not in self._histograms:
51
+ self._histograms[name] = self._meter.create_histogram(name)
52
+ return self._histograms[name]
53
+
54
+ def _gauge(self, name: str) -> Any:
55
+ if name not in self._gauges:
56
+ self._gauges[name] = self._meter.create_gauge(name)
57
+ return self._gauges[name]
58
+
59
+ def incr(self, name: str, value: float = 1.0, labels: Labels | None = None) -> None:
60
+ """Add ``value`` to the named OTel counter instrument."""
61
+ self._counter(name).add(value, attributes=self._attributes(labels))
62
+
63
+ def observe(self, name: str, value: float, labels: Labels | None = None) -> None:
64
+ """Record ``value`` on the named OTel histogram instrument."""
65
+ self._histogram(name).record(value, attributes=self._attributes(labels))
66
+
67
+ def gauge(self, name: str, value: float, labels: Labels | None = None) -> None:
68
+ """Set the named OTel gauge instrument to ``value``."""
69
+ self._gauge(name).set(value, attributes=self._attributes(labels))
@@ -0,0 +1,78 @@
1
+ """Prometheus adapter for :class:`MetricsSink` (behind the ``[prometheus]`` extra)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ from .metrics import ALLOWED_LABELS, Labels
8
+
9
+
10
+ class PrometheusMetrics:
11
+ """Lazily create Counter/Histogram/Gauge collectors keyed by metric name.
12
+
13
+ Every metric is registered with the full :data:`ALLOWED_LABELS` label set; unspecified
14
+ labels are filled with an empty string so cardinality stays predictable.
15
+ """
16
+
17
+ def __init__(self, namespace: str = "dbkit", registry: Any = None) -> None:
18
+ """Raises ``RuntimeError`` if ``prometheus-client`` isn't installed."""
19
+ try:
20
+ from prometheus_client import REGISTRY, Counter, Gauge, Histogram
21
+ except ImportError as exc: # pragma: no cover
22
+ raise RuntimeError(
23
+ "PrometheusMetrics requires prometheus-client — install dbkit[prometheus]"
24
+ ) from exc
25
+ self._Counter = Counter
26
+ self._Gauge = Gauge
27
+ self._Histogram = Histogram
28
+ self._registry = registry if registry is not None else REGISTRY
29
+ self._namespace = namespace
30
+ self._labelnames = sorted(ALLOWED_LABELS)
31
+ #: Prometheus requires every declared label name to be present in each ``.labels()``
32
+ #: call, so ``_fill()`` can never return fewer than all of them — but rebuilding that
33
+ #: full set from scratch via a per-key ``.get()`` on every single metric emission is
34
+ #: avoidable. This template is built once and merged (a cheap dict-merge) rather than
35
+ #: reconstructed key-by-key per call (performance review §4/§11 Finding #5).
36
+ self._empty_labels: dict[str, str] = dict.fromkeys(self._labelnames, "")
37
+ self._counters: dict[str, Any] = {}
38
+ self._gauges: dict[str, Any] = {}
39
+ self._histograms: dict[str, Any] = {}
40
+
41
+ def _fill(self, labels: Labels | None) -> dict[str, str]:
42
+ if not labels:
43
+ return dict(self._empty_labels)
44
+ bad = set(labels) - ALLOWED_LABELS
45
+ if bad:
46
+ raise ValueError(f"disallowed metric labels: {sorted(bad)}")
47
+ return {**self._empty_labels, **{k: str(v) for k, v in labels.items()}}
48
+
49
+ def _counter(self, name: str) -> Any:
50
+ if name not in self._counters:
51
+ self._counters[name] = self._Counter(
52
+ name, name, self._labelnames, registry=self._registry
53
+ )
54
+ return self._counters[name]
55
+
56
+ def _gauge_metric(self, name: str) -> Any:
57
+ if name not in self._gauges:
58
+ self._gauges[name] = self._Gauge(name, name, self._labelnames, registry=self._registry)
59
+ return self._gauges[name]
60
+
61
+ def _histogram(self, name: str) -> Any:
62
+ if name not in self._histograms:
63
+ self._histograms[name] = self._Histogram(
64
+ name, name, self._labelnames, registry=self._registry
65
+ )
66
+ return self._histograms[name]
67
+
68
+ def incr(self, name: str, value: float = 1.0, labels: Labels | None = None) -> None:
69
+ """Increment the named Prometheus ``Counter``."""
70
+ self._counter(name).labels(**self._fill(labels)).inc(value)
71
+
72
+ def observe(self, name: str, value: float, labels: Labels | None = None) -> None:
73
+ """Record an observation on the named Prometheus ``Histogram``."""
74
+ self._histogram(name).labels(**self._fill(labels)).observe(value)
75
+
76
+ def gauge(self, name: str, value: float, labels: Labels | None = None) -> None:
77
+ """Set the named Prometheus ``Gauge``."""
78
+ self._gauge_metric(name).labels(**self._fill(labels)).set(value)
@@ -0,0 +1,128 @@
1
+ """OpenTelemetry tracing (§25.2), with a graceful no-op fallback.
2
+
3
+ If the ``opentelemetry-api`` package is not installed, or config disables tracing, every
4
+ call becomes a cheap no-op — the facade never needs to check whether tracing is available.
5
+ SQL statement text is never recorded on spans (redaction, §25.2, §29): only the logical query
6
+ name, target metadata, and outcome are attached.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import contextlib
12
+ import importlib.metadata
13
+ from collections.abc import Iterator, Mapping
14
+ from typing import Any
15
+
16
+ try:
17
+ from opentelemetry import trace
18
+
19
+ _OTEL_AVAILABLE = True
20
+ except ImportError: # pragma: no cover - exercised by the no-otel test lane
21
+ _OTEL_AVAILABLE = False
22
+
23
+ try:
24
+ _LIBRARY_VERSION = importlib.metadata.version("dbkit")
25
+ except importlib.metadata.PackageNotFoundError: # pragma: no cover - unbuilt source checkout
26
+ _LIBRARY_VERSION = "0.0.0"
27
+
28
+
29
+ class Tracer:
30
+ """Wraps an OpenTelemetry tracer; every method is a no-op if OTel isn't installed/enabled.
31
+
32
+ ``tracer_provider``/``schema_url``/``attributes`` are the same parameters
33
+ ``opentelemetry.trace.get_tracer`` itself accepts — pass a ``tracer_provider`` to bind to a
34
+ specific (non-global) provider, e.g. per-tenant tracing or test isolation.
35
+ """
36
+
37
+ def __init__(
38
+ self,
39
+ *,
40
+ enabled: bool,
41
+ tracer_provider: Any = None,
42
+ schema_url: str | None = None,
43
+ attributes: Mapping[str, Any] | None = None,
44
+ ) -> None:
45
+ """A no-op tracer if ``enabled`` is False or ``opentelemetry-api`` isn't installed."""
46
+ self._enabled = enabled and _OTEL_AVAILABLE
47
+ self._tracer = (
48
+ trace.get_tracer(
49
+ "dbkit",
50
+ _LIBRARY_VERSION,
51
+ tracer_provider=tracer_provider,
52
+ schema_url=schema_url,
53
+ attributes=attributes,
54
+ )
55
+ if self._enabled
56
+ else None
57
+ )
58
+
59
+ @property
60
+ def enabled(self) -> bool:
61
+ """Whether this tracer actually records spans (False = full no-op)."""
62
+ return self._enabled
63
+
64
+ @contextlib.contextmanager
65
+ def span(
66
+ self,
67
+ name: str,
68
+ *,
69
+ db_system: str = "postgresql",
70
+ operation_type: str | None = None,
71
+ query_name: str | None = None,
72
+ database: str | None = None,
73
+ shard: str | None = None,
74
+ role: str | None = None,
75
+ ) -> Iterator[SpanHandle]:
76
+ """A database operation span (§25.2). Attributes never include SQL text or params."""
77
+ if not self._enabled or self._tracer is None:
78
+ yield SpanHandle(None)
79
+ return
80
+ attributes: dict[str, Any] = {"db.system": db_system}
81
+ if operation_type:
82
+ attributes["db.operation.type"] = operation_type
83
+ if query_name:
84
+ attributes["db.query.name"] = query_name
85
+ if database:
86
+ attributes["db.namespace"] = database
87
+ if shard:
88
+ attributes["db.shard.id"] = shard
89
+ if role:
90
+ attributes["db.target.role"] = role
91
+ # start_as_current_span records exceptions and sets ERROR status automatically when
92
+ # the exception propagates out of this block (record_exception/set_status_on_exception
93
+ # default to True) — no manual handling needed here. kind=CLIENT per the OTel semantic
94
+ # conventions for database client spans (the default is INTERNAL).
95
+ with self._tracer.start_as_current_span(
96
+ name, kind=trace.SpanKind.CLIENT, attributes=attributes
97
+ ) as span:
98
+ yield SpanHandle(span)
99
+
100
+
101
+ class SpanHandle:
102
+ """Thin wrapper so callers can set attributes without checking ``None``/OTel presence."""
103
+
104
+ def __init__(self, span: Any) -> None:
105
+ """Wrap ``span``, or ``None`` for a no-op handle."""
106
+ self._span = span
107
+
108
+ def set_attribute(self, key: str, value: Any) -> None:
109
+ """Set a span attribute; a no-op if this handle wraps no real span."""
110
+ if self._span is not None:
111
+ self._span.set_attribute(key, value)
112
+
113
+
114
+ def make_tracer(
115
+ enabled: bool,
116
+ *,
117
+ tracer_provider: Any = None,
118
+ schema_url: str | None = None,
119
+ attributes: Mapping[str, Any] | None = None,
120
+ ) -> Tracer:
121
+ """Build a :class:`Tracer` (see its docstring for ``tracer_provider``/``schema_url``/
122
+ ``attributes``)."""
123
+ return Tracer(
124
+ enabled=enabled,
125
+ tracer_provider=tracer_provider,
126
+ schema_url=schema_url,
127
+ attributes=attributes,
128
+ )
@@ -0,0 +1 @@
1
+ """dbkit postgres — see docs/roadmap.md (later phases)."""
dbkit/postgres/copy.py ADDED
@@ -0,0 +1,77 @@
1
+ """PostgreSQL COPY via the psycopg3 raw-driver escape hatch (§7.3, §19.2).
2
+
3
+ COPY is the fastest bulk-ingest path. It is PostgreSQL + psycopg specific, so it lives here and
4
+ is reached only through :meth:`AsyncDatabase.copy_records` / :meth:`Database.copy_records`,
5
+ which check the dialect/driver first. Both an async and a sync implementation are provided
6
+ (this module is not unasync-generated); the frontends dispatch via ``_compat``.
7
+
8
+ Memory stays bounded: rows are written one at a time to the COPY stream and never fully
9
+ buffered. The identifiers (table/columns) come from the application, not end users — callers
10
+ must not pass untrusted identifiers (§18.5).
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from collections.abc import AsyncIterable, Iterable, Sequence
16
+ from typing import Any
17
+
18
+ from ..errors import DatabaseUnsupportedOperationError
19
+
20
+
21
+ def _copy_sql(table: str, columns: Sequence[str]) -> str:
22
+ cols = ", ".join(f'"{c}"' for c in columns)
23
+ return f'COPY "{table}" ({cols}) FROM STDIN'
24
+
25
+
26
+ def _require_psycopg(driver_conn: Any) -> None:
27
+ # Both psycopg and asyncpg's raw connections expose a `cursor` method (incompatible
28
+ # signatures/purposes — asyncpg's is for server-side result cursors, not COPY), so `cursor`
29
+ # alone can't distinguish them. `pipeline` is psycopg-only (mirrors postgres/pipeline.py's
30
+ # own check) and reliably identifies a real psycopg connection.
31
+ if not hasattr(driver_conn, "pipeline"):
32
+ raise DatabaseUnsupportedOperationError(
33
+ "COPY requires the psycopg driver; this target uses a different driver"
34
+ )
35
+
36
+
37
+ async def copy_records_async(
38
+ sa_conn: Any,
39
+ table: str,
40
+ columns: Sequence[str],
41
+ records: Iterable[Sequence[Any]] | AsyncIterable[Sequence[Any]],
42
+ ) -> int:
43
+ """Stream ``records`` into ``table`` via COPY on the async psycopg connection. Returns the
44
+ number of rows written."""
45
+ raw = await sa_conn.get_raw_connection()
46
+ driver_conn = raw.driver_connection
47
+ _require_psycopg(driver_conn)
48
+ stmt = _copy_sql(table, columns)
49
+ written = 0
50
+ async with driver_conn.cursor() as cur, cur.copy(stmt) as copy:
51
+ if hasattr(records, "__aiter__"):
52
+ async for row in records:
53
+ await copy.write_row(row)
54
+ written += 1
55
+ else:
56
+ for row in records:
57
+ await copy.write_row(row)
58
+ written += 1
59
+ return written
60
+
61
+
62
+ def copy_records_sync(
63
+ sa_conn: Any,
64
+ table: str,
65
+ columns: Sequence[str],
66
+ records: Iterable[Sequence[Any]],
67
+ ) -> int:
68
+ """Stream ``records`` into ``table`` via COPY on the sync psycopg connection."""
69
+ driver_conn = sa_conn.connection.driver_connection
70
+ _require_psycopg(driver_conn)
71
+ stmt = _copy_sql(table, columns)
72
+ written = 0
73
+ with driver_conn.cursor() as cur, cur.copy(stmt) as copy:
74
+ for row in records:
75
+ copy.write_row(row)
76
+ written += 1
77
+ return written
@@ -0,0 +1,48 @@
1
+ """psycopg3 pipeline mode — batch several dependent statements into one round trip (§7.3).
2
+
3
+ In pipeline mode the client sends statements to the server without waiting for each response
4
+ before sending the next, then results are processed together. This is a raw-driver escape
5
+ hatch (like COPY): PostgreSQL + psycopg only, reached through
6
+ :meth:`AsyncConnectionScope.pipeline` / the sync equivalent, and useful for a handful of
7
+ dependent-but-batchable statements in the same transaction (e.g. a business write plus its
8
+ inbox record, §28).
9
+
10
+ Because SQLAlchemy's cursor execution transparently triggers a pipeline sync whenever a result
11
+ is actually fetched (rowcount, ``RETURNING``, a `SELECT``), ordinary ``tx.execute(...)`` calls
12
+ inside the block still work — the win comes from the driver not waiting for each response
13
+ before sending the next statement, not from changing the calling code.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import contextlib
19
+ from collections.abc import AsyncIterator, Iterator
20
+ from typing import Any
21
+
22
+ from ..errors import DatabaseUnsupportedOperationError
23
+
24
+
25
+ def _require_psycopg(driver_conn: Any) -> None:
26
+ if not hasattr(driver_conn, "pipeline"):
27
+ raise DatabaseUnsupportedOperationError(
28
+ "pipeline mode requires the psycopg driver; this target uses a different driver"
29
+ )
30
+
31
+
32
+ @contextlib.asynccontextmanager
33
+ async def pipeline_scope_async(sa_conn: Any) -> AsyncIterator[None]:
34
+ """Enter psycopg pipeline mode on ``sa_conn``'s raw driver connection (async)."""
35
+ raw = await sa_conn.get_raw_connection()
36
+ driver_conn = raw.driver_connection
37
+ _require_psycopg(driver_conn)
38
+ async with driver_conn.pipeline():
39
+ yield
40
+
41
+
42
+ @contextlib.contextmanager
43
+ def pipeline_scope_sync(sa_conn: Any) -> Iterator[None]:
44
+ """Enter psycopg pipeline mode on ``sa_conn``'s raw driver connection (sync)."""
45
+ driver_conn = sa_conn.connection.driver_connection
46
+ _require_psycopg(driver_conn)
47
+ with driver_conn.pipeline():
48
+ yield
@@ -0,0 +1,73 @@
1
+ """PostgreSQL ``unnest()`` bulk insert/upsert — a mid-tier strategy between ``execute_many``
2
+ and ``COPY`` (§19).
3
+
4
+ ``execute_many`` binds one parameter *per column per row*, so a batch is capped by
5
+ PostgreSQL's 65535 bind-parameter ceiling divided by column count. ``unnest`` instead binds
6
+ one parameter *per column* — each holding the full array of that column's values for the
7
+ batch — so batch size is bounded only by memory/payload, not the parameter ceiling. It sends
8
+ exactly one statement per batch, like ``execute_many``, but without that per-row multiplication.
9
+
10
+ This is plain SQL executed through the normal connection — unlike COPY, it needs no raw-driver
11
+ escape hatch. psycopg (and most PostgreSQL drivers) adapt a Python list bound to a single
12
+ placeholder into a native PostgreSQL array automatically.
13
+
14
+ Each array parameter is bound with an explicit ``::type[]`` cast (derived from the target
15
+ table's column types) — without it, PostgreSQL cannot resolve which ``unnest`` overload
16
+ applies when several array-typed parameters are passed with an unknown element type
17
+ (``AmbiguousFunction``). This covers ordinary scalar types (int, text, bool, timestamps,
18
+ uuid); composite/enum/array-of-array columns may still need the caller to avoid this strategy.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ from collections.abc import Mapping, Sequence
24
+ from typing import Any
25
+
26
+ from sqlalchemy import TextClause, text
27
+
28
+
29
+ def unnest_insert_sql(
30
+ table: str,
31
+ columns: Sequence[str],
32
+ column_types: Mapping[str, str],
33
+ *,
34
+ conflict_index_elements: Sequence[str] | None = None,
35
+ update_columns: Sequence[str] | None = None,
36
+ ) -> TextClause:
37
+ """Build ``INSERT ... SELECT * FROM unnest(:c1::t1[], :c2::t2[], ...)``, optionally with
38
+ ``ON CONFLICT``.
39
+
40
+ ``column_types`` maps each column name to its PostgreSQL type (e.g. ``"INTEGER"``,
41
+ ``"TEXT"``) — every column needs an explicit array cast, or PostgreSQL cannot resolve
42
+ which ``unnest`` overload applies when several unknown-typed array parameters are passed
43
+ (``AmbiguousFunction``).
44
+
45
+ ``conflict_index_elements=None`` builds a plain insert. Given index elements with
46
+ ``update_columns=None`` builds ``ON CONFLICT (...) DO NOTHING``; with columns given,
47
+ ``ON CONFLICT (...) DO UPDATE SET col = EXCLUDED.col`` for each.
48
+ """
49
+ col_list = ", ".join(f'"{c}"' for c in columns)
50
+ # CAST(:name AS type[]), not the ":name::type[]" shorthand — SQLAlchemy's text() bind-param
51
+ # regex does not recognize a name immediately followed by "::" and silently leaves it
52
+ # unbound, which the driver then rejects as invalid syntax.
53
+ unnest_args = ", ".join(f"CAST(:{c} AS {column_types[c]}[])" for c in columns)
54
+ alias_cols = ", ".join(f'"{c}"' for c in columns)
55
+ sql = (
56
+ f'INSERT INTO "{table}" ({col_list}) '
57
+ f"SELECT * FROM unnest({unnest_args}) AS _dbkit_unnest({alias_cols})"
58
+ )
59
+ if conflict_index_elements is not None:
60
+ idx = ", ".join(f'"{c}"' for c in conflict_index_elements)
61
+ if update_columns is None:
62
+ sql += f" ON CONFLICT ({idx}) DO NOTHING"
63
+ else:
64
+ set_clause = ", ".join(f'"{c}" = EXCLUDED."{c}"' for c in update_columns)
65
+ sql += f" ON CONFLICT ({idx}) DO UPDATE SET {set_clause}"
66
+ return text(sql)
67
+
68
+
69
+ def columnar_params(
70
+ rows: Sequence[Mapping[str, Any]], columns: Sequence[str]
71
+ ) -> dict[str, list[Any]]:
72
+ """Transpose row-dicts into ``{column: [value, value, ...]}`` — one array bind per column."""
73
+ return {c: [row[c] for row in rows] for c in columns}
@@ -0,0 +1 @@
1
+ """dbkit testing — see docs/roadmap.md (later phases)."""