dbkit-sql 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dbkit/__init__.py +77 -0
- dbkit/_async/__init__.py +17 -0
- dbkit/_async/_compat.py +111 -0
- dbkit/_async/connection.py +328 -0
- dbkit/_async/database.py +884 -0
- dbkit/_async/engine.py +232 -0
- dbkit/_async/executor.py +329 -0
- dbkit/_async/health.py +53 -0
- dbkit/_async/resilience.py +148 -0
- dbkit/_async/streaming.py +137 -0
- dbkit/_async/transaction.py +264 -0
- dbkit/_core/__init__.py +1 -0
- dbkit/_core/bulk.py +91 -0
- dbkit/_core/circuit.py +130 -0
- dbkit/_core/config.py +613 -0
- dbkit/_core/errors/__init__.py +74 -0
- dbkit/_core/errors/base.py +344 -0
- dbkit/_core/errors/classifier.py +138 -0
- dbkit/_core/errors/redaction.py +96 -0
- dbkit/_core/errors/sqlstate.py +74 -0
- dbkit/_core/idempotency_lint.py +57 -0
- dbkit/_core/keys.py +29 -0
- dbkit/_core/policies.py +105 -0
- dbkit/_core/query.py +142 -0
- dbkit/_core/result.py +109 -0
- dbkit/_core/routing.py +248 -0
- dbkit/_pool.py +202 -0
- dbkit/_sync/__init__.py +20 -0
- dbkit/_sync/_compat.py +82 -0
- dbkit/_sync/connection.py +331 -0
- dbkit/_sync/database.py +887 -0
- dbkit/_sync/engine.py +235 -0
- dbkit/_sync/executor.py +332 -0
- dbkit/_sync/health.py +56 -0
- dbkit/_sync/resilience.py +151 -0
- dbkit/_sync/streaming.py +140 -0
- dbkit/_sync/transaction.py +267 -0
- dbkit/cli/__init__.py +1 -0
- dbkit/cli/main.py +256 -0
- dbkit/errors.py +6 -0
- dbkit/integrations/__init__.py +21 -0
- dbkit/integrations/batch.py +81 -0
- dbkit/integrations/inbox.py +153 -0
- dbkit/observability/__init__.py +21 -0
- dbkit/observability/logging.py +154 -0
- dbkit/observability/metrics.py +110 -0
- dbkit/observability/otel_metrics.py +69 -0
- dbkit/observability/prometheus.py +78 -0
- dbkit/observability/tracing.py +128 -0
- dbkit/postgres/__init__.py +1 -0
- dbkit/postgres/copy.py +77 -0
- dbkit/postgres/pipeline.py +48 -0
- dbkit/postgres/unnest.py +73 -0
- dbkit/testing/__init__.py +1 -0
- dbkit_sql-0.1.0.dist-info/METADATA +555 -0
- dbkit_sql-0.1.0.dist-info/RECORD +59 -0
- dbkit_sql-0.1.0.dist-info/WHEEL +4 -0
- dbkit_sql-0.1.0.dist-info/entry_points.txt +2 -0
- dbkit_sql-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Metrics protocol and a no-op default; a Prometheus adapter lives behind the extra (§25.1).
|
|
2
|
+
|
|
3
|
+
The core depends only on the :class:`MetricsSink` protocol, so applications can plug in any
|
|
4
|
+
backend. Label sets are restricted to the low-cardinality allowlist in §25.1 — never raw SQL,
|
|
5
|
+
user/tenant/message/request ids, or exception strings.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Mapping
|
|
11
|
+
from typing import Any, Protocol, runtime_checkable
|
|
12
|
+
|
|
13
|
+
Labels = Mapping[str, str]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@runtime_checkable
|
|
17
|
+
class MetricsSink(Protocol):
|
|
18
|
+
"""Minimal metrics surface dbkit emits to."""
|
|
19
|
+
|
|
20
|
+
def incr(self, name: str, value: float = 1.0, labels: Labels | None = None) -> None:
|
|
21
|
+
"""Increment a counter metric by ``value``."""
|
|
22
|
+
...
|
|
23
|
+
|
|
24
|
+
def observe(self, name: str, value: float, labels: Labels | None = None) -> None:
|
|
25
|
+
"""Record one observation of a histogram/summary metric."""
|
|
26
|
+
...
|
|
27
|
+
|
|
28
|
+
def gauge(self, name: str, value: float, labels: Labels | None = None) -> None:
|
|
29
|
+
"""Set a gauge metric to its current ``value``."""
|
|
30
|
+
...
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class NoopMetrics:
|
|
34
|
+
"""Default sink — does nothing, allocates nothing meaningful on the hot path."""
|
|
35
|
+
|
|
36
|
+
def incr(self, name: str, value: float = 1.0, labels: Labels | None = None) -> None:
|
|
37
|
+
"""Does nothing."""
|
|
38
|
+
|
|
39
|
+
def observe(self, name: str, value: float, labels: Labels | None = None) -> None:
|
|
40
|
+
"""Does nothing."""
|
|
41
|
+
|
|
42
|
+
def gauge(self, name: str, value: float, labels: Labels | None = None) -> None:
|
|
43
|
+
"""Does nothing."""
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# Metric names (§25.1). Kept as constants so labels/names stay consistent across the codebase.
|
|
47
|
+
OP_TOTAL = "db_operation_total"
|
|
48
|
+
OP_DURATION = "db_operation_duration_seconds"
|
|
49
|
+
OP_ERRORS = "db_operation_errors_total"
|
|
50
|
+
OP_RETRIES = "db_operation_retries_total"
|
|
51
|
+
OP_TIMEOUTS = "db_operation_timeouts_total"
|
|
52
|
+
OP_CANCELLED = "db_operation_cancelled_total"
|
|
53
|
+
|
|
54
|
+
POOL_SIZE = "db_pool_size"
|
|
55
|
+
POOL_CHECKED_OUT = "db_pool_checked_out"
|
|
56
|
+
POOL_OVERFLOW = "db_pool_overflow"
|
|
57
|
+
POOL_WAIT_SECONDS = "db_pool_wait_seconds"
|
|
58
|
+
POOL_TIMEOUT_TOTAL = "db_pool_timeout_total"
|
|
59
|
+
POOL_INVALIDATIONS = "db_pool_invalidations_total"
|
|
60
|
+
CONN_CREATED = "db_connections_created_total"
|
|
61
|
+
CONN_CLOSED = "db_connections_closed_total"
|
|
62
|
+
CONN_CHECKOUT_DURATION = "db_connection_checkout_duration_seconds"
|
|
63
|
+
CONN_HOLD_DURATION = "db_connection_hold_duration_seconds"
|
|
64
|
+
|
|
65
|
+
TX_TOTAL = "db_transaction_total"
|
|
66
|
+
TX_DURATION = "db_transaction_duration_seconds"
|
|
67
|
+
TX_ROLLBACK = "db_transaction_rollback_total"
|
|
68
|
+
COMMIT_UNKNOWN = "db_commit_unknown_total"
|
|
69
|
+
|
|
70
|
+
#: Current circuit breaker state per db+shard+role: 0=closed, 1=half_open, 2=open (§16).
|
|
71
|
+
CIRCUIT_STATE = "db_circuit_breaker_state"
|
|
72
|
+
|
|
73
|
+
STREAM_ROWS = "db_stream_rows_total"
|
|
74
|
+
STREAM_BYTES = "db_stream_bytes_total"
|
|
75
|
+
BULK_ROWS = "db_bulk_rows_total"
|
|
76
|
+
BULK_BATCH_SIZE = "db_bulk_batch_size"
|
|
77
|
+
#: Rows silently dropped by a best_effort/split_on_failure bulk write (never retried) — the
|
|
78
|
+
#: only metric signal for otherwise-silent data loss under those failure modes (§19.3).
|
|
79
|
+
BULK_ROWS_DROPPED = "db_bulk_rows_dropped_total"
|
|
80
|
+
|
|
81
|
+
#: Labels dbkit is allowed to attach (§25.1). Adapters should reject anything outside this set.
|
|
82
|
+
ALLOWED_LABELS = frozenset(
|
|
83
|
+
{
|
|
84
|
+
"service",
|
|
85
|
+
"environment",
|
|
86
|
+
"database",
|
|
87
|
+
"shard",
|
|
88
|
+
"role",
|
|
89
|
+
"query_name",
|
|
90
|
+
"operation",
|
|
91
|
+
"error_category",
|
|
92
|
+
"retry_attempt",
|
|
93
|
+
"api", # "sync" | "async" — distinguishes the two frontends
|
|
94
|
+
}
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def try_prometheus_sink(namespace: str = "dbkit") -> MetricsSink:
|
|
99
|
+
"""Construct a Prometheus-backed sink, or raise if ``prometheus_client`` is missing."""
|
|
100
|
+
from .prometheus import PrometheusMetrics
|
|
101
|
+
|
|
102
|
+
return PrometheusMetrics(namespace=namespace)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def try_otel_metrics_sink(meter_name: str = "dbkit", meter_provider: Any = None) -> MetricsSink:
|
|
106
|
+
"""Construct an OpenTelemetry Metrics-backed sink, or raise if ``opentelemetry-api`` is
|
|
107
|
+
missing. An alternative to :func:`try_prometheus_sink` — pick one per deployment."""
|
|
108
|
+
from .otel_metrics import OTelMetrics
|
|
109
|
+
|
|
110
|
+
return OTelMetrics(meter_name=meter_name, meter_provider=meter_provider)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""OpenTelemetry Metrics adapter for :class:`MetricsSink` (behind the ``otel`` extra).
|
|
2
|
+
|
|
3
|
+
An alternative to the Prometheus adapter: routes the same counters/histograms/gauges through
|
|
4
|
+
``opentelemetry.metrics`` instead, so they flow to whatever exporter the application's
|
|
5
|
+
``MeterProvider`` is configured with (OTLP, Prometheus-via-OTel, console, etc.).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .metrics import ALLOWED_LABELS, Labels
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class OTelMetrics:
|
|
16
|
+
"""Lazily create one OTel instrument per metric name via ``opentelemetry.metrics``.
|
|
17
|
+
|
|
18
|
+
Unlike the Prometheus adapter, OTel attributes don't need a fixed pre-declared label set —
|
|
19
|
+
each measurement carries only the labels it was given (filtered against
|
|
20
|
+
:data:`ALLOWED_LABELS`).
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def __init__(self, *, meter_provider: Any = None, meter_name: str = "dbkit") -> None:
|
|
24
|
+
"""Raises ``RuntimeError`` if ``opentelemetry-api`` isn't installed."""
|
|
25
|
+
try:
|
|
26
|
+
from opentelemetry import metrics
|
|
27
|
+
except ImportError as exc: # pragma: no cover
|
|
28
|
+
raise RuntimeError(
|
|
29
|
+
"OTelMetrics requires opentelemetry-api — install dbkit[otel]"
|
|
30
|
+
) from exc
|
|
31
|
+
self._meter = metrics.get_meter(meter_name, meter_provider=meter_provider)
|
|
32
|
+
self._counters: dict[str, Any] = {}
|
|
33
|
+
self._histograms: dict[str, Any] = {}
|
|
34
|
+
self._gauges: dict[str, Any] = {}
|
|
35
|
+
|
|
36
|
+
@staticmethod
|
|
37
|
+
def _attributes(labels: Labels | None) -> dict[str, str]:
|
|
38
|
+
provided = labels or {}
|
|
39
|
+
bad = set(provided) - ALLOWED_LABELS
|
|
40
|
+
if bad:
|
|
41
|
+
raise ValueError(f"disallowed metric labels: {sorted(bad)}")
|
|
42
|
+
return {k: str(v) for k, v in provided.items()}
|
|
43
|
+
|
|
44
|
+
def _counter(self, name: str) -> Any:
|
|
45
|
+
if name not in self._counters:
|
|
46
|
+
self._counters[name] = self._meter.create_counter(name)
|
|
47
|
+
return self._counters[name]
|
|
48
|
+
|
|
49
|
+
def _histogram(self, name: str) -> Any:
|
|
50
|
+
if name not in self._histograms:
|
|
51
|
+
self._histograms[name] = self._meter.create_histogram(name)
|
|
52
|
+
return self._histograms[name]
|
|
53
|
+
|
|
54
|
+
def _gauge(self, name: str) -> Any:
|
|
55
|
+
if name not in self._gauges:
|
|
56
|
+
self._gauges[name] = self._meter.create_gauge(name)
|
|
57
|
+
return self._gauges[name]
|
|
58
|
+
|
|
59
|
+
def incr(self, name: str, value: float = 1.0, labels: Labels | None = None) -> None:
|
|
60
|
+
"""Add ``value`` to the named OTel counter instrument."""
|
|
61
|
+
self._counter(name).add(value, attributes=self._attributes(labels))
|
|
62
|
+
|
|
63
|
+
def observe(self, name: str, value: float, labels: Labels | None = None) -> None:
|
|
64
|
+
"""Record ``value`` on the named OTel histogram instrument."""
|
|
65
|
+
self._histogram(name).record(value, attributes=self._attributes(labels))
|
|
66
|
+
|
|
67
|
+
def gauge(self, name: str, value: float, labels: Labels | None = None) -> None:
|
|
68
|
+
"""Set the named OTel gauge instrument to ``value``."""
|
|
69
|
+
self._gauge(name).set(value, attributes=self._attributes(labels))
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Prometheus adapter for :class:`MetricsSink` (behind the ``[prometheus]`` extra)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .metrics import ALLOWED_LABELS, Labels
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class PrometheusMetrics:
|
|
11
|
+
"""Lazily create Counter/Histogram/Gauge collectors keyed by metric name.
|
|
12
|
+
|
|
13
|
+
Every metric is registered with the full :data:`ALLOWED_LABELS` label set; unspecified
|
|
14
|
+
labels are filled with an empty string so cardinality stays predictable.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, namespace: str = "dbkit", registry: Any = None) -> None:
|
|
18
|
+
"""Raises ``RuntimeError`` if ``prometheus-client`` isn't installed."""
|
|
19
|
+
try:
|
|
20
|
+
from prometheus_client import REGISTRY, Counter, Gauge, Histogram
|
|
21
|
+
except ImportError as exc: # pragma: no cover
|
|
22
|
+
raise RuntimeError(
|
|
23
|
+
"PrometheusMetrics requires prometheus-client — install dbkit[prometheus]"
|
|
24
|
+
) from exc
|
|
25
|
+
self._Counter = Counter
|
|
26
|
+
self._Gauge = Gauge
|
|
27
|
+
self._Histogram = Histogram
|
|
28
|
+
self._registry = registry if registry is not None else REGISTRY
|
|
29
|
+
self._namespace = namespace
|
|
30
|
+
self._labelnames = sorted(ALLOWED_LABELS)
|
|
31
|
+
#: Prometheus requires every declared label name to be present in each ``.labels()``
|
|
32
|
+
#: call, so ``_fill()`` can never return fewer than all of them — but rebuilding that
|
|
33
|
+
#: full set from scratch via a per-key ``.get()`` on every single metric emission is
|
|
34
|
+
#: avoidable. This template is built once and merged (a cheap dict-merge) rather than
|
|
35
|
+
#: reconstructed key-by-key per call (performance review §4/§11 Finding #5).
|
|
36
|
+
self._empty_labels: dict[str, str] = dict.fromkeys(self._labelnames, "")
|
|
37
|
+
self._counters: dict[str, Any] = {}
|
|
38
|
+
self._gauges: dict[str, Any] = {}
|
|
39
|
+
self._histograms: dict[str, Any] = {}
|
|
40
|
+
|
|
41
|
+
def _fill(self, labels: Labels | None) -> dict[str, str]:
|
|
42
|
+
if not labels:
|
|
43
|
+
return dict(self._empty_labels)
|
|
44
|
+
bad = set(labels) - ALLOWED_LABELS
|
|
45
|
+
if bad:
|
|
46
|
+
raise ValueError(f"disallowed metric labels: {sorted(bad)}")
|
|
47
|
+
return {**self._empty_labels, **{k: str(v) for k, v in labels.items()}}
|
|
48
|
+
|
|
49
|
+
def _counter(self, name: str) -> Any:
|
|
50
|
+
if name not in self._counters:
|
|
51
|
+
self._counters[name] = self._Counter(
|
|
52
|
+
name, name, self._labelnames, registry=self._registry
|
|
53
|
+
)
|
|
54
|
+
return self._counters[name]
|
|
55
|
+
|
|
56
|
+
def _gauge_metric(self, name: str) -> Any:
|
|
57
|
+
if name not in self._gauges:
|
|
58
|
+
self._gauges[name] = self._Gauge(name, name, self._labelnames, registry=self._registry)
|
|
59
|
+
return self._gauges[name]
|
|
60
|
+
|
|
61
|
+
def _histogram(self, name: str) -> Any:
|
|
62
|
+
if name not in self._histograms:
|
|
63
|
+
self._histograms[name] = self._Histogram(
|
|
64
|
+
name, name, self._labelnames, registry=self._registry
|
|
65
|
+
)
|
|
66
|
+
return self._histograms[name]
|
|
67
|
+
|
|
68
|
+
def incr(self, name: str, value: float = 1.0, labels: Labels | None = None) -> None:
|
|
69
|
+
"""Increment the named Prometheus ``Counter``."""
|
|
70
|
+
self._counter(name).labels(**self._fill(labels)).inc(value)
|
|
71
|
+
|
|
72
|
+
def observe(self, name: str, value: float, labels: Labels | None = None) -> None:
|
|
73
|
+
"""Record an observation on the named Prometheus ``Histogram``."""
|
|
74
|
+
self._histogram(name).labels(**self._fill(labels)).observe(value)
|
|
75
|
+
|
|
76
|
+
def gauge(self, name: str, value: float, labels: Labels | None = None) -> None:
|
|
77
|
+
"""Set the named Prometheus ``Gauge``."""
|
|
78
|
+
self._gauge_metric(name).labels(**self._fill(labels)).set(value)
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""OpenTelemetry tracing (§25.2), with a graceful no-op fallback.
|
|
2
|
+
|
|
3
|
+
If the ``opentelemetry-api`` package is not installed, or config disables tracing, every
|
|
4
|
+
call becomes a cheap no-op — the facade never needs to check whether tracing is available.
|
|
5
|
+
SQL statement text is never recorded on spans (redaction, §25.2, §29): only the logical query
|
|
6
|
+
name, target metadata, and outcome are attached.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import importlib.metadata
|
|
13
|
+
from collections.abc import Iterator, Mapping
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
from opentelemetry import trace
|
|
18
|
+
|
|
19
|
+
_OTEL_AVAILABLE = True
|
|
20
|
+
except ImportError: # pragma: no cover - exercised by the no-otel test lane
|
|
21
|
+
_OTEL_AVAILABLE = False
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
_LIBRARY_VERSION = importlib.metadata.version("dbkit")
|
|
25
|
+
except importlib.metadata.PackageNotFoundError: # pragma: no cover - unbuilt source checkout
|
|
26
|
+
_LIBRARY_VERSION = "0.0.0"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class Tracer:
|
|
30
|
+
"""Wraps an OpenTelemetry tracer; every method is a no-op if OTel isn't installed/enabled.
|
|
31
|
+
|
|
32
|
+
``tracer_provider``/``schema_url``/``attributes`` are the same parameters
|
|
33
|
+
``opentelemetry.trace.get_tracer`` itself accepts — pass a ``tracer_provider`` to bind to a
|
|
34
|
+
specific (non-global) provider, e.g. per-tenant tracing or test isolation.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(
|
|
38
|
+
self,
|
|
39
|
+
*,
|
|
40
|
+
enabled: bool,
|
|
41
|
+
tracer_provider: Any = None,
|
|
42
|
+
schema_url: str | None = None,
|
|
43
|
+
attributes: Mapping[str, Any] | None = None,
|
|
44
|
+
) -> None:
|
|
45
|
+
"""A no-op tracer if ``enabled`` is False or ``opentelemetry-api`` isn't installed."""
|
|
46
|
+
self._enabled = enabled and _OTEL_AVAILABLE
|
|
47
|
+
self._tracer = (
|
|
48
|
+
trace.get_tracer(
|
|
49
|
+
"dbkit",
|
|
50
|
+
_LIBRARY_VERSION,
|
|
51
|
+
tracer_provider=tracer_provider,
|
|
52
|
+
schema_url=schema_url,
|
|
53
|
+
attributes=attributes,
|
|
54
|
+
)
|
|
55
|
+
if self._enabled
|
|
56
|
+
else None
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def enabled(self) -> bool:
|
|
61
|
+
"""Whether this tracer actually records spans (False = full no-op)."""
|
|
62
|
+
return self._enabled
|
|
63
|
+
|
|
64
|
+
@contextlib.contextmanager
|
|
65
|
+
def span(
|
|
66
|
+
self,
|
|
67
|
+
name: str,
|
|
68
|
+
*,
|
|
69
|
+
db_system: str = "postgresql",
|
|
70
|
+
operation_type: str | None = None,
|
|
71
|
+
query_name: str | None = None,
|
|
72
|
+
database: str | None = None,
|
|
73
|
+
shard: str | None = None,
|
|
74
|
+
role: str | None = None,
|
|
75
|
+
) -> Iterator[SpanHandle]:
|
|
76
|
+
"""A database operation span (§25.2). Attributes never include SQL text or params."""
|
|
77
|
+
if not self._enabled or self._tracer is None:
|
|
78
|
+
yield SpanHandle(None)
|
|
79
|
+
return
|
|
80
|
+
attributes: dict[str, Any] = {"db.system": db_system}
|
|
81
|
+
if operation_type:
|
|
82
|
+
attributes["db.operation.type"] = operation_type
|
|
83
|
+
if query_name:
|
|
84
|
+
attributes["db.query.name"] = query_name
|
|
85
|
+
if database:
|
|
86
|
+
attributes["db.namespace"] = database
|
|
87
|
+
if shard:
|
|
88
|
+
attributes["db.shard.id"] = shard
|
|
89
|
+
if role:
|
|
90
|
+
attributes["db.target.role"] = role
|
|
91
|
+
# start_as_current_span records exceptions and sets ERROR status automatically when
|
|
92
|
+
# the exception propagates out of this block (record_exception/set_status_on_exception
|
|
93
|
+
# default to True) — no manual handling needed here. kind=CLIENT per the OTel semantic
|
|
94
|
+
# conventions for database client spans (the default is INTERNAL).
|
|
95
|
+
with self._tracer.start_as_current_span(
|
|
96
|
+
name, kind=trace.SpanKind.CLIENT, attributes=attributes
|
|
97
|
+
) as span:
|
|
98
|
+
yield SpanHandle(span)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class SpanHandle:
|
|
102
|
+
"""Thin wrapper so callers can set attributes without checking ``None``/OTel presence."""
|
|
103
|
+
|
|
104
|
+
def __init__(self, span: Any) -> None:
|
|
105
|
+
"""Wrap ``span``, or ``None`` for a no-op handle."""
|
|
106
|
+
self._span = span
|
|
107
|
+
|
|
108
|
+
def set_attribute(self, key: str, value: Any) -> None:
|
|
109
|
+
"""Set a span attribute; a no-op if this handle wraps no real span."""
|
|
110
|
+
if self._span is not None:
|
|
111
|
+
self._span.set_attribute(key, value)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def make_tracer(
|
|
115
|
+
enabled: bool,
|
|
116
|
+
*,
|
|
117
|
+
tracer_provider: Any = None,
|
|
118
|
+
schema_url: str | None = None,
|
|
119
|
+
attributes: Mapping[str, Any] | None = None,
|
|
120
|
+
) -> Tracer:
|
|
121
|
+
"""Build a :class:`Tracer` (see its docstring for ``tracer_provider``/``schema_url``/
|
|
122
|
+
``attributes``)."""
|
|
123
|
+
return Tracer(
|
|
124
|
+
enabled=enabled,
|
|
125
|
+
tracer_provider=tracer_provider,
|
|
126
|
+
schema_url=schema_url,
|
|
127
|
+
attributes=attributes,
|
|
128
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""dbkit postgres — see docs/roadmap.md (later phases)."""
|
dbkit/postgres/copy.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""PostgreSQL COPY via the psycopg3 raw-driver escape hatch (§7.3, §19.2).
|
|
2
|
+
|
|
3
|
+
COPY is the fastest bulk-ingest path. It is PostgreSQL + psycopg specific, so it lives here and
|
|
4
|
+
is reached only through :meth:`AsyncDatabase.copy_records` / :meth:`Database.copy_records`,
|
|
5
|
+
which check the dialect/driver first. Both an async and a sync implementation are provided
|
|
6
|
+
(this module is not unasync-generated); the frontends dispatch via ``_compat``.
|
|
7
|
+
|
|
8
|
+
Memory stays bounded: rows are written one at a time to the COPY stream and never fully
|
|
9
|
+
buffered. The identifiers (table/columns) come from the application, not end users — callers
|
|
10
|
+
must not pass untrusted identifiers (§18.5).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from collections.abc import AsyncIterable, Iterable, Sequence
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from ..errors import DatabaseUnsupportedOperationError
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _copy_sql(table: str, columns: Sequence[str]) -> str:
|
|
22
|
+
cols = ", ".join(f'"{c}"' for c in columns)
|
|
23
|
+
return f'COPY "{table}" ({cols}) FROM STDIN'
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _require_psycopg(driver_conn: Any) -> None:
|
|
27
|
+
# Both psycopg and asyncpg's raw connections expose a `cursor` method (incompatible
|
|
28
|
+
# signatures/purposes — asyncpg's is for server-side result cursors, not COPY), so `cursor`
|
|
29
|
+
# alone can't distinguish them. `pipeline` is psycopg-only (mirrors postgres/pipeline.py's
|
|
30
|
+
# own check) and reliably identifies a real psycopg connection.
|
|
31
|
+
if not hasattr(driver_conn, "pipeline"):
|
|
32
|
+
raise DatabaseUnsupportedOperationError(
|
|
33
|
+
"COPY requires the psycopg driver; this target uses a different driver"
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
async def copy_records_async(
|
|
38
|
+
sa_conn: Any,
|
|
39
|
+
table: str,
|
|
40
|
+
columns: Sequence[str],
|
|
41
|
+
records: Iterable[Sequence[Any]] | AsyncIterable[Sequence[Any]],
|
|
42
|
+
) -> int:
|
|
43
|
+
"""Stream ``records`` into ``table`` via COPY on the async psycopg connection. Returns the
|
|
44
|
+
number of rows written."""
|
|
45
|
+
raw = await sa_conn.get_raw_connection()
|
|
46
|
+
driver_conn = raw.driver_connection
|
|
47
|
+
_require_psycopg(driver_conn)
|
|
48
|
+
stmt = _copy_sql(table, columns)
|
|
49
|
+
written = 0
|
|
50
|
+
async with driver_conn.cursor() as cur, cur.copy(stmt) as copy:
|
|
51
|
+
if hasattr(records, "__aiter__"):
|
|
52
|
+
async for row in records:
|
|
53
|
+
await copy.write_row(row)
|
|
54
|
+
written += 1
|
|
55
|
+
else:
|
|
56
|
+
for row in records:
|
|
57
|
+
await copy.write_row(row)
|
|
58
|
+
written += 1
|
|
59
|
+
return written
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def copy_records_sync(
|
|
63
|
+
sa_conn: Any,
|
|
64
|
+
table: str,
|
|
65
|
+
columns: Sequence[str],
|
|
66
|
+
records: Iterable[Sequence[Any]],
|
|
67
|
+
) -> int:
|
|
68
|
+
"""Stream ``records`` into ``table`` via COPY on the sync psycopg connection."""
|
|
69
|
+
driver_conn = sa_conn.connection.driver_connection
|
|
70
|
+
_require_psycopg(driver_conn)
|
|
71
|
+
stmt = _copy_sql(table, columns)
|
|
72
|
+
written = 0
|
|
73
|
+
with driver_conn.cursor() as cur, cur.copy(stmt) as copy:
|
|
74
|
+
for row in records:
|
|
75
|
+
copy.write_row(row)
|
|
76
|
+
written += 1
|
|
77
|
+
return written
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""psycopg3 pipeline mode — batch several dependent statements into one round trip (§7.3).
|
|
2
|
+
|
|
3
|
+
In pipeline mode the client sends statements to the server without waiting for each response
|
|
4
|
+
before sending the next, then results are processed together. This is a raw-driver escape
|
|
5
|
+
hatch (like COPY): PostgreSQL + psycopg only, reached through
|
|
6
|
+
:meth:`AsyncConnectionScope.pipeline` / the sync equivalent, and useful for a handful of
|
|
7
|
+
dependent-but-batchable statements in the same transaction (e.g. a business write plus its
|
|
8
|
+
inbox record, §28).
|
|
9
|
+
|
|
10
|
+
Because SQLAlchemy's cursor execution transparently triggers a pipeline sync whenever a result
|
|
11
|
+
is actually fetched (rowcount, ``RETURNING``, a `SELECT``), ordinary ``tx.execute(...)`` calls
|
|
12
|
+
inside the block still work — the win comes from the driver not waiting for each response
|
|
13
|
+
before sending the next statement, not from changing the calling code.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import contextlib
|
|
19
|
+
from collections.abc import AsyncIterator, Iterator
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
from ..errors import DatabaseUnsupportedOperationError
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _require_psycopg(driver_conn: Any) -> None:
|
|
26
|
+
if not hasattr(driver_conn, "pipeline"):
|
|
27
|
+
raise DatabaseUnsupportedOperationError(
|
|
28
|
+
"pipeline mode requires the psycopg driver; this target uses a different driver"
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@contextlib.asynccontextmanager
|
|
33
|
+
async def pipeline_scope_async(sa_conn: Any) -> AsyncIterator[None]:
|
|
34
|
+
"""Enter psycopg pipeline mode on ``sa_conn``'s raw driver connection (async)."""
|
|
35
|
+
raw = await sa_conn.get_raw_connection()
|
|
36
|
+
driver_conn = raw.driver_connection
|
|
37
|
+
_require_psycopg(driver_conn)
|
|
38
|
+
async with driver_conn.pipeline():
|
|
39
|
+
yield
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@contextlib.contextmanager
|
|
43
|
+
def pipeline_scope_sync(sa_conn: Any) -> Iterator[None]:
|
|
44
|
+
"""Enter psycopg pipeline mode on ``sa_conn``'s raw driver connection (sync)."""
|
|
45
|
+
driver_conn = sa_conn.connection.driver_connection
|
|
46
|
+
_require_psycopg(driver_conn)
|
|
47
|
+
with driver_conn.pipeline():
|
|
48
|
+
yield
|
dbkit/postgres/unnest.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""PostgreSQL ``unnest()`` bulk insert/upsert — a mid-tier strategy between ``execute_many``
|
|
2
|
+
and ``COPY`` (§19).
|
|
3
|
+
|
|
4
|
+
``execute_many`` binds one parameter *per column per row*, so a batch is capped by
|
|
5
|
+
PostgreSQL's 65535 bind-parameter ceiling divided by column count. ``unnest`` instead binds
|
|
6
|
+
one parameter *per column* — each holding the full array of that column's values for the
|
|
7
|
+
batch — so batch size is bounded only by memory/payload, not the parameter ceiling. It sends
|
|
8
|
+
exactly one statement per batch, like ``execute_many``, but without that per-row multiplication.
|
|
9
|
+
|
|
10
|
+
This is plain SQL executed through the normal connection — unlike COPY, it needs no raw-driver
|
|
11
|
+
escape hatch. psycopg (and most PostgreSQL drivers) adapt a Python list bound to a single
|
|
12
|
+
placeholder into a native PostgreSQL array automatically.
|
|
13
|
+
|
|
14
|
+
Each array parameter is bound with an explicit ``::type[]`` cast (derived from the target
|
|
15
|
+
table's column types) — without it, PostgreSQL cannot resolve which ``unnest`` overload
|
|
16
|
+
applies when several array-typed parameters are passed with an unknown element type
|
|
17
|
+
(``AmbiguousFunction``). This covers ordinary scalar types (int, text, bool, timestamps,
|
|
18
|
+
uuid); composite/enum/array-of-array columns may still need the caller to avoid this strategy.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from collections.abc import Mapping, Sequence
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
from sqlalchemy import TextClause, text
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def unnest_insert_sql(
|
|
30
|
+
table: str,
|
|
31
|
+
columns: Sequence[str],
|
|
32
|
+
column_types: Mapping[str, str],
|
|
33
|
+
*,
|
|
34
|
+
conflict_index_elements: Sequence[str] | None = None,
|
|
35
|
+
update_columns: Sequence[str] | None = None,
|
|
36
|
+
) -> TextClause:
|
|
37
|
+
"""Build ``INSERT ... SELECT * FROM unnest(:c1::t1[], :c2::t2[], ...)``, optionally with
|
|
38
|
+
``ON CONFLICT``.
|
|
39
|
+
|
|
40
|
+
``column_types`` maps each column name to its PostgreSQL type (e.g. ``"INTEGER"``,
|
|
41
|
+
``"TEXT"``) — every column needs an explicit array cast, or PostgreSQL cannot resolve
|
|
42
|
+
which ``unnest`` overload applies when several unknown-typed array parameters are passed
|
|
43
|
+
(``AmbiguousFunction``).
|
|
44
|
+
|
|
45
|
+
``conflict_index_elements=None`` builds a plain insert. Given index elements with
|
|
46
|
+
``update_columns=None`` builds ``ON CONFLICT (...) DO NOTHING``; with columns given,
|
|
47
|
+
``ON CONFLICT (...) DO UPDATE SET col = EXCLUDED.col`` for each.
|
|
48
|
+
"""
|
|
49
|
+
col_list = ", ".join(f'"{c}"' for c in columns)
|
|
50
|
+
# CAST(:name AS type[]), not the ":name::type[]" shorthand — SQLAlchemy's text() bind-param
|
|
51
|
+
# regex does not recognize a name immediately followed by "::" and silently leaves it
|
|
52
|
+
# unbound, which the driver then rejects as invalid syntax.
|
|
53
|
+
unnest_args = ", ".join(f"CAST(:{c} AS {column_types[c]}[])" for c in columns)
|
|
54
|
+
alias_cols = ", ".join(f'"{c}"' for c in columns)
|
|
55
|
+
sql = (
|
|
56
|
+
f'INSERT INTO "{table}" ({col_list}) '
|
|
57
|
+
f"SELECT * FROM unnest({unnest_args}) AS _dbkit_unnest({alias_cols})"
|
|
58
|
+
)
|
|
59
|
+
if conflict_index_elements is not None:
|
|
60
|
+
idx = ", ".join(f'"{c}"' for c in conflict_index_elements)
|
|
61
|
+
if update_columns is None:
|
|
62
|
+
sql += f" ON CONFLICT ({idx}) DO NOTHING"
|
|
63
|
+
else:
|
|
64
|
+
set_clause = ", ".join(f'"{c}" = EXCLUDED."{c}"' for c in update_columns)
|
|
65
|
+
sql += f" ON CONFLICT ({idx}) DO UPDATE SET {set_clause}"
|
|
66
|
+
return text(sql)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def columnar_params(
|
|
70
|
+
rows: Sequence[Mapping[str, Any]], columns: Sequence[str]
|
|
71
|
+
) -> dict[str, list[Any]]:
|
|
72
|
+
"""Transpose row-dicts into ``{column: [value, value, ...]}`` — one array bind per column."""
|
|
73
|
+
return {c: [row[c] for row in rows] for c in columns}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""dbkit testing — see docs/roadmap.md (later phases)."""
|