pgcommon 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pgcommon/__init__.py +91 -0
- pgcommon/_internal/__init__.py +1 -0
- pgcommon/_internal/dsn.py +15 -0
- pgcommon/_internal/duration.py +38 -0
- pgcommon/_internal/query_event.py +22 -0
- pgcommon/_internal/sql_verb.py +36 -0
- pgcommon/_internal/sqlstate.py +20 -0
- pgcommon/config.py +222 -0
- pgcommon/errors.py +123 -0
- pgcommon/guc.py +142 -0
- pgcommon/integrations/__init__.py +5 -0
- pgcommon/integrations/fastapi.py +151 -0
- pgcommon/logger.py +71 -0
- pgcommon/metrics.py +179 -0
- pgcommon/migrate/__init__.py +8 -0
- pgcommon/migrate/__main__.py +6 -0
- pgcommon/migrate/cli.py +78 -0
- pgcommon/migrate/errors.py +12 -0
- pgcommon/migrate/runner.py +261 -0
- pgcommon/pool.py +346 -0
- pgcommon/py.typed +0 -0
- pgcommon/retry.py +121 -0
- pgcommon/tracer.py +85 -0
- pgcommon/tx.py +98 -0
- pgcommon-0.1.0.dist-info/METADATA +1369 -0
- pgcommon-0.1.0.dist-info/RECORD +29 -0
- pgcommon-0.1.0.dist-info/WHEEL +4 -0
- pgcommon-0.1.0.dist-info/entry_points.txt +2 -0
- pgcommon-0.1.0.dist-info/licenses/LICENSE +201 -0
pgcommon/guc.py
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""RLS tenant identity (`GUCSet`) and its request-scoped propagation via `contextvars`.
|
|
2
|
+
|
|
3
|
+
Go carries the active identity as an explicit `context.Context` value threaded through every
|
|
4
|
+
call. Python has no equivalent explicit parameter — `contextvars.ContextVar` is the mechanism
|
|
5
|
+
used instead, and it is scoped per `asyncio` task (each request gets its own copy once a task
|
|
6
|
+
is spawned for it). See ARCHITECTURE.md for the exact set/reset timing this requires in ASGI
|
|
7
|
+
middleware — getting that wrong is the single most likely place for a cross-request GUC leak.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections.abc import Awaitable, Iterator
|
|
13
|
+
from contextlib import contextmanager
|
|
14
|
+
from contextvars import ContextVar, Token
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Protocol
|
|
17
|
+
|
|
18
|
+
from pgcommon.errors import EmptyTenantRoleError, InconsistentGUCSetError, InvalidTenantRoleError
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"GUCSet",
|
|
22
|
+
"guc_set_from_context",
|
|
23
|
+
"inject_gucs",
|
|
24
|
+
"with_guc_set",
|
|
25
|
+
"with_validated_guc_set",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True, slots=True)
|
|
30
|
+
class GUCSet:
|
|
31
|
+
"""The RLS identity injected as `app.user_id` / `app.tenant_id` / `app.tenant_roles`."""
|
|
32
|
+
|
|
33
|
+
user_id: str = ""
|
|
34
|
+
tenant_id: str = ""
|
|
35
|
+
tenant_roles: tuple[str, ...] = field(default_factory=tuple)
|
|
36
|
+
|
|
37
|
+
def is_zero(self) -> bool:
|
|
38
|
+
"""True when every field is empty — the pool skips GUC injection entirely in this case."""
|
|
39
|
+
return self.user_id == "" and self.tenant_id == "" and not self.tenant_roles
|
|
40
|
+
|
|
41
|
+
def validate(self) -> None:
|
|
42
|
+
"""Raise if this GUCSet is internally inconsistent.
|
|
43
|
+
|
|
44
|
+
Checks, in order: `user_id` set without `tenant_id` (InconsistentGUCSetError), a role
|
|
45
|
+
containing a comma — the GUC serialization separator (InvalidTenantRoleError), and an
|
|
46
|
+
empty-string role (EmptyTenantRoleError).
|
|
47
|
+
"""
|
|
48
|
+
if self.user_id and not self.tenant_id:
|
|
49
|
+
raise InconsistentGUCSetError("GUCSet.user_id is set but tenant_id is empty")
|
|
50
|
+
for role in self.tenant_roles:
|
|
51
|
+
if "," in role:
|
|
52
|
+
raise InvalidTenantRoleError(f"GUCSet.tenant_roles entry {role!r} contains a comma")
|
|
53
|
+
if role == "":
|
|
54
|
+
raise EmptyTenantRoleError("GUCSet.tenant_roles contains an empty-string entry")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
_current_guc_set: ContextVar[GUCSet | None] = ContextVar("pgcommon_guc_set", default=None)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@contextmanager
|
|
61
|
+
def with_guc_set(guc_set: GUCSet) -> Iterator[None]:
|
|
62
|
+
"""Store `guc_set` on the current context for the duration of the `with` block.
|
|
63
|
+
|
|
64
|
+
Does not validate — use `with_validated_guc_set` in auth middleware so an inconsistent
|
|
65
|
+
identity is rejected before it reaches connection acquisition.
|
|
66
|
+
"""
|
|
67
|
+
token: Token[GUCSet | None] = _current_guc_set.set(guc_set)
|
|
68
|
+
try:
|
|
69
|
+
yield
|
|
70
|
+
finally:
|
|
71
|
+
_current_guc_set.reset(token)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@contextmanager
|
|
75
|
+
def with_validated_guc_set(guc_set: GUCSet) -> Iterator[None]:
|
|
76
|
+
"""Like `with_guc_set`, but calls `guc_set.validate()` first.
|
|
77
|
+
|
|
78
|
+
Mirrors the Go README's `WithValidatedGUCSet` — intended for auth middleware, which should
|
|
79
|
+
catch the raised error and respond 401 rather than let an inconsistent identity reach the
|
|
80
|
+
database layer.
|
|
81
|
+
"""
|
|
82
|
+
guc_set.validate()
|
|
83
|
+
with with_guc_set(guc_set):
|
|
84
|
+
yield
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def guc_set_from_context() -> GUCSet | None:
|
|
88
|
+
"""Return the `GUCSet` stored by the innermost enclosing `with_guc_set`, or `None`.
|
|
89
|
+
|
|
90
|
+
Pass this directly as `Config.guc_provider` for automatic RLS injection.
|
|
91
|
+
"""
|
|
92
|
+
return _current_guc_set.get()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class _GUCExecutable(Protocol):
|
|
96
|
+
"""Structural shape shared by `asyncpg.Connection` and `asyncpg.pool.PoolConnectionProxy`."""
|
|
97
|
+
|
|
98
|
+
def execute(self, query: str, *args: object) -> Awaitable[str]: ...
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
_INJECT_SQL = (
|
|
102
|
+
"SELECT set_config('app.user_id', $1, {is_local}), "
|
|
103
|
+
"set_config('app.tenant_id', $2, {is_local}), "
|
|
104
|
+
"set_config('app.tenant_roles', $3, {is_local})"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
async def inject_gucs(conn: _GUCExecutable, guc_set: GUCSet) -> None:
|
|
109
|
+
"""Session-scoped GUC injection (`is_local=false`) — always session-scoped, regardless of
|
|
110
|
+
`Config.pgbouncer_mode`. This is for manual, one-off injection; the pool's automatic
|
|
111
|
+
injection respects the configured mode (see `pgcommon.tx` for the PgBouncer-mode path).
|
|
112
|
+
|
|
113
|
+
A zero `GUCSet` (see `GUCSet.is_zero`) is silently skipped — no GUCs are set, and RLS
|
|
114
|
+
policies fall through to their own default (typically deny).
|
|
115
|
+
"""
|
|
116
|
+
guc_set.validate()
|
|
117
|
+
if guc_set.is_zero():
|
|
118
|
+
return
|
|
119
|
+
await conn.execute(
|
|
120
|
+
_INJECT_SQL.format(is_local="false"),
|
|
121
|
+
guc_set.user_id,
|
|
122
|
+
guc_set.tenant_id,
|
|
123
|
+
",".join(guc_set.tenant_roles),
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
async def _inject_gucs_local(conn: _GUCExecutable, guc_set: GUCSet) -> None:
|
|
128
|
+
"""Transaction-local GUC injection (`is_local=true`) for PgBouncer transaction-pooling mode.
|
|
129
|
+
|
|
130
|
+
Must be called inside an already-open transaction — `is_local=true` settings are cleared
|
|
131
|
+
automatically on `COMMIT` / `ROLLBACK`, which is exactly what makes this safe when the
|
|
132
|
+
backend connection is handed back to PgBouncer afterward.
|
|
133
|
+
"""
|
|
134
|
+
guc_set.validate()
|
|
135
|
+
if guc_set.is_zero():
|
|
136
|
+
return
|
|
137
|
+
await conn.execute(
|
|
138
|
+
_INJECT_SQL.format(is_local="true"),
|
|
139
|
+
guc_set.user_id,
|
|
140
|
+
guc_set.tenant_id,
|
|
141
|
+
",".join(guc_set.tenant_roles),
|
|
142
|
+
)
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""Optional framework glue. Nothing under `pgcommon.integrations` is imported by `pgcommon`
|
|
2
|
+
itself — each submodule pulls in its own framework only when explicitly imported, so the core
|
|
3
|
+
library stays framework-agnostic. `pgcommon.integrations.fastapi` requires the `pgcommon[fastapi]`
|
|
4
|
+
extra.
|
|
5
|
+
"""
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
"""FastAPI integration: pool lifespan wiring, GUC-injection middleware, and a `/healthz` example.
|
|
2
|
+
|
|
3
|
+
Optional — requires the `pgcommon[fastapi]` extra. The core library (`pgcommon.*`) never
|
|
4
|
+
imports `fastapi` or `starlette`; this is the only module in the package that does, and
|
|
5
|
+
`pgcommon/__init__.py` does not import it — importing `pgcommon.integrations.fastapi` is
|
|
6
|
+
always an explicit, opt-in step for a consuming service.
|
|
7
|
+
|
|
8
|
+
## contextvars and request isolation — read this before wiring `GUCAuthMiddleware`
|
|
9
|
+
|
|
10
|
+
pgcommon propagates the active `GUCSet` via `contextvars.ContextVar`, not an explicit context
|
|
11
|
+
parameter threaded through every call the way Go's `context.Context` is. Under ASGI, each
|
|
12
|
+
inbound request is handled by its own `asyncio.Task`, and task creation captures a *copy* of
|
|
13
|
+
the current `contextvars.Context` at the moment the task starts — so a `ContextVar.set()` call
|
|
14
|
+
made while handling one request is invisible to a concurrently-running request's task, even
|
|
15
|
+
though both run on the same event loop. That per-task copy is what makes it safe to stash a
|
|
16
|
+
`GUCSet` in middleware without a lock or an explicit per-request object.
|
|
17
|
+
|
|
18
|
+
The failure mode to avoid: **never call `ContextVar.set()` without capturing and later
|
|
19
|
+
`reset()`-ing its token in the same middleware invocation.** `GUCAuthMiddleware` below uses
|
|
20
|
+
`with_validated_guc_set(...)` — a context manager — wrapped tightly around `call_next(request)`
|
|
21
|
+
for exactly this reason: the token is reset in a `finally` block the moment the response for
|
|
22
|
+
*this* request has been produced, so the GUCSet cannot leak into code that runs later on the
|
|
23
|
+
same task (a connection-pool cleanup callback, a background job scheduled from the handler, or
|
|
24
|
+
— on a pathological ASGI server that reuses tasks across requests — the next request entirely).
|
|
25
|
+
Setting a `ContextVar` and forgetting to reset it is the single most likely place for a
|
|
26
|
+
cross-request GUC leak in this stack, more so than in Go, where the value is an explicit
|
|
27
|
+
argument and cannot silently outlive its caller.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
33
|
+
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
|
34
|
+
from typing import Protocol
|
|
35
|
+
|
|
36
|
+
from fastapi import Depends, FastAPI, Request
|
|
37
|
+
from fastapi.responses import JSONResponse
|
|
38
|
+
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
|
|
39
|
+
from starlette.responses import Response
|
|
40
|
+
from starlette.types import ASGIApp
|
|
41
|
+
|
|
42
|
+
from pgcommon.config import Config
|
|
43
|
+
from pgcommon.errors import EmptyTenantRoleError, InconsistentGUCSetError, InvalidTenantRoleError
|
|
44
|
+
from pgcommon.guc import GUCSet, with_validated_guc_set
|
|
45
|
+
from pgcommon.pool import Pool
|
|
46
|
+
|
|
47
|
+
__all__ = [
|
|
48
|
+
"GUCAuthMiddleware",
|
|
49
|
+
"IdentityExtractor",
|
|
50
|
+
"create_lifespan",
|
|
51
|
+
"get_pool",
|
|
52
|
+
"healthz",
|
|
53
|
+
]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class IdentityExtractor(Protocol):
|
|
57
|
+
"""Extracts the caller's identity from a request. Implement this for your auth scheme
|
|
58
|
+
(JWT claims, session lookup, an upstream `platform-gincommon`-style request context, ...).
|
|
59
|
+
|
|
60
|
+
Return `None` for requests with no identity (unauthenticated, internal tooling) — no GUCSet
|
|
61
|
+
is stored and downstream RLS policies apply their own default (typically deny-all).
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __call__(self, request: Request) -> GUCSet | None: ...
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def create_lifespan(
|
|
68
|
+
config: Config,
|
|
69
|
+
) -> Callable[[FastAPI], AbstractAsyncContextManager[None]]:
|
|
70
|
+
"""Build an ASGI lifespan that opens the `Pool` once at startup and drains it at shutdown.
|
|
71
|
+
|
|
72
|
+
The pool is stored on `app.state.pool` — never reconstructed per request. Usage::
|
|
73
|
+
|
|
74
|
+
app = FastAPI(lifespan=create_lifespan(config))
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
@asynccontextmanager
|
|
78
|
+
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
|
79
|
+
pool = await Pool.create(config)
|
|
80
|
+
app.state.pool = pool
|
|
81
|
+
try:
|
|
82
|
+
yield
|
|
83
|
+
finally:
|
|
84
|
+
await pool.drain_and_close()
|
|
85
|
+
|
|
86
|
+
return lifespan
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def get_pool(request: Request) -> Pool:
|
|
90
|
+
"""FastAPI dependency yielding the shared `Pool` built by `create_lifespan`.
|
|
91
|
+
|
|
92
|
+
Usage in a route handler::
|
|
93
|
+
|
|
94
|
+
@app.get("/orders")
|
|
95
|
+
async def list_orders(pool: Pool = Depends(get_pool)):
|
|
96
|
+
return await pool.with_conn(...)
|
|
97
|
+
"""
|
|
98
|
+
return request.app.state.pool # type: ignore[no-any-return]
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class GUCAuthMiddleware(BaseHTTPMiddleware):
|
|
102
|
+
"""Extracts a `GUCSet` via `identity_extractor` and scopes it to this request only.
|
|
103
|
+
|
|
104
|
+
Responds `401` if the extracted identity fails `GUCSet.validate()` (e.g. a user ID without
|
|
105
|
+
a tenant ID, or a role containing a comma) — rejecting an inconsistent identity here, before
|
|
106
|
+
it ever reaches connection acquisition, is exactly what the Go README's
|
|
107
|
+
`WithValidatedGUCSet` does in the equivalent middleware example.
|
|
108
|
+
"""
|
|
109
|
+
|
|
110
|
+
def __init__(self, app: ASGIApp, identity_extractor: IdentityExtractor) -> None:
|
|
111
|
+
super().__init__(app)
|
|
112
|
+
self._identity_extractor = identity_extractor
|
|
113
|
+
|
|
114
|
+
async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response:
|
|
115
|
+
guc_set = self._identity_extractor(request)
|
|
116
|
+
if guc_set is None:
|
|
117
|
+
return await call_next(request)
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
with with_validated_guc_set(guc_set):
|
|
121
|
+
return await call_next(request)
|
|
122
|
+
except (InconsistentGUCSetError, InvalidTenantRoleError, EmptyTenantRoleError):
|
|
123
|
+
return JSONResponse({"detail": "invalid identity"}, status_code=401)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def healthz(*, utilization_threshold: float = 0.9) -> Callable[..., Awaitable[JSONResponse]]:
|
|
127
|
+
"""Build a `/healthz` route handler with utilization-based load shedding.
|
|
128
|
+
|
|
129
|
+
Returns `503` when the database is unreachable, or when the pool's utilization is at or
|
|
130
|
+
above `utilization_threshold` (default 90%) — shedding new requests before the pool is
|
|
131
|
+
fully exhausted protects the connections already serving in-flight requests. Usage::
|
|
132
|
+
|
|
133
|
+
app.add_api_route("/healthz", healthz(), methods=["GET"])
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
async def handler(pool: Pool = Depends(get_pool)) -> JSONResponse:
|
|
137
|
+
status = await pool.health()
|
|
138
|
+
if not status.healthy:
|
|
139
|
+
return JSONResponse({"detail": "database unavailable"}, status_code=503)
|
|
140
|
+
if status.utilization >= utilization_threshold:
|
|
141
|
+
return JSONResponse({"detail": "service overloaded"}, status_code=503)
|
|
142
|
+
return JSONResponse(
|
|
143
|
+
{
|
|
144
|
+
"healthy": status.healthy,
|
|
145
|
+
"acquired_conns": status.acquired_conns,
|
|
146
|
+
"max_conns": status.max_conns,
|
|
147
|
+
"utilization": round(status.utilization, 3),
|
|
148
|
+
}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
return handler
|
pgcommon/logger.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""The `Logger` protocol pgcommon logs through, plus two reference adapters.
|
|
2
|
+
|
|
3
|
+
Mirrors the Go repo's `port.Logger` interface. `pgcommon` never imports a concrete logging
|
|
4
|
+
framework at the top level — callers wire in whichever adapter fits, or write their own by
|
|
5
|
+
implementing the four methods below.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from typing import Any, Protocol, runtime_checkable
|
|
12
|
+
|
|
13
|
+
__all__ = ["Logger", "StdlibLogger", "StructlogLogger"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@runtime_checkable
|
|
17
|
+
class Logger(Protocol):
|
|
18
|
+
"""Structured logger contract used throughout pgcommon (slow-query warnings, migration steps)."""
|
|
19
|
+
|
|
20
|
+
def debug(self, msg: str, /, **fields: object) -> None: ...
|
|
21
|
+
|
|
22
|
+
def info(self, msg: str, /, **fields: object) -> None: ...
|
|
23
|
+
|
|
24
|
+
def warn(self, msg: str, /, **fields: object) -> None: ...
|
|
25
|
+
|
|
26
|
+
def error(self, msg: str, /, **fields: object) -> None: ...
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class StdlibLogger:
|
|
30
|
+
"""Adapts `logging.Logger` to the `Logger` protocol; fields are passed via the `extra` dict."""
|
|
31
|
+
|
|
32
|
+
def __init__(self, logger: logging.Logger | None = None) -> None:
|
|
33
|
+
self._logger = logger if logger is not None else logging.getLogger("pgcommon")
|
|
34
|
+
|
|
35
|
+
def debug(self, msg: str, /, **fields: object) -> None:
|
|
36
|
+
self._logger.debug(msg, extra={"pgcommon_fields": fields} if fields else None)
|
|
37
|
+
|
|
38
|
+
def info(self, msg: str, /, **fields: object) -> None:
|
|
39
|
+
self._logger.info(msg, extra={"pgcommon_fields": fields} if fields else None)
|
|
40
|
+
|
|
41
|
+
def warn(self, msg: str, /, **fields: object) -> None:
|
|
42
|
+
self._logger.warning(msg, extra={"pgcommon_fields": fields} if fields else None)
|
|
43
|
+
|
|
44
|
+
def error(self, msg: str, /, **fields: object) -> None:
|
|
45
|
+
self._logger.error(msg, extra={"pgcommon_fields": fields} if fields else None)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class StructlogLogger:
|
|
49
|
+
"""Adapts a `structlog` logger to the `Logger` protocol — the reference implementation
|
|
50
|
+
mirroring the Go repo's `NewZapLogger`. Requires the `pgcommon[structlog]` extra."""
|
|
51
|
+
|
|
52
|
+
def __init__(self, logger: Any | None = None) -> None:
|
|
53
|
+
try:
|
|
54
|
+
import structlog
|
|
55
|
+
except ImportError as exc: # pragma: no cover - exercised only without the extra
|
|
56
|
+
raise ImportError(
|
|
57
|
+
"StructlogLogger requires the 'structlog' package — install pgcommon[structlog]"
|
|
58
|
+
) from exc
|
|
59
|
+
self._logger = logger if logger is not None else structlog.get_logger("pgcommon")
|
|
60
|
+
|
|
61
|
+
def debug(self, msg: str, /, **fields: object) -> None:
|
|
62
|
+
self._logger.debug(msg, **fields)
|
|
63
|
+
|
|
64
|
+
def info(self, msg: str, /, **fields: object) -> None:
|
|
65
|
+
self._logger.info(msg, **fields)
|
|
66
|
+
|
|
67
|
+
def warn(self, msg: str, /, **fields: object) -> None:
|
|
68
|
+
self._logger.warning(msg, **fields)
|
|
69
|
+
|
|
70
|
+
def error(self, msg: str, /, **fields: object) -> None:
|
|
71
|
+
self._logger.error(msg, **fields)
|
pgcommon/metrics.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Prometheus metrics registration and the OpenTelemetry `db.query` span helper.
|
|
2
|
+
|
|
3
|
+
Mirrors the Go repo's `pkg/pgmetrics`. Both Prometheus and OTel are optional: the pool,
|
|
4
|
+
transactions, and GUC injection all work without ever calling `init` or supplying
|
|
5
|
+
`Config.tracer`.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import time
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
|
|
13
|
+
from opentelemetry.trace import SpanKind, Status, StatusCode, Tracer
|
|
14
|
+
from prometheus_client import REGISTRY as _DEFAULT_REGISTRY
|
|
15
|
+
from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram
|
|
16
|
+
|
|
17
|
+
from pgcommon._internal.query_event import QueryEvent
|
|
18
|
+
from pgcommon._internal.sql_verb import sql_operation
|
|
19
|
+
|
|
20
|
+
__all__ = ["Metrics", "OTelQueryRecorder", "current", "init", "init_with_registry"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True, slots=True)
|
|
24
|
+
class Metrics:
|
|
25
|
+
"""The handles `init` / `init_with_registry` return — pass to `OTelQueryRecorder` or use directly."""
|
|
26
|
+
|
|
27
|
+
query_total: Counter
|
|
28
|
+
query_duration: Histogram
|
|
29
|
+
pool_acquire_total: Counter
|
|
30
|
+
pool_acquire_duration: Histogram
|
|
31
|
+
slow_query_total: Counter
|
|
32
|
+
retry_total: Counter
|
|
33
|
+
retry_exhausted_total: Counter
|
|
34
|
+
build_info: Gauge
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
_metrics: Metrics | None = None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def current() -> Metrics | None:
|
|
41
|
+
"""Return the `Metrics` handle registered by `init` / `init_with_registry`, or `None`.
|
|
42
|
+
|
|
43
|
+
Used internally by `Pool`, `run_in_tx`, and the retry helpers to record metrics only when
|
|
44
|
+
observability has been opted into — none of them require it to function.
|
|
45
|
+
"""
|
|
46
|
+
return _metrics
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def init(service: str, version: str) -> Metrics:
|
|
50
|
+
"""Register Prometheus metrics against the default registry. Idempotent — first caller wins."""
|
|
51
|
+
return init_with_registry(service, version, _DEFAULT_REGISTRY)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def init_with_registry(service: str, version: str, registry: CollectorRegistry) -> Metrics:
|
|
55
|
+
"""Register Prometheus metrics against a specific registry (use a fresh one in tests).
|
|
56
|
+
|
|
57
|
+
Idempotent per process: subsequent calls (against any registry) return the handles from
|
|
58
|
+
the first call rather than re-registering, mirroring the Go repo's `sync.Once` guard.
|
|
59
|
+
"""
|
|
60
|
+
global _metrics
|
|
61
|
+
if _metrics is not None:
|
|
62
|
+
return _metrics
|
|
63
|
+
|
|
64
|
+
build_info = Gauge(
|
|
65
|
+
"pgcommon_build_info",
|
|
66
|
+
"Static build metadata for the service using pgcommon; value is always 1.",
|
|
67
|
+
["service", "version"],
|
|
68
|
+
registry=registry,
|
|
69
|
+
)
|
|
70
|
+
build_info.labels(service=service, version=version).set(1)
|
|
71
|
+
|
|
72
|
+
_metrics = Metrics(
|
|
73
|
+
query_total=Counter(
|
|
74
|
+
"pgcommon_query_total",
|
|
75
|
+
"Queries executed, labeled by SQL verb and outcome.",
|
|
76
|
+
["operation", "status"],
|
|
77
|
+
registry=registry,
|
|
78
|
+
),
|
|
79
|
+
query_duration=Histogram(
|
|
80
|
+
"pgcommon_query_duration_seconds",
|
|
81
|
+
"Query duration in seconds, labeled by SQL verb.",
|
|
82
|
+
["operation"],
|
|
83
|
+
registry=registry,
|
|
84
|
+
),
|
|
85
|
+
pool_acquire_total=Counter(
|
|
86
|
+
"pgcommon_pool_acquire_total",
|
|
87
|
+
"Connection acquire attempts, labeled by outcome.",
|
|
88
|
+
["status"],
|
|
89
|
+
registry=registry,
|
|
90
|
+
),
|
|
91
|
+
pool_acquire_duration=Histogram(
|
|
92
|
+
"pgcommon_pool_acquire_duration_seconds",
|
|
93
|
+
"Time spent waiting to acquire a connection from the pool.",
|
|
94
|
+
registry=registry,
|
|
95
|
+
),
|
|
96
|
+
slow_query_total=Counter(
|
|
97
|
+
"pgcommon_slow_query_total",
|
|
98
|
+
"Queries exceeding Config.slow_query_threshold, counted regardless of log rate-limiting.",
|
|
99
|
+
registry=registry,
|
|
100
|
+
),
|
|
101
|
+
retry_total=Counter(
|
|
102
|
+
"pgcommon_retry_total",
|
|
103
|
+
"Retried transaction attempts, labeled by retry reason.",
|
|
104
|
+
["reason"],
|
|
105
|
+
registry=registry,
|
|
106
|
+
),
|
|
107
|
+
retry_exhausted_total=Counter(
|
|
108
|
+
"pgcommon_retry_exhausted_total",
|
|
109
|
+
"Retry sequences that exhausted all attempts without success, labeled by reason.",
|
|
110
|
+
["reason"],
|
|
111
|
+
registry=registry,
|
|
112
|
+
),
|
|
113
|
+
build_info=build_info,
|
|
114
|
+
)
|
|
115
|
+
return _metrics
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class OTelQueryRecorder:
|
|
119
|
+
"""Records one `db.query` span per query, reconstructed after the fact from `elapsed`.
|
|
120
|
+
|
|
121
|
+
asyncpg's query logger fires once the query has already completed, so there is no live
|
|
122
|
+
span to start before the query and end after it (pgx's `QueryTracer` gets both a start and
|
|
123
|
+
an end hook). Instead this back-dates the span's start time using `record.elapsed`, which
|
|
124
|
+
yields the correct duration and still nests correctly under the caller's active span
|
|
125
|
+
because `asyncio.Task`/`call_soon` scheduling preserves the `contextvars` context that
|
|
126
|
+
carries OTel's current-span pointer.
|
|
127
|
+
"""
|
|
128
|
+
|
|
129
|
+
def __init__(
|
|
130
|
+
self,
|
|
131
|
+
tracer: Tracer,
|
|
132
|
+
*,
|
|
133
|
+
allow_full_statements: bool = False,
|
|
134
|
+
metrics: Metrics | None = None,
|
|
135
|
+
) -> None:
|
|
136
|
+
self._tracer = tracer
|
|
137
|
+
self._allow_full_statements = allow_full_statements
|
|
138
|
+
self._metrics = metrics
|
|
139
|
+
|
|
140
|
+
def allow_full_sql(self) -> None:
|
|
141
|
+
"""Opt in to including the full SQL text in `db.statement` (default: verb only)."""
|
|
142
|
+
self._allow_full_statements = True
|
|
143
|
+
|
|
144
|
+
def __call__(self, record: QueryEvent) -> None:
|
|
145
|
+
"""Register this as an asyncpg query logger: `conn.add_query_logger(recorder)`."""
|
|
146
|
+
end = time.time()
|
|
147
|
+
start = end - record.elapsed
|
|
148
|
+
operation = sql_operation(record.query)
|
|
149
|
+
|
|
150
|
+
attributes: dict[str, str | int] = {
|
|
151
|
+
"db.system": "postgresql",
|
|
152
|
+
"db.statement": record.query if self._allow_full_statements else operation,
|
|
153
|
+
}
|
|
154
|
+
params = record.conn_params
|
|
155
|
+
user = getattr(params, "user", None)
|
|
156
|
+
host = getattr(params, "host", None)
|
|
157
|
+
port = getattr(params, "port", None)
|
|
158
|
+
if user:
|
|
159
|
+
attributes["db.user"] = user
|
|
160
|
+
if host:
|
|
161
|
+
attributes["net.peer.name"] = host
|
|
162
|
+
if port:
|
|
163
|
+
attributes["net.peer.port"] = port
|
|
164
|
+
|
|
165
|
+
span = self._tracer.start_span(
|
|
166
|
+
"db.query",
|
|
167
|
+
kind=SpanKind.CLIENT,
|
|
168
|
+
attributes=attributes,
|
|
169
|
+
start_time=int(start * 1_000_000_000),
|
|
170
|
+
)
|
|
171
|
+
status = "error" if record.exception is not None else "success"
|
|
172
|
+
if record.exception is not None:
|
|
173
|
+
span.record_exception(record.exception)
|
|
174
|
+
span.set_status(Status(StatusCode.ERROR))
|
|
175
|
+
span.end(end_time=int(end * 1_000_000_000))
|
|
176
|
+
|
|
177
|
+
if self._metrics is not None:
|
|
178
|
+
self._metrics.query_total.labels(operation=operation, status=status).inc()
|
|
179
|
+
self._metrics.query_duration.labels(operation=operation).observe(record.elapsed)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Migration runner — a minimal, dependency-light golang-migrate equivalent over plain .sql files."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from pgcommon.migrate.errors import MigrationDirtyError, NoMigrationsFoundError
|
|
6
|
+
from pgcommon.migrate.runner import Migration, Runner
|
|
7
|
+
|
|
8
|
+
__all__ = ["Migration", "MigrationDirtyError", "NoMigrationsFoundError", "Runner"]
|
pgcommon/migrate/cli.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""`pgcommon-migrate` console script — also runnable as `python -m pgcommon.migrate`.
|
|
2
|
+
|
|
3
|
+
Intended for a Kubernetes init container or a CI/CD pre-deploy job — never as a sidecar
|
|
4
|
+
running alongside the live application, and never invoked from every replica's own boot path
|
|
5
|
+
simultaneously. See the README's migration deployment-pattern table.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import asyncio
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
from pgcommon.errors import MigrationDirtyError, NoMigrationsFoundError
|
|
16
|
+
from pgcommon.logger import StdlibLogger
|
|
17
|
+
from pgcommon.migrate.runner import Runner
|
|
18
|
+
|
|
19
|
+
__all__ = ["main"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _build_parser() -> argparse.ArgumentParser:
|
|
23
|
+
parser = argparse.ArgumentParser(
|
|
24
|
+
prog="pgcommon-migrate", description="Apply or roll back pgcommon SQL migrations."
|
|
25
|
+
)
|
|
26
|
+
parser.add_argument("--dsn", default=None, help="Postgres DSN (default: $DATABASE_URL)")
|
|
27
|
+
parser.add_argument(
|
|
28
|
+
"--source",
|
|
29
|
+
default="migrations",
|
|
30
|
+
help="Directory of *.up.sql / *.down.sql files (default: ./migrations)",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
34
|
+
subparsers.add_parser("up", help="Apply all pending migrations")
|
|
35
|
+
|
|
36
|
+
down_parser = subparsers.add_parser("down", help="Roll back N steps (default: 1)")
|
|
37
|
+
down_parser.add_argument("n", type=int, nargs="?", default=1)
|
|
38
|
+
|
|
39
|
+
steps_parser = subparsers.add_parser("steps", help="Apply +N or roll back -N steps")
|
|
40
|
+
steps_parser.add_argument("n", type=int)
|
|
41
|
+
|
|
42
|
+
subparsers.add_parser("version", help="Print the current schema version")
|
|
43
|
+
return parser
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
async def _run(args: argparse.Namespace) -> int:
|
|
47
|
+
dsn = args.dsn or os.environ.get("DATABASE_URL")
|
|
48
|
+
if not dsn:
|
|
49
|
+
print("error: --dsn or DATABASE_URL is required", file=sys.stderr)
|
|
50
|
+
return 2
|
|
51
|
+
|
|
52
|
+
runner = Runner(dsn, args.source, logger=StdlibLogger())
|
|
53
|
+
try:
|
|
54
|
+
if args.command == "up":
|
|
55
|
+
await runner.up()
|
|
56
|
+
elif args.command == "down":
|
|
57
|
+
await runner.down(args.n)
|
|
58
|
+
elif args.command == "steps":
|
|
59
|
+
await runner.steps(args.n)
|
|
60
|
+
elif args.command == "version":
|
|
61
|
+
version, dirty = await runner.version()
|
|
62
|
+
print(f"version={version} dirty={dirty}")
|
|
63
|
+
except MigrationDirtyError as exc:
|
|
64
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
65
|
+
return 1
|
|
66
|
+
except NoMigrationsFoundError as exc:
|
|
67
|
+
print(f"error: {exc}", file=sys.stderr)
|
|
68
|
+
return 1
|
|
69
|
+
return 0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def main(argv: list[str] | None = None) -> int:
|
|
73
|
+
args = _build_parser().parse_args(argv)
|
|
74
|
+
return asyncio.run(_run(args))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__": # pragma: no cover
|
|
78
|
+
sys.exit(main())
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Re-exports of the migration-specific sentinel errors defined in `pgcommon.errors`.
|
|
2
|
+
|
|
3
|
+
Kept as a separate import path (`pgcommon.migrate.errors`) to mirror the Go repo's package
|
|
4
|
+
layout; the classes themselves live in `pgcommon.errors` so `isinstance` checks against
|
|
5
|
+
errors raised anywhere else in the library still work.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pgcommon.errors import MigrationDirtyError, NoMigrationsFoundError
|
|
11
|
+
|
|
12
|
+
__all__ = ["MigrationDirtyError", "NoMigrationsFoundError"]
|