gnomon-postgres 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gnomon/__init__.py +140 -0
- gnomon/_util.py +82 -0
- gnomon/aio.py +360 -0
- gnomon/asyncpg.py +171 -0
- gnomon/client.py +361 -0
- gnomon/django.py +68 -0
- gnomon/errors.py +65 -0
- gnomon/pytest_plugin.py +62 -0
- gnomon/sqlalchemy.py +32 -0
- gnomon/testing.py +129 -0
- gnomon/tools.json +40 -0
- gnomon/tools.py +103 -0
- gnomon_postgres-1.0.0.dist-info/METADATA +79 -0
- gnomon_postgres-1.0.0.dist-info/RECORD +18 -0
- gnomon_postgres-1.0.0.dist-info/WHEEL +5 -0
- gnomon_postgres-1.0.0.dist-info/entry_points.txt +2 -0
- gnomon_postgres-1.0.0.dist-info/licenses/LICENSE +17 -0
- gnomon_postgres-1.0.0.dist-info/top_level.txt +1 -0
gnomon/asyncpg.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""asyncpg for the async SDK.
|
|
2
|
+
|
|
3
|
+
import asyncpg
|
|
4
|
+
from gnomon import AsyncGnomon
|
|
5
|
+
from gnomon.asyncpg import asyncpg_source
|
|
6
|
+
|
|
7
|
+
app = await asyncpg.create_pool("postgresql://app@db/shop")
|
|
8
|
+
runs = await asyncpg.create_pool("postgresql://runs@db/shop")
|
|
9
|
+
gn = AsyncGnomon(app=asyncpg_source(app), runs=asyncpg_source(runs))
|
|
10
|
+
|
|
11
|
+
The SDK speaks psycopg's interface; this hands it connections from an asyncpg pool (or a single
|
|
12
|
+
asyncpg connection) that answer the part of it the SDK uses: ``%s`` placeholders become ``$n``,
|
|
13
|
+
rows come back as dicts (or tuples), row counts come from the command status, and PostgreSQL's
|
|
14
|
+
errors come back as psycopg errors carrying the SQLSTATE and hint, so they map to the SDK's typed
|
|
15
|
+
errors. A statement outside a transaction runs in one of its own, so behind a transaction pooler
|
|
16
|
+
(pgbouncer) its prepare and execute stay on one server connection.
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
import contextlib
|
|
22
|
+
import re
|
|
23
|
+
from typing import Any, AsyncIterator, Optional
|
|
24
|
+
|
|
25
|
+
import psycopg
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def to_dollar(sql: str) -> str:
|
|
29
|
+
"""psycopg's placeholders (%s, %% for a percent sign) as PostgreSQL's ($1, $2, ...), leaving
|
|
30
|
+
quoted strings, identifiers, dollar-quoted bodies and comments alone."""
|
|
31
|
+
out, i, n, k = [], 0, len(sql), 0
|
|
32
|
+
while i < n:
|
|
33
|
+
c = sql[i]
|
|
34
|
+
if c in "'\"":
|
|
35
|
+
j = i + 1
|
|
36
|
+
while j < n:
|
|
37
|
+
if sql[j] == c:
|
|
38
|
+
if j + 1 < n and sql[j + 1] == c:
|
|
39
|
+
j += 2
|
|
40
|
+
continue
|
|
41
|
+
break
|
|
42
|
+
j += 1
|
|
43
|
+
out.append(sql[i:j + 1])
|
|
44
|
+
i = j + 1
|
|
45
|
+
elif c == "$" and (m := re.match(r"\$([A-Za-z_][A-Za-z_0-9]*)?\$", sql[i:])):
|
|
46
|
+
tag = m.group(0)
|
|
47
|
+
j = sql.find(tag, i + len(tag))
|
|
48
|
+
j = n if j < 0 else j + len(tag)
|
|
49
|
+
out.append(sql[i:j])
|
|
50
|
+
i = j
|
|
51
|
+
elif sql.startswith("--", i):
|
|
52
|
+
j = sql.find("\n", i)
|
|
53
|
+
j = n if j < 0 else j
|
|
54
|
+
out.append(sql[i:j])
|
|
55
|
+
i = j
|
|
56
|
+
elif sql.startswith("/*", i):
|
|
57
|
+
j = sql.find("*/", i + 2)
|
|
58
|
+
j = n if j < 0 else j + 2
|
|
59
|
+
out.append(sql[i:j])
|
|
60
|
+
i = j
|
|
61
|
+
elif sql.startswith("%s", i):
|
|
62
|
+
k += 1
|
|
63
|
+
out.append(f"${k}")
|
|
64
|
+
i += 2
|
|
65
|
+
elif sql.startswith("%%", i):
|
|
66
|
+
out.append("%")
|
|
67
|
+
i += 2
|
|
68
|
+
else:
|
|
69
|
+
out.append(c)
|
|
70
|
+
i += 1
|
|
71
|
+
return "".join(out)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class _Diag:
|
|
75
|
+
def __init__(self, e: BaseException):
|
|
76
|
+
self.message_primary = getattr(e, "message", None) or str(e)
|
|
77
|
+
self.message_detail = getattr(e, "detail", None)
|
|
78
|
+
self.message_hint = getattr(e, "hint", None)
|
|
79
|
+
self.sqlstate = getattr(e, "sqlstate", None)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class AsyncpgError(psycopg.Error):
|
|
83
|
+
"""An asyncpg PostgreSQL error, as the SDK reads psycopg's."""
|
|
84
|
+
|
|
85
|
+
def __init__(self, e: BaseException):
|
|
86
|
+
self.cause = e # (first: psycopg's constructor reads sqlstate)
|
|
87
|
+
super().__init__(getattr(e, "message", None) or str(e))
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def sqlstate(self) -> Optional[str]: # type: ignore[override]
|
|
91
|
+
return getattr(self.cause, "sqlstate", None)
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def diag(self) -> _Diag: # type: ignore[override]
|
|
95
|
+
return _Diag(self.cause)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class _Cursor:
|
|
99
|
+
def __init__(self, conn: Any, dicts: bool):
|
|
100
|
+
self.conn, self.dicts = conn, dicts
|
|
101
|
+
self.description: Optional[list[str]] = None
|
|
102
|
+
self.rowcount = -1
|
|
103
|
+
self.statusmessage: Optional[str] = None
|
|
104
|
+
self._rows: list = []
|
|
105
|
+
|
|
106
|
+
async def __aenter__(self) -> "_Cursor":
|
|
107
|
+
return self
|
|
108
|
+
|
|
109
|
+
async def __aexit__(self, *exc) -> bool:
|
|
110
|
+
return False
|
|
111
|
+
|
|
112
|
+
async def execute(self, sql: str, params: Any = None, prepare: Any = None) -> None:
|
|
113
|
+
q = to_dollar(sql) if params else sql
|
|
114
|
+
args = list(params or ())
|
|
115
|
+
|
|
116
|
+
async def run():
|
|
117
|
+
ps = await self.conn.prepare(q)
|
|
118
|
+
return ps, await ps.fetch(*args)
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
if self.conn.is_in_transaction():
|
|
122
|
+
ps, rows = await run()
|
|
123
|
+
else:
|
|
124
|
+
async with self.conn.transaction():
|
|
125
|
+
ps, rows = await run()
|
|
126
|
+
except Exception as e: # asyncpg.PostgresError carries sqlstate
|
|
127
|
+
if getattr(e, "sqlstate", None):
|
|
128
|
+
raise AsyncpgError(e) from e
|
|
129
|
+
raise
|
|
130
|
+
attrs = list(ps.get_attributes())
|
|
131
|
+
self.description = [a.name for a in attrs] or None
|
|
132
|
+
self._rows = [dict(r) if self.dicts else tuple(r) for r in rows] if attrs else []
|
|
133
|
+
self.statusmessage = ps.get_statusmsg()
|
|
134
|
+
m = re.search(r"(\d+)\s*$", self.statusmessage or "")
|
|
135
|
+
self.rowcount = int(m.group(1)) if m else len(self._rows)
|
|
136
|
+
|
|
137
|
+
async def fetchall(self) -> list:
|
|
138
|
+
return self._rows
|
|
139
|
+
|
|
140
|
+
async def fetchone(self) -> Any:
|
|
141
|
+
return self._rows[0] if self._rows else None
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class _Conn:
|
|
145
|
+
def __init__(self, conn: Any):
|
|
146
|
+
self._c = conn
|
|
147
|
+
|
|
148
|
+
def transaction(self):
|
|
149
|
+
return self._c.transaction()
|
|
150
|
+
|
|
151
|
+
def cursor(self, row_factory: Any = None) -> _Cursor:
|
|
152
|
+
return _Cursor(self._c, row_factory is not None)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
class asyncpg_source:
|
|
156
|
+
"""A login for AsyncGnomon from an asyncpg pool (a connection per call) or one asyncpg
|
|
157
|
+
connection (used one call at a time)."""
|
|
158
|
+
|
|
159
|
+
def __init__(self, source: Any):
|
|
160
|
+
self.source = source
|
|
161
|
+
self.lock = asyncio.Lock()
|
|
162
|
+
|
|
163
|
+
@contextlib.asynccontextmanager
|
|
164
|
+
async def connection(self) -> AsyncIterator[_Conn]:
|
|
165
|
+
s = self.source
|
|
166
|
+
if hasattr(s, "acquire"):
|
|
167
|
+
async with s.acquire() as c:
|
|
168
|
+
yield _Conn(c)
|
|
169
|
+
else:
|
|
170
|
+
async with self.lock:
|
|
171
|
+
yield _Conn(s)
|
gnomon/client.py
ADDED
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
"""The gnomon SDK, sync (the specification is sdk/spec/SPEC.md).
|
|
2
|
+
|
|
3
|
+
Generated from ``aio.py`` by ``tools/unasync.py``: do not edit; change aio.py.
|
|
4
|
+
|
|
5
|
+
gn = Gnomon(app="postgresql://app@db/app", runs="postgresql://runs@db/app")
|
|
6
|
+
with gn.start_run("Refund order 3", actor="support-bot") as run:
|
|
7
|
+
run.sql("update orders set status = 'refunded' where id = 3")
|
|
8
|
+
summary = run.summary()
|
|
9
|
+
if summary["checks"]["passed"] and summary["changes"]["rows"] <= 20:
|
|
10
|
+
run.merge()
|
|
11
|
+
else:
|
|
12
|
+
run.submit("Refund order 3")
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import threading
|
|
17
|
+
import time
|
|
18
|
+
import contextlib
|
|
19
|
+
import json
|
|
20
|
+
from typing import Any, Iterator, Callable, Optional
|
|
21
|
+
|
|
22
|
+
import psycopg
|
|
23
|
+
import psycopg.conninfo
|
|
24
|
+
from psycopg.rows import dict_row
|
|
25
|
+
|
|
26
|
+
from ._util import parse_json, split_sql
|
|
27
|
+
from .errors import (GnomonError, IncompatibleVersionError, MergeConflictError, SerializationError,
|
|
28
|
+
to_gnomon_error)
|
|
29
|
+
|
|
30
|
+
SUPPORTED_API = (2, 2)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Adapter:
|
|
34
|
+
"""A login: a connection string (a connection per call), a psycopg connection (used one call
|
|
35
|
+
at a time), or a pool with a ``connection()`` context manager (psycopg_pool)."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, source: Any):
|
|
38
|
+
self.source = source
|
|
39
|
+
self.lock = threading.Lock()
|
|
40
|
+
|
|
41
|
+
@contextlib.contextmanager
|
|
42
|
+
def connection(self) -> Iterator[psycopg.Connection]:
|
|
43
|
+
s = self.source
|
|
44
|
+
if isinstance(s, str):
|
|
45
|
+
conn = psycopg.Connection.connect(s, autocommit=True)
|
|
46
|
+
try:
|
|
47
|
+
yield conn
|
|
48
|
+
finally:
|
|
49
|
+
conn.close()
|
|
50
|
+
elif hasattr(s, "connection") and not isinstance(s, psycopg.Connection):
|
|
51
|
+
with s.connection() as conn:
|
|
52
|
+
yield conn
|
|
53
|
+
else:
|
|
54
|
+
with self.lock:
|
|
55
|
+
yield s
|
|
56
|
+
|
|
57
|
+
def query(self, sql: str, params: tuple = ()) -> list[dict]:
|
|
58
|
+
with self.connection() as conn:
|
|
59
|
+
with conn.cursor(row_factory=dict_row) as cur:
|
|
60
|
+
cur.execute(sql, params)
|
|
61
|
+
return cur.fetchall() if cur.description else []
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Gnomon:
|
|
65
|
+
"""The application's side: starts runs, reads them, accepts them; review; core."""
|
|
66
|
+
|
|
67
|
+
def __init__(self, app: Any, runs: Any = None, *, statement_timeout_ms: int = 30000, lock_timeout_ms: int = 5000,
|
|
68
|
+
max_rows: int = 1000, on_statement: Optional[Callable[[dict], None]] = None, skip_version_check: bool = False,
|
|
69
|
+
url: Optional[str] = None):
|
|
70
|
+
# url: the database's connection string (host, port, database), for run.url() and
|
|
71
|
+
# run.connection(); default: the app login's, when it is a string
|
|
72
|
+
self.url = url or (app if isinstance(app, str) else None)
|
|
73
|
+
self.app = Adapter(app)
|
|
74
|
+
self.runs = Adapter(runs) if runs is not None else None
|
|
75
|
+
self.statement_timeout_ms, self.lock_timeout_ms, self.max_rows = statement_timeout_ms, lock_timeout_ms, max_rows
|
|
76
|
+
self.on_statement = on_statement
|
|
77
|
+
self._checked = skip_version_check
|
|
78
|
+
self.requests = Requests(self)
|
|
79
|
+
self.branches = Branches(self)
|
|
80
|
+
|
|
81
|
+
def ready(self) -> None:
|
|
82
|
+
"""Check once that the extension speaks this SDK's API."""
|
|
83
|
+
if self._checked:
|
|
84
|
+
return
|
|
85
|
+
try:
|
|
86
|
+
rows = self.app.query("select gnomon.api_version() as v")
|
|
87
|
+
except psycopg.Error as e:
|
|
88
|
+
raise IncompatibleVersionError("this server's gnomon has no gnomon.api_version(): it predates this SDK (needs API 2)", cause=e)
|
|
89
|
+
v = rows[0]["v"]
|
|
90
|
+
if not SUPPORTED_API[0] <= v <= SUPPORTED_API[1]:
|
|
91
|
+
raise IncompatibleVersionError(f"gnomon API {v} on the server; this SDK speaks {SUPPORTED_API[0]}..{SUPPORTED_API[1]}")
|
|
92
|
+
self._checked = True
|
|
93
|
+
|
|
94
|
+
def _event(self, login: str, run: Optional[str], sql: str, ok: bool, error: Optional[GnomonError] = None) -> None:
|
|
95
|
+
if self.on_statement:
|
|
96
|
+
self.on_statement({"login": login, "run": run, "statement": sql, "ok": ok, "error": error})
|
|
97
|
+
|
|
98
|
+
def query(self, sql: str, params: tuple = ()) -> list[dict]:
|
|
99
|
+
"""A statement on the app login, errors typed."""
|
|
100
|
+
self.ready()
|
|
101
|
+
try:
|
|
102
|
+
rows = self.app.query(sql, params)
|
|
103
|
+
except psycopg.Error as e:
|
|
104
|
+
ge = to_gnomon_error(e)
|
|
105
|
+
self._event("app", None, sql, False, ge)
|
|
106
|
+
raise ge from e
|
|
107
|
+
self._event("app", None, sql, True)
|
|
108
|
+
return rows
|
|
109
|
+
|
|
110
|
+
def value(self, sql: str, params: tuple = ()) -> Any:
|
|
111
|
+
rows = self.query(sql, params)
|
|
112
|
+
return next(iter(rows[0].values())) if rows else None
|
|
113
|
+
|
|
114
|
+
def json(self, sql: str, params: tuple = ()) -> Any:
|
|
115
|
+
return parse_json(self.value(f"select ({sql})::text", params))
|
|
116
|
+
|
|
117
|
+
def start_run(self, task: Optional[str] = None, *, actor: Optional[str] = None, kind: str = "logged",
|
|
118
|
+
ttl: Optional[str] = None, parent: str = "main") -> "Run":
|
|
119
|
+
if self.runs is None:
|
|
120
|
+
raise GnomonError("start_run needs the runs login: Gnomon(app=..., runs=...)")
|
|
121
|
+
branch = self.value("select gnomon.run_start(%s, %s, %s, %s::interval, %s)", (task, actor, kind, ttl, parent))
|
|
122
|
+
return Run(self, branch, task=task, actor=actor)
|
|
123
|
+
|
|
124
|
+
def run(self, branch: str, *, task: Optional[str] = None, actor: Optional[str] = None) -> "Run":
|
|
125
|
+
return Run(self, branch, task=task, actor=actor)
|
|
126
|
+
|
|
127
|
+
# ---- core
|
|
128
|
+
def status(self) -> dict:
|
|
129
|
+
return self.json("select gnomon.status()")
|
|
130
|
+
|
|
131
|
+
def doctor(self) -> list[dict]:
|
|
132
|
+
return self.query("select check_name, status, detail from gnomon.doctor()")
|
|
133
|
+
|
|
134
|
+
def describe(self, schema: Optional[str] = None) -> dict:
|
|
135
|
+
return self.json("select gnomon.describe(%s)", (schema,))
|
|
136
|
+
|
|
137
|
+
def diff(self, a: str, b: Optional[str] = None) -> list[dict]:
|
|
138
|
+
rows = self.query("select tbl::text as table, key::text as key, kind, col as column, old::text as old, new::text as new from gnomon.diff(%s, %s)", (a, b))
|
|
139
|
+
return [{**r, "key": parse_json(r["key"]), "old": parse_json(r["old"]), "new": parse_json(r["new"])} for r in rows]
|
|
140
|
+
|
|
141
|
+
def history(self, table: str, key: Any, *, branch: str = "main") -> list[dict]:
|
|
142
|
+
rows = self.query("select vkey, xid::text as xid, role, op, at, actor, reason, row_image::text as row_image, on_branch from gnomon.history(%s, %s::jsonb, %s)",
|
|
143
|
+
(table, json.dumps(key), branch))
|
|
144
|
+
return [{**r, "row_image": parse_json(r["row_image"])} for r in rows]
|
|
145
|
+
|
|
146
|
+
def blame(self, table: str, key: Any, *, branch: str = "main") -> list[dict]:
|
|
147
|
+
rows = self.query("select col, value::text as value, xid::text as xid, role, actor, reason, at, on_branch from gnomon.blame(%s, %s::jsonb, %s)",
|
|
148
|
+
(table, json.dumps(key), branch))
|
|
149
|
+
return [{**r, "value": parse_json(r["value"])} for r in rows]
|
|
150
|
+
|
|
151
|
+
def revert(self, table: str, at: str, *, keys: Optional[list] = None, filter: Optional[str] = None,
|
|
152
|
+
columns: Optional[list] = None, branch: str = "main", dry_run: bool = False) -> dict:
|
|
153
|
+
return self.json("select gnomon.revert(%s, %s, %s::jsonb, %s, %s::text[], %s, %s)",
|
|
154
|
+
(table, at, json.dumps(keys) if keys is not None else None, filter, columns, branch, dry_run))
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
class Run:
|
|
158
|
+
"""One agent run: a branch, its envelope, and what the application does with it."""
|
|
159
|
+
|
|
160
|
+
def __init__(self, gnomon: Gnomon, branch: str, *, task: Optional[str] = None, actor: Optional[str] = None):
|
|
161
|
+
self.gnomon, self.branch, self.task, self.actor = gnomon, branch, task, actor
|
|
162
|
+
self.state = "open"
|
|
163
|
+
self._seq = 0
|
|
164
|
+
|
|
165
|
+
def __enter__(self) -> "Run":
|
|
166
|
+
return self
|
|
167
|
+
|
|
168
|
+
def __exit__(self, *exc) -> None:
|
|
169
|
+
if self.state == "open":
|
|
170
|
+
self.discard()
|
|
171
|
+
|
|
172
|
+
def _check_open(self) -> None:
|
|
173
|
+
if self.state != "open":
|
|
174
|
+
raise GnomonError(f"run {self.branch} is {self.state}", code="branch_closed")
|
|
175
|
+
|
|
176
|
+
def _envelope(self, cur, reason: Optional[str], timeout_ms: Optional[int]) -> None:
|
|
177
|
+
g = self.gnomon
|
|
178
|
+
self._seq += 1
|
|
179
|
+
cur.execute("select gnomon.begin(%s)", (self.branch,))
|
|
180
|
+
cur.execute("select gnomon.note(%s, %s, %s::jsonb)", (self.actor or "agent", reason,
|
|
181
|
+
json.dumps({"run": self.branch, "call": self._seq, "task": self.task})))
|
|
182
|
+
cur.execute("select set_config('statement_timeout', %s, true), set_config('lock_timeout', %s, true)",
|
|
183
|
+
(str(timeout_ms or g.statement_timeout_ms), str(g.lock_timeout_ms)))
|
|
184
|
+
|
|
185
|
+
def sql(self, statement: str, params: tuple = (), *, reason: Optional[str] = None, timeout_ms: Optional[int] = None,
|
|
186
|
+
max_rows: Optional[int] = None) -> dict:
|
|
187
|
+
"""One statement on the run (the envelope). Rows past the cap: ``truncated``. Retried on a
|
|
188
|
+
serialization failure (the envelope held nothing else)."""
|
|
189
|
+
self._check_open()
|
|
190
|
+
g = self.gnomon
|
|
191
|
+
g.ready()
|
|
192
|
+
if len(split_sql(statement)) > 1:
|
|
193
|
+
raise GnomonError("one statement per call: this one holds several (split it, or use transaction())", code="one_statement", sqlstate="42601")
|
|
194
|
+
cap = max_rows or g.max_rows
|
|
195
|
+
for attempt in range(4):
|
|
196
|
+
try:
|
|
197
|
+
with g.runs.connection() as conn:
|
|
198
|
+
with conn.transaction():
|
|
199
|
+
with conn.cursor(row_factory=dict_row) as cur:
|
|
200
|
+
self._envelope(cur, reason, timeout_ms)
|
|
201
|
+
# (the extended protocol: psycopg sends parameters, so PostgreSQL refuses a second statement)
|
|
202
|
+
cur.execute(statement, params or None, prepare=False)
|
|
203
|
+
rows = cur.fetchall() if cur.description else []
|
|
204
|
+
rowcount = cur.rowcount
|
|
205
|
+
g._event("runs", self.branch, statement, True)
|
|
206
|
+
return {"rows": rows[:cap], "row_count": rowcount, "truncated": len(rows) > cap}
|
|
207
|
+
except psycopg.Error as e:
|
|
208
|
+
ge = to_gnomon_error(e)
|
|
209
|
+
g._event("runs", self.branch, statement, False, ge)
|
|
210
|
+
if isinstance(ge, SerializationError) and attempt < 3:
|
|
211
|
+
time.sleep(0.02 * 2 ** attempt)
|
|
212
|
+
continue
|
|
213
|
+
raise ge from e
|
|
214
|
+
|
|
215
|
+
@contextlib.contextmanager
|
|
216
|
+
def transaction(self, *, reason: Optional[str] = None, timeout_ms: Optional[int] = None) -> Iterator[Any]:
|
|
217
|
+
"""Several statements in one envelope: ``with run.transaction() as tx: tx.execute(...)``."""
|
|
218
|
+
self._check_open()
|
|
219
|
+
g = self.gnomon
|
|
220
|
+
g.ready()
|
|
221
|
+
try:
|
|
222
|
+
with g.runs.connection() as conn:
|
|
223
|
+
with conn.transaction():
|
|
224
|
+
with conn.cursor(row_factory=dict_row) as cur:
|
|
225
|
+
self._envelope(cur, reason, timeout_ms)
|
|
226
|
+
yield cur
|
|
227
|
+
except psycopg.Error as e:
|
|
228
|
+
raise to_gnomon_error(e) from e
|
|
229
|
+
|
|
230
|
+
def diff(self) -> list[dict]:
|
|
231
|
+
return self.gnomon.diff(self.branch)
|
|
232
|
+
|
|
233
|
+
def summary(self, *, run_checks: bool = True) -> dict:
|
|
234
|
+
return self.gnomon.json("select gnomon.run_summary(%s, %s)", (self.branch, run_checks))
|
|
235
|
+
|
|
236
|
+
def checks(self) -> list[dict]:
|
|
237
|
+
return self.gnomon.query("select check_name, passed, detail from gnomon.run_checks(%s)", (self.branch,))
|
|
238
|
+
|
|
239
|
+
def merge(self, *, close: bool = True) -> dict:
|
|
240
|
+
g = self.gnomon
|
|
241
|
+
mid = g.value("select gnomon.merge(%s, null, 'atomic', null, null, null, %s)", (self.branch, close))
|
|
242
|
+
m = g.json("select to_jsonb(m) from gnomon.merge m where id = %s", (mid,))
|
|
243
|
+
if m["status"] == "conflicts":
|
|
244
|
+
c = g.json("select coalesce(jsonb_agg(to_jsonb(c)), '[]') from gnomon.conflict c where merge_id = %s", (mid,))
|
|
245
|
+
raise MergeConflictError(f"merging run {self.branch} stopped on {len(c)} conflict(s)", mid, c)
|
|
246
|
+
self.state = "merged"
|
|
247
|
+
return {"merge": mid, "status": m["status"], "stats": m["stats"]}
|
|
248
|
+
|
|
249
|
+
def submit(self, title: Optional[str] = None, body: Optional[str] = None) -> "Request":
|
|
250
|
+
cr = self.gnomon.value("select gnomon.open_change_request(%s, %s, %s)",
|
|
251
|
+
(self.branch, title or self.task or f"Run {self.branch}", body))
|
|
252
|
+
self.state = "submitted"
|
|
253
|
+
return Request(self.gnomon, cr)
|
|
254
|
+
|
|
255
|
+
def discard(self) -> None:
|
|
256
|
+
if self.state == "discarded":
|
|
257
|
+
return
|
|
258
|
+
self.gnomon.query("select gnomon.run_end(%s, false)", (self.branch,))
|
|
259
|
+
self.state = "discarded"
|
|
260
|
+
|
|
261
|
+
def url(self, base: Optional[str] = None) -> str:
|
|
262
|
+
"""A connection string pinned to the run, for a tool that wants one: a login of the run's own
|
|
263
|
+
(gnomon.run_login), confined to the run's branch, dropped when the run ends."""
|
|
264
|
+
self._check_open()
|
|
265
|
+
b = base or self.gnomon.url
|
|
266
|
+
if not b:
|
|
267
|
+
raise GnomonError("run.url() needs a base connection string (the url option, or an argument)", code="run_url_base")
|
|
268
|
+
login = self.gnomon.json("select gnomon.run_login(%s)", (self.branch,))
|
|
269
|
+
return psycopg.conninfo.make_conninfo(b, user=login["role"], password=login["password"])
|
|
270
|
+
|
|
271
|
+
def connection(self, base: Optional[str] = None) -> psycopg.Connection:
|
|
272
|
+
"""A connection pinned to the run (close it when done)."""
|
|
273
|
+
return psycopg.Connection.connect(self.url(base))
|
|
274
|
+
|
|
275
|
+
def checkpoint(self, name: str) -> str:
|
|
276
|
+
return self.gnomon.value("select gnomon.checkpoint(%s, %s)", (f"{self.branch}:{name}", self.branch))
|
|
277
|
+
|
|
278
|
+
def rewind(self, name: str) -> dict:
|
|
279
|
+
return self.gnomon.json("select gnomon.rewind(%s, %s)", (self.branch, f"{self.branch}:{name}"))
|
|
280
|
+
|
|
281
|
+
def dry_run(self, statements: list[str], *, checks: bool = True) -> dict:
|
|
282
|
+
return self.gnomon.json("select gnomon.dry_run(%s::text[], %s, %s)", (statements, self.branch, checks))
|
|
283
|
+
|
|
284
|
+
def describe(self, schema: Optional[str] = None) -> dict:
|
|
285
|
+
with self.gnomon.runs.connection() as conn:
|
|
286
|
+
with conn.transaction():
|
|
287
|
+
with conn.cursor() as cur:
|
|
288
|
+
cur.execute("select gnomon.begin(%s)", (self.branch,))
|
|
289
|
+
cur.execute("select gnomon.describe(%s)::text", (schema,))
|
|
290
|
+
return parse_json((cur.fetchone())[0])
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
class Requests:
|
|
294
|
+
def __init__(self, g: Gnomon):
|
|
295
|
+
self.gnomon = g
|
|
296
|
+
|
|
297
|
+
def open(self) -> list["Request"]:
|
|
298
|
+
rows = self.gnomon.query("select id from gnomon.change_request where status in ('open', 'approved') order by id")
|
|
299
|
+
return [Request(self.gnomon, r["id"]) for r in rows]
|
|
300
|
+
|
|
301
|
+
def get(self, id: int) -> "Request":
|
|
302
|
+
return Request(self.gnomon, id)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
class Request:
|
|
306
|
+
"""A change request: what a product's own Approve, Reject and Undo buttons call."""
|
|
307
|
+
|
|
308
|
+
def __init__(self, g: Gnomon, id: int):
|
|
309
|
+
self.gnomon, self.id = g, id
|
|
310
|
+
|
|
311
|
+
def status(self) -> dict:
|
|
312
|
+
return self.gnomon.json("select gnomon.change_request_status(%s)", (self.id,))
|
|
313
|
+
|
|
314
|
+
def diff(self) -> list[dict]:
|
|
315
|
+
return self.gnomon.diff((self.status())["branch"])
|
|
316
|
+
|
|
317
|
+
def approve(self, body: Optional[str] = None) -> None:
|
|
318
|
+
self.gnomon.query("select gnomon.review(%s, 'approve', %s)", (self.id, body))
|
|
319
|
+
|
|
320
|
+
def reject(self, note: Optional[str] = None) -> None:
|
|
321
|
+
self.gnomon.query("select gnomon.review(%s, 'reject', %s)", (self.id, note))
|
|
322
|
+
|
|
323
|
+
def comment(self, body: str, *, table: Optional[str] = None, key: Any = None, column: Optional[str] = None) -> int:
|
|
324
|
+
return self.gnomon.value("select gnomon.comment(%s, %s, %s::regclass, %s::jsonb, %s)",
|
|
325
|
+
(self.id, body, table, json.dumps(key) if key is not None else None, column))
|
|
326
|
+
|
|
327
|
+
def merge(self, *, close: bool = True) -> int:
|
|
328
|
+
return self.gnomon.value("select gnomon.merge(%s, null, 'atomic', null, null, null, %s)", ((self.status())["branch"], close))
|
|
329
|
+
|
|
330
|
+
def undo(self) -> dict:
|
|
331
|
+
s = self.status()
|
|
332
|
+
if not s.get("merge"):
|
|
333
|
+
raise GnomonError(f"change request {self.id} is not merged")
|
|
334
|
+
return self.gnomon.json("select gnomon.undo_merge(%s)", (s["merge"]["id"],))
|
|
335
|
+
|
|
336
|
+
def wait(self, timeout_s: float = 300) -> dict:
|
|
337
|
+
return self.gnomon.json("select gnomon.wait_for_review(%s, %s::interval)", (self.id, f"{timeout_s} seconds"))
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
class Branches:
|
|
341
|
+
def __init__(self, g: Gnomon):
|
|
342
|
+
self.gnomon = g
|
|
343
|
+
|
|
344
|
+
def create(self, name: str, *, parent: str = "main", kind: str = "logged", at: Optional[str] = None) -> str:
|
|
345
|
+
self.gnomon.query("select gnomon.create_branch(%s, %s, %s, %s)", (name, parent, kind, at))
|
|
346
|
+
return name
|
|
347
|
+
|
|
348
|
+
def scratch(self, prefix: str = "test", parent: str = "main") -> str:
|
|
349
|
+
return self.gnomon.value("select gnomon.create_scratch(%s, %s)", (prefix, parent))
|
|
350
|
+
|
|
351
|
+
def drop(self, name: str) -> None:
|
|
352
|
+
self.gnomon.query("select gnomon.drop_branch(%s, true)", (name,))
|
|
353
|
+
|
|
354
|
+
def rebase(self, name: str, *, dry_run: bool = False) -> dict:
|
|
355
|
+
return self.gnomon.json("select gnomon.rebase(%s, null, %s)", (name, dry_run))
|
|
356
|
+
|
|
357
|
+
def list(self) -> list[dict]:
|
|
358
|
+
return self.gnomon.query("select * from gnomon.branches order by created_at")
|
|
359
|
+
|
|
360
|
+
def options(self, name: str, *, c_module: bool = False) -> str:
|
|
361
|
+
return self.gnomon.value("select gnomon.connection_options(%s, %s)", (name, c_module))
|
gnomon/django.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Django integration.
|
|
2
|
+
|
|
3
|
+
Add ``"gnomon.django.BranchMiddleware"`` to MIDDLEWARE. A request with an ``X-Gnomon-Branch``
|
|
4
|
+
header (or a ``request.gnomon_branch`` attribute set by an earlier middleware) runs on that branch.
|
|
5
|
+
|
|
6
|
+
* Default: the request's connection is put on the branch with gnomon.use() before the view and
|
|
7
|
+
reset to main afterwards, so a persistent connection never carries the branch to the next request.
|
|
8
|
+
* ``GNOMON_TRANSACTION_POOLING = True`` (behind pgbouncer in transaction mode): every transaction
|
|
9
|
+
of the request starts with gnomon.begin(), which lasts exactly one transaction. This needs
|
|
10
|
+
``ATOMIC_REQUESTS = True`` so the request's queries share a transaction.
|
|
11
|
+
|
|
12
|
+
``GNOMON_BRANCH_HEADER`` changes the header name.
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import contextvars
|
|
17
|
+
|
|
18
|
+
from django.conf import settings
|
|
19
|
+
from django.core.exceptions import ImproperlyConfigured
|
|
20
|
+
from django.db import connections
|
|
21
|
+
|
|
22
|
+
_current = contextvars.ContextVar("gnomon_branch", default=None)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def current_branch():
|
|
26
|
+
"""The branch of the request being served, or None."""
|
|
27
|
+
return _current.get()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _BeginEachTransaction:
|
|
31
|
+
def __init__(self, conn, branch):
|
|
32
|
+
self.conn, self.branch, self.done = conn, branch, set()
|
|
33
|
+
|
|
34
|
+
def __call__(self, execute, sql, params, many, context):
|
|
35
|
+
if not self.conn.in_atomic_block:
|
|
36
|
+
raise ImproperlyConfigured("GNOMON_TRANSACTION_POOLING needs ATOMIC_REQUESTS = True")
|
|
37
|
+
key = id(self.conn.atomic_blocks[0])
|
|
38
|
+
if key not in self.done:
|
|
39
|
+
self.done.add(key)
|
|
40
|
+
context["cursor"].execute("select gnomon.begin(%s)", [self.branch])
|
|
41
|
+
return execute(sql, params, many, context)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class BranchMiddleware:
|
|
45
|
+
def __init__(self, get_response):
|
|
46
|
+
self.get_response = get_response
|
|
47
|
+
name = getattr(settings, "GNOMON_BRANCH_HEADER", "X-Gnomon-Branch")
|
|
48
|
+
self.header = "HTTP_" + name.upper().replace("-", "_")
|
|
49
|
+
|
|
50
|
+
def __call__(self, request):
|
|
51
|
+
b = getattr(request, "gnomon_branch", None) or request.META.get(self.header)
|
|
52
|
+
if not b or b == "main":
|
|
53
|
+
return self.get_response(request)
|
|
54
|
+
conn = connections["default"]
|
|
55
|
+
token = _current.set(b)
|
|
56
|
+
try:
|
|
57
|
+
if getattr(settings, "GNOMON_TRANSACTION_POOLING", False):
|
|
58
|
+
with conn.execute_wrapper(_BeginEachTransaction(conn, b)):
|
|
59
|
+
return self.get_response(request)
|
|
60
|
+
with conn.cursor() as cur:
|
|
61
|
+
cur.execute("select gnomon.use(%s)", [b])
|
|
62
|
+
try:
|
|
63
|
+
return self.get_response(request)
|
|
64
|
+
finally:
|
|
65
|
+
with conn.cursor() as cur:
|
|
66
|
+
cur.execute("select gnomon.use('main')")
|
|
67
|
+
finally:
|
|
68
|
+
_current.reset(token)
|
gnomon/errors.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Typed errors (sdk/spec/SPEC.md, Errors). The SQLSTATE stays the standard one; the slug in the
|
|
2
|
+
hint (``[gnomon:<slug>]``) is gnomon's stable code."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from typing import Any, Optional
|
|
7
|
+
|
|
8
|
+
_SLUG = re.compile(r"\[gnomon:([a-z_]+)\]")
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class GnomonError(Exception):
|
|
12
|
+
def __init__(self, message: str, *, code: Optional[str] = None, sqlstate: Optional[str] = None,
|
|
13
|
+
hint: Optional[str] = None, cause: Optional[BaseException] = None):
|
|
14
|
+
super().__init__(message)
|
|
15
|
+
self.code, self.sqlstate, self.hint, self.cause = code, sqlstate, hint, cause
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SerializationError(GnomonError): ...
|
|
19
|
+
class PermissionError(GnomonError): ... # noqa: A001 (gnomon's, not the builtin)
|
|
20
|
+
class BranchClosedError(GnomonError): ...
|
|
21
|
+
class StatementTimeoutError(GnomonError): ...
|
|
22
|
+
class UnsupportedOnBranchError(GnomonError): ...
|
|
23
|
+
class NotRoutedError(GnomonError): ...
|
|
24
|
+
class OverQuotaError(GnomonError): ...
|
|
25
|
+
class IncompatibleVersionError(GnomonError): ...
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ChecksFailedError(GnomonError):
|
|
29
|
+
results: Any = None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class MergeConflictError(GnomonError):
|
|
33
|
+
def __init__(self, message: str, merge_id: int, conflicts: list):
|
|
34
|
+
super().__init__(message, code="merge_conflict")
|
|
35
|
+
self.merge_id, self.conflicts = merge_id, conflicts
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def to_gnomon_error(err: BaseException) -> GnomonError:
|
|
39
|
+
"""Map a psycopg error to a typed gnomon error."""
|
|
40
|
+
if isinstance(err, GnomonError):
|
|
41
|
+
return err
|
|
42
|
+
sqlstate = getattr(err, "sqlstate", None)
|
|
43
|
+
diag = getattr(err, "diag", None)
|
|
44
|
+
hint = getattr(diag, "message_hint", None) if diag is not None else None
|
|
45
|
+
msg = getattr(diag, "message_primary", None) or str(err)
|
|
46
|
+
m = _SLUG.search(f"{hint or ''} {msg}")
|
|
47
|
+
slug = m.group(1) if m else None
|
|
48
|
+
kw = dict(code=slug, sqlstate=sqlstate, hint=hint, cause=err)
|
|
49
|
+
if sqlstate == "40001":
|
|
50
|
+
return SerializationError(msg, **kw)
|
|
51
|
+
if sqlstate == "57014":
|
|
52
|
+
return StatementTimeoutError(msg, **kw)
|
|
53
|
+
if sqlstate == "42P10" or slug in ("sql_only_dml", "upsert_on_branch"):
|
|
54
|
+
return UnsupportedOnBranchError(msg + " (on a branch, write an upsert as UPDATE then INSERT ... WHERE NOT EXISTS, or load gnomon's C module)", **kw)
|
|
55
|
+
if slug == "not_routed":
|
|
56
|
+
return NotRoutedError(msg, **kw)
|
|
57
|
+
if sqlstate == "25006" or slug == "branch_closed":
|
|
58
|
+
return BranchClosedError(msg, **kw)
|
|
59
|
+
if sqlstate == "53400" or slug == "over_quota":
|
|
60
|
+
return OverQuotaError(msg, **kw)
|
|
61
|
+
if sqlstate == "23514":
|
|
62
|
+
return ChecksFailedError(msg, **kw)
|
|
63
|
+
if sqlstate == "42501":
|
|
64
|
+
return PermissionError(msg, **kw)
|
|
65
|
+
return GnomonError(msg, **kw)
|