pgdevkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pgdevkit/__init__.py +0 -0
- pgdevkit/cli.py +217 -0
- pgdevkit/connection.py +56 -0
- pgdevkit/db/__init__.py +37 -0
- pgdevkit/db/connection.py +74 -0
- pgdevkit/db/crud.py +184 -0
- pgdevkit/db/loader.py +19 -0
- pgdevkit/db/model.py +23 -0
- pgdevkit/diff.py +253 -0
- pgdevkit/docs/database-layout.md +145 -0
- pgdevkit/fetch_missing.py +229 -0
- pgdevkit/introspect.py +190 -0
- pgdevkit/lakebase.py +90 -0
- pgdevkit/models.py +102 -0
- pgdevkit/parser.py +336 -0
- pgdevkit/skills/pgdevkit/SKILL.md +283 -0
- pgdevkit/skills/pgdevkit/references/complex_helper.py +234 -0
- pgdevkit/skills/pgdevkit/references/dynamic-sql.md +92 -0
- pgdevkit/skills/pgdevkit/references/temporal-tables.md +33 -0
- pgdevkit/testdb/__init__.py +3 -0
- pgdevkit/testdb/api.py +137 -0
- pgdevkit/testdb/config.py +51 -0
- pgdevkit/testdb/constants.py +11 -0
- pgdevkit/testdb/container.py +95 -0
- pgdevkit/testdb/naming.py +41 -0
- pgdevkit/testdb/query.py +65 -0
- pgdevkit/testdb/schema.py +188 -0
- pgdevkit-0.1.0.dist-info/METADATA +140 -0
- pgdevkit-0.1.0.dist-info/RECORD +31 -0
- pgdevkit-0.1.0.dist-info/WHEEL +4 -0
- pgdevkit-0.1.0.dist-info/entry_points.txt +2 -0
pgdevkit/parser.py
ADDED
|
@@ -0,0 +1,336 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
import logging
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import sqlglot
|
|
8
|
+
import sqlglot.expressions as exp
|
|
9
|
+
|
|
10
|
+
from .models import (
|
|
11
|
+
ColumnDef, ConstraintDef, CompositeTypeDef, DatabaseSchema,
|
|
12
|
+
EnumDef, FunctionDef, IndexDef, TableDef, ViewDef,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
16
|
+
|
|
17
|
+
# Regex to extract dollar-quoted body
|
|
18
|
+
_DOLLAR_BODY = re.compile(r'\$(\w*)\$(.*?)\$\1\$', re.DOTALL | re.IGNORECASE)
|
|
19
|
+
|
|
20
|
+
# Regex for CREATE TYPE AS ENUM inside DO blocks
|
|
21
|
+
_DO_ENUM = re.compile(
|
|
22
|
+
r'CREATE\s+TYPE\s+(\w+(?:\.\w+)?)\s+AS\s+ENUM\s*\(([^)]+)\)',
|
|
23
|
+
re.IGNORECASE | re.DOTALL,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# Regex for CREATE TYPE AS composite inside DO blocks
|
|
27
|
+
_DO_COMPOSITE = re.compile(
|
|
28
|
+
r'CREATE\s+TYPE\s+(\w+(?:\.\w+)?)\s+AS\s*\(([^)]+)\)',
|
|
29
|
+
re.IGNORECASE | re.DOTALL,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def parse_directory(scripts_dir: Path) -> DatabaseSchema:
|
|
34
|
+
db_schema = DatabaseSchema()
|
|
35
|
+
for sql_file in sorted(scripts_dir.rglob("*.sql")):
|
|
36
|
+
_parse_file(sql_file, db_schema)
|
|
37
|
+
return db_schema
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _parse_file(path: Path, db_schema: DatabaseSchema) -> None:
|
|
41
|
+
content = path.read_text(encoding="utf-8")
|
|
42
|
+
try:
|
|
43
|
+
exprs = sqlglot.parse(content, dialect="postgres", error_level=sqlglot.ErrorLevel.WARN)
|
|
44
|
+
except Exception as e:
|
|
45
|
+
logger.warning("sqlglot failed on %s: %s", path.name, e)
|
|
46
|
+
exprs = []
|
|
47
|
+
|
|
48
|
+
for expr in exprs:
|
|
49
|
+
if expr is None:
|
|
50
|
+
continue
|
|
51
|
+
try:
|
|
52
|
+
_handle_expr(expr, content, db_schema)
|
|
53
|
+
except Exception as e:
|
|
54
|
+
logger.debug("Skipping expression in %s: %s", path.name, e)
|
|
55
|
+
|
|
56
|
+
_extract_do_block_objects(content, db_schema)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _handle_expr(expr: exp.Expression, raw: str, db_schema: DatabaseSchema) -> None:
|
|
60
|
+
if not isinstance(expr, exp.Create):
|
|
61
|
+
return
|
|
62
|
+
kind = (expr.args.get("kind") or "").upper()
|
|
63
|
+
if kind == "TABLE":
|
|
64
|
+
_handle_table(expr, db_schema)
|
|
65
|
+
elif kind == "VIEW":
|
|
66
|
+
_handle_view(expr, db_schema)
|
|
67
|
+
elif kind in ("FUNCTION", "PROCEDURE"):
|
|
68
|
+
_handle_function(expr, raw, db_schema, kind.lower())
|
|
69
|
+
elif kind == "TYPE":
|
|
70
|
+
_handle_type(expr, db_schema)
|
|
71
|
+
elif kind == "SCHEMA":
|
|
72
|
+
_handle_schema_create(expr, db_schema)
|
|
73
|
+
elif kind == "INDEX":
|
|
74
|
+
_handle_index(expr, db_schema)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _resolve_name(expr: exp.Create) -> tuple[str, str] | None:
|
|
78
|
+
"""Return (schema, name) from a CREATE expression."""
|
|
79
|
+
this = expr.this
|
|
80
|
+
if isinstance(this, exp.Schema):
|
|
81
|
+
table_node = this.this
|
|
82
|
+
else:
|
|
83
|
+
table_node = this
|
|
84
|
+
|
|
85
|
+
if isinstance(table_node, exp.Table):
|
|
86
|
+
db_node = table_node.args.get("db")
|
|
87
|
+
schema = db_node.name if db_node else "public"
|
|
88
|
+
return schema, table_node.name
|
|
89
|
+
return None
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _handle_table(expr: exp.Create, db_schema: DatabaseSchema) -> None:
|
|
93
|
+
result = _resolve_name(expr)
|
|
94
|
+
if not result:
|
|
95
|
+
return
|
|
96
|
+
tschema, tname = result
|
|
97
|
+
properties = expr.args.get("properties")
|
|
98
|
+
is_partition = any(
|
|
99
|
+
isinstance(p, exp.PartitionedOfProperty) for p in (properties.expressions if properties else [])
|
|
100
|
+
)
|
|
101
|
+
table = TableDef(schema=tschema, name=tname, is_partition=is_partition)
|
|
102
|
+
|
|
103
|
+
this = expr.this
|
|
104
|
+
items = this.expressions if isinstance(this, exp.Schema) else []
|
|
105
|
+
|
|
106
|
+
pk_columns: set[str] = set()
|
|
107
|
+
for item in items:
|
|
108
|
+
if isinstance(item, exp.ColumnDef):
|
|
109
|
+
col = _parse_column_def(item)
|
|
110
|
+
if col:
|
|
111
|
+
table.columns.append(col)
|
|
112
|
+
else:
|
|
113
|
+
constr = _parse_table_constraint(item)
|
|
114
|
+
if constr:
|
|
115
|
+
table.constraints.append(constr)
|
|
116
|
+
pk_columns |= _extract_primary_key_columns(item)
|
|
117
|
+
|
|
118
|
+
for col in table.columns:
|
|
119
|
+
if col.name in pk_columns:
|
|
120
|
+
col.is_nullable = False
|
|
121
|
+
|
|
122
|
+
db_schema.tables[table.qualified_name] = table
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _extract_primary_key_columns(item: exp.Expression) -> set[str]:
|
|
126
|
+
inner = item
|
|
127
|
+
if isinstance(item, exp.Constraint):
|
|
128
|
+
inner = item.args.get("kind") or (item.expressions[0] if item.expressions else None)
|
|
129
|
+
if isinstance(inner, exp.PrimaryKey):
|
|
130
|
+
return {e.name for e in inner.expressions if hasattr(e, "name")}
|
|
131
|
+
return set()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _parse_column_def(col: exp.ColumnDef) -> ColumnDef | None:
|
|
135
|
+
name = col.name
|
|
136
|
+
if not name or col.kind is None:
|
|
137
|
+
return None
|
|
138
|
+
data_type = col.kind.sql(dialect="postgres").lower()
|
|
139
|
+
is_serial = col.kind.this in (
|
|
140
|
+
exp.DataType.Type.SERIAL, exp.DataType.Type.SMALLSERIAL, exp.DataType.Type.BIGSERIAL,
|
|
141
|
+
)
|
|
142
|
+
is_nullable = not is_serial
|
|
143
|
+
default = None
|
|
144
|
+
is_generated = False
|
|
145
|
+
|
|
146
|
+
for c in col.constraints:
|
|
147
|
+
ck = c.kind
|
|
148
|
+
if isinstance(ck, (exp.NotNullColumnConstraint, exp.PrimaryKeyColumnConstraint)):
|
|
149
|
+
is_nullable = False
|
|
150
|
+
elif isinstance(ck, exp.DefaultColumnConstraint):
|
|
151
|
+
default = ck.this.sql(dialect="postgres") if ck.this else None
|
|
152
|
+
elif isinstance(ck, exp.GeneratedAsIdentityColumnConstraint):
|
|
153
|
+
is_generated = True
|
|
154
|
+
is_nullable = False
|
|
155
|
+
elif isinstance(ck, exp.ComputedColumnConstraint):
|
|
156
|
+
is_generated = True
|
|
157
|
+
|
|
158
|
+
return ColumnDef(name=name, data_type=data_type, is_nullable=is_nullable, default=default, is_generated=is_generated)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _parse_table_constraint(item: exp.Expression) -> ConstraintDef | None:
|
|
162
|
+
name = None
|
|
163
|
+
kind = "UNKNOWN"
|
|
164
|
+
|
|
165
|
+
if isinstance(item, exp.Constraint):
|
|
166
|
+
name = item.name or None
|
|
167
|
+
inner = item.args.get("kind") or (item.expressions[0] if item.expressions else None)
|
|
168
|
+
else:
|
|
169
|
+
inner = item
|
|
170
|
+
|
|
171
|
+
if isinstance(inner, exp.PrimaryKey):
|
|
172
|
+
kind = "PRIMARY KEY"
|
|
173
|
+
elif isinstance(inner, exp.UniqueColumnConstraint):
|
|
174
|
+
kind = "UNIQUE"
|
|
175
|
+
elif isinstance(inner, exp.ForeignKey):
|
|
176
|
+
kind = "FOREIGN KEY"
|
|
177
|
+
elif isinstance(inner, exp.Check):
|
|
178
|
+
kind = "CHECK"
|
|
179
|
+
else:
|
|
180
|
+
return None
|
|
181
|
+
|
|
182
|
+
definition = item.sql(dialect="postgres").lower()
|
|
183
|
+
return ConstraintDef(name=name, kind=kind, definition=definition)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _handle_view(expr: exp.Create, db_schema: DatabaseSchema) -> None:
|
|
187
|
+
result = _resolve_name(expr)
|
|
188
|
+
if not result:
|
|
189
|
+
return
|
|
190
|
+
vschema, vname = result
|
|
191
|
+
query = expr.expression
|
|
192
|
+
definition = query.sql(dialect="postgres").lower() if query else ""
|
|
193
|
+
view = ViewDef(schema=vschema, name=vname, definition=definition)
|
|
194
|
+
db_schema.views[view.qualified_name] = view
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _handle_function(expr: exp.Create, raw: str, db_schema: DatabaseSchema, kind: str) -> None:
|
|
198
|
+
# Get name/schema from sqlglot
|
|
199
|
+
func_node = expr.this
|
|
200
|
+
fname = func_node.name if hasattr(func_node, "name") else ""
|
|
201
|
+
if fname:
|
|
202
|
+
db_node = func_node.args.get("db") if hasattr(func_node, "args") else None
|
|
203
|
+
fschema = db_node.name if db_node else "public"
|
|
204
|
+
else:
|
|
205
|
+
# sqlglot (30.11.0) parses "CREATE FUNCTION myapp.greet(...)" as a
|
|
206
|
+
# UserDefinedFunction wrapping a Table (this=Identifier(greet),
|
|
207
|
+
# db=Identifier(myapp)). UserDefinedFunction.name doesn't unwrap
|
|
208
|
+
# that nested Table, so pull the name/schema from it directly.
|
|
209
|
+
inner = func_node.this if hasattr(func_node, "this") else None
|
|
210
|
+
if isinstance(inner, exp.Table) and inner.name:
|
|
211
|
+
fname = inner.name
|
|
212
|
+
db_node = inner.args.get("db")
|
|
213
|
+
fschema = db_node.name if db_node else "public"
|
|
214
|
+
else:
|
|
215
|
+
result = _resolve_name(expr)
|
|
216
|
+
if not result:
|
|
217
|
+
return
|
|
218
|
+
fschema, fname = result
|
|
219
|
+
|
|
220
|
+
# Extract args, return type, language, body with regex on raw SQL
|
|
221
|
+
args, return_type, language, body = _parse_function_details(raw)
|
|
222
|
+
|
|
223
|
+
func = FunctionDef(
|
|
224
|
+
schema=fschema,
|
|
225
|
+
name=fname,
|
|
226
|
+
args=args,
|
|
227
|
+
return_type=return_type,
|
|
228
|
+
language=language,
|
|
229
|
+
body=body,
|
|
230
|
+
kind=kind,
|
|
231
|
+
)
|
|
232
|
+
db_schema.functions[func.qualified_name] = func
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
_FUNC_SIG = re.compile(
|
|
236
|
+
r'CREATE\s+(?:OR\s+REPLACE\s+)?(?:FUNCTION|PROCEDURE)\s+'
|
|
237
|
+
r'(?:\w+\.)?(\w+)\s*\(([^)]*(?:\([^)]*\)[^)]*)*)\)\s*'
|
|
238
|
+
r'(?:RETURNS\s+((?:TABLE\s*\([^)]+\)|SETOF\s+\S+|\S+)))?\s*'
|
|
239
|
+
r'LANGUAGE\s+(\w+)',
|
|
240
|
+
re.IGNORECASE | re.DOTALL,
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _parse_function_details(sql: str) -> tuple[str, str, str, str]:
|
|
245
|
+
args, return_type, language, body = "", "", "", ""
|
|
246
|
+
|
|
247
|
+
m = _FUNC_SIG.search(sql)
|
|
248
|
+
if m:
|
|
249
|
+
args = re.sub(r'\s+', ' ', m.group(2) or "").strip().lower()
|
|
250
|
+
return_type = (m.group(3) or "").strip().lower()
|
|
251
|
+
language = (m.group(4) or "").strip().lower()
|
|
252
|
+
|
|
253
|
+
dm = _DOLLAR_BODY.search(sql)
|
|
254
|
+
if dm:
|
|
255
|
+
raw_body = dm.group(2)
|
|
256
|
+
lines = [l.strip() for l in raw_body.splitlines()]
|
|
257
|
+
body = "\n".join(l.lower() for l in lines if l)
|
|
258
|
+
|
|
259
|
+
return args, return_type, language, body
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _handle_type(expr: exp.Create, db_schema: DatabaseSchema) -> None:
|
|
263
|
+
result = _resolve_name(expr)
|
|
264
|
+
if not result:
|
|
265
|
+
return
|
|
266
|
+
tschema, tname = result
|
|
267
|
+
|
|
268
|
+
expression = expr.expression
|
|
269
|
+
if expression is None:
|
|
270
|
+
return
|
|
271
|
+
|
|
272
|
+
if isinstance(expression, exp.DataType) and expression.this == exp.DataType.Type.ENUM:
|
|
273
|
+
values = [lit.name for lit in expression.expressions if isinstance(lit, exp.Literal)]
|
|
274
|
+
enum = EnumDef(schema=tschema, name=tname, values=values)
|
|
275
|
+
db_schema.enums[enum.qualified_name] = enum
|
|
276
|
+
elif isinstance(expression, exp.Schema):
|
|
277
|
+
# Composite type: fields are ColumnDef-like
|
|
278
|
+
fields = []
|
|
279
|
+
for col in expression.expressions:
|
|
280
|
+
if isinstance(col, exp.ColumnDef) and col.kind:
|
|
281
|
+
fields.append((col.name, col.kind.sql(dialect="postgres").lower()))
|
|
282
|
+
comp = CompositeTypeDef(schema=tschema, name=tname, fields=fields)
|
|
283
|
+
db_schema.composites[comp.qualified_name] = comp
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _handle_schema_create(expr: exp.Create, db_schema: DatabaseSchema) -> None:
|
|
287
|
+
this = expr.this
|
|
288
|
+
name = this.name if hasattr(this, "name") else ""
|
|
289
|
+
if not name:
|
|
290
|
+
# sqlglot (30.11.0) parses "CREATE SCHEMA myapp" as a Table node
|
|
291
|
+
# whose `db` arg holds the schema name and `.name` (the table
|
|
292
|
+
# identifier) is empty.
|
|
293
|
+
db_node = this.args.get("db") if hasattr(this, "args") else None
|
|
294
|
+
name = db_node.name if db_node else ""
|
|
295
|
+
if name:
|
|
296
|
+
db_schema.schemas.add(name)
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def _handle_index(expr: exp.Create, db_schema: DatabaseSchema) -> None:
|
|
300
|
+
this = expr.this
|
|
301
|
+
index_name = this.name if hasattr(this, "name") else ""
|
|
302
|
+
table_node = expr.find(exp.Table)
|
|
303
|
+
if not table_node:
|
|
304
|
+
return
|
|
305
|
+
db_node = table_node.args.get("db")
|
|
306
|
+
tschema = db_node.name if db_node else "public"
|
|
307
|
+
tname = table_node.name
|
|
308
|
+
definition = expr.sql(dialect="postgres").lower()
|
|
309
|
+
idx = IndexDef(schema=tschema, table=tname, name=index_name, definition=definition)
|
|
310
|
+
db_schema.indexes[idx.qualified_name] = idx
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _extract_do_block_objects(sql: str, db_schema: DatabaseSchema) -> None:
|
|
314
|
+
"""Extract CREATE TYPE statements from DO $$ ... $$ blocks using regex."""
|
|
315
|
+
for dm in _DOLLAR_BODY.finditer(sql):
|
|
316
|
+
block = dm.group(2)
|
|
317
|
+
for m in _DO_ENUM.finditer(block):
|
|
318
|
+
qualified = m.group(1)
|
|
319
|
+
parts = qualified.split(".")
|
|
320
|
+
tschema, tname = (parts[0], parts[1]) if len(parts) == 2 else ("public", parts[0])
|
|
321
|
+
values_raw = m.group(2)
|
|
322
|
+
values = [v.strip().strip("'\"") for v in values_raw.split(",") if v.strip()]
|
|
323
|
+
enum = EnumDef(schema=tschema, name=tname, values=values)
|
|
324
|
+
db_schema.enums.setdefault(enum.qualified_name, enum)
|
|
325
|
+
for m in _DO_COMPOSITE.finditer(block):
|
|
326
|
+
qualified = m.group(1)
|
|
327
|
+
parts = qualified.split(".")
|
|
328
|
+
tschema, tname = (parts[0], parts[1]) if len(parts) == 2 else ("public", parts[0])
|
|
329
|
+
fields_raw = m.group(2)
|
|
330
|
+
fields = []
|
|
331
|
+
for field_def in fields_raw.split(","):
|
|
332
|
+
parts2 = field_def.strip().split()
|
|
333
|
+
if len(parts2) >= 2:
|
|
334
|
+
fields.append((parts2[0], " ".join(parts2[1:]).lower()))
|
|
335
|
+
comp = CompositeTypeDef(schema=tschema, name=tname, fields=fields)
|
|
336
|
+
db_schema.composites.setdefault(comp.qualified_name, comp)
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: pgdevkit
|
|
3
|
+
plugin: coding
|
|
4
|
+
description: >
|
|
5
|
+
Use pgdevkit for any PostgreSQL work in a Python project: a local
|
|
6
|
+
Docker/Podman test database (`pgdb testdb`), importable ORM-free CRUD
|
|
7
|
+
helpers (`pgdevkit.db`), and the `database/`-folder schema-as-code
|
|
8
|
+
convention. Supersedes the old postgres-test-setup, postgres-best-practices,
|
|
9
|
+
and database-in-source skills — pgdevkit is a real dependency now, not
|
|
10
|
+
copy-pasted reference files. Use whenever the user wants to set up a local
|
|
11
|
+
test Postgres, write or review psycopg code, add a table/view/function to
|
|
12
|
+
a `database/` folder, or asks things like "add a database query", "create
|
|
13
|
+
a repository", "set up a test database", "reset the database", "add a
|
|
14
|
+
table", "migration script", "backfill missing objects", or mentions
|
|
15
|
+
psycopg/psycopg2/asyncpg/SQLAlchemy where the user seems open to a
|
|
16
|
+
different approach.
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
# pgdevkit — Postgres for Python projects
|
|
20
|
+
|
|
21
|
+
One dependency covers three things that used to be three separate skills
|
|
22
|
+
with copy-pasted reference files:
|
|
23
|
+
|
|
24
|
+
| Old skill | Replaced by |
|
|
25
|
+
|---|---|
|
|
26
|
+
| `postgres-test-setup` | `pgdb testdb` (container + schema apply) |
|
|
27
|
+
| `postgres-best-practices` | `pgdevkit.db` (importable CRUD helpers) |
|
|
28
|
+
| `database-in-source` | The `database/` folder convention — see [docs/database-layout.md](../../docs/database-layout.md) |
|
|
29
|
+
|
|
30
|
+
Core rules for every piece of database code in a project using pgdevkit:
|
|
31
|
+
|
|
32
|
+
- **No ORM** — use [psycopg](https://www.psycopg.org/psycopg3/) directly, via `pgdevkit.db`'s helpers or hand-written queries.
|
|
33
|
+
- **Inline SQL** — trivial queries of **4 lines or fewer** may be written inline in Python. Anything with JOINs, subqueries, CTEs, aggregations, or multiple conditions lives in its own `.sql` file.
|
|
34
|
+
- **Named parameters** — always `%(name)s` style, never positional `%s`.
|
|
35
|
+
- **The `database/` folder is the source of truth for the schema** — see [docs/database-layout.md](../../docs/database-layout.md) for the layer/object-type/file-naming conventions.
|
|
36
|
+
- **Result mapping** — every query result maps to a Pydantic model; table-mapped models extend `pgdevkit.db.PostgresTableModel`.
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
uv add pgdevkit[cli,db]
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
`cli` pulls in `typer`/`rich` for the `pgdb` command; `db` pulls in `pydantic`/`psycopg-pool` for the importable CRUD helpers. Skip either extra if the project doesn't need it (e.g. a project using only `pgdb testdb` doesn't need `db`).
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## Local test database — `pgdb testdb`
|
|
51
|
+
|
|
52
|
+
Add to `pyproject.toml`:
|
|
53
|
+
|
|
54
|
+
```toml
|
|
55
|
+
[tool.pgdevkit]
|
|
56
|
+
database_dir = "database" # optional, defaults to "database"
|
|
57
|
+
env_prefix = "MDM_" # optional, defaults to "{name.upper()}_"
|
|
58
|
+
extensions = ["vector"] # optional, CREATE EXTENSION IF NOT EXISTS
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
`name` is optional too — it falls back to the repo directory name.
|
|
62
|
+
|
|
63
|
+
In `tests/conftest.py`:
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
import pytest
|
|
67
|
+
from pgdevkit.testdb import ensure_testdb
|
|
68
|
+
|
|
69
|
+
@pytest.fixture(scope="session", autouse=True)
|
|
70
|
+
def _testdb_env():
|
|
71
|
+
env = ensure_testdb()
|
|
72
|
+
for key, value in env.items():
|
|
73
|
+
os.environ[key] = value
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
`ensure_testdb()` starts the shared `pgdevkit-postgres` container if needed (via `podman`), creates a database scoped to this project+branch (so different worktrees/branches never collide), and applies every `.sql` file under `database_dir` in dependency order, seeding any `.test_data.json` sidecar files.
|
|
77
|
+
|
|
78
|
+
| What do you need? | Command |
|
|
79
|
+
|---|---|
|
|
80
|
+
| First-time setup / apply new files | `pgdb testdb up` |
|
|
81
|
+
| Breaking change (rename/drop column) | `pgdb testdb reset` |
|
|
82
|
+
| Inspect test DB data | `pgdb testdb run-sql --sql "SELECT ..." --results` |
|
|
83
|
+
| Re-apply one file (e.g. a function/view) | `pgdb testdb run-sql database/path/to/file.sql` |
|
|
84
|
+
| Drop this workspace's database | `pgdb testdb clean` |
|
|
85
|
+
| Drop every database for this project (all branches) | `pgdb testdb clean --all` |
|
|
86
|
+
|
|
87
|
+
### CI
|
|
88
|
+
|
|
89
|
+
Podman needs to be available on the runner (`apt-get install -y podman` on `ubuntu-latest` if not preinstalled) — `ensure_testdb()`/`ensure_container()` shell out to it directly. There's no `PGDEVKIT_SKIP_CONTAINER` + service-container escape hatch wired through every fixture yet — if a project's CI can't run podman, it needs its own workaround for now.
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Application-side DB code — `pgdevkit.db`
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
app/
|
|
97
|
+
├── db/
|
|
98
|
+
│ ├── queries/
|
|
99
|
+
│ │ ├── users/
|
|
100
|
+
│ │ │ ├── get_user_by_id.sql
|
|
101
|
+
│ │ │ └── list_active_users.sql
|
|
102
|
+
│ └── repositories/
|
|
103
|
+
│ └── user_repository.py
|
|
104
|
+
├── models/
|
|
105
|
+
│ └── user_models.py # Pydantic models for the user domain
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
SQL files live under `db/queries/<topic>/`. Every custom query gets its own file — no multi-statement files that lump unrelated queries together.
|
|
109
|
+
|
|
110
|
+
### Connection pool
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
# db/connection.py
|
|
114
|
+
from pgdevkit.db import PgPool
|
|
115
|
+
|
|
116
|
+
pool = PgPool(env_prefix="APP_POSTGRES_") # matches ensure_testdb()'s {env_prefix}POSTGRES_* vars
|
|
117
|
+
|
|
118
|
+
async def startup():
|
|
119
|
+
await pool.open()
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Models
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
# models/user_models.py
|
|
126
|
+
from __future__ import annotations
|
|
127
|
+
from datetime import datetime
|
|
128
|
+
from pydantic import BaseModel, ConfigDict
|
|
129
|
+
from pgdevkit.db import PostgresTableModel
|
|
130
|
+
|
|
131
|
+
class UserRow(PostgresTableModel):
|
|
132
|
+
model_config = ConfigDict(from_attributes=True)
|
|
133
|
+
id: int
|
|
134
|
+
email: str
|
|
135
|
+
display_name: str
|
|
136
|
+
created_at: datetime
|
|
137
|
+
|
|
138
|
+
@staticmethod
|
|
139
|
+
def get_table_name() -> tuple[str, str]:
|
|
140
|
+
return ("public", "users")
|
|
141
|
+
|
|
142
|
+
@staticmethod
|
|
143
|
+
def get_primary_key() -> list[str]:
|
|
144
|
+
return ["id"]
|
|
145
|
+
|
|
146
|
+
class UserSummary(BaseModel):
|
|
147
|
+
model_config = ConfigDict(from_attributes=True)
|
|
148
|
+
id: int
|
|
149
|
+
display_name: str
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Models that represent partial results (joins, aggregations, partial selects) extend `BaseModel` directly instead of `PostgresTableModel`.
|
|
153
|
+
|
|
154
|
+
### CRUD helpers
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
from pgdevkit.db import pg_retrieve, pg_insert, pg_upsert, pg_delete
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
| Helper | Purpose |
|
|
161
|
+
|--------|---------|
|
|
162
|
+
| `pg_retrieve` | Fetch single row by PK |
|
|
163
|
+
| `pg_retrieve_many` | Fetch rows matching filter dict |
|
|
164
|
+
| `pg_insert` | Insert one row, `RETURNING *` |
|
|
165
|
+
| `pg_update` / `pg_update_dict` | Update by PK |
|
|
166
|
+
| `pg_upsert` / `pg_upsert_dict` | `INSERT ... ON CONFLICT ... DO UPDATE` |
|
|
167
|
+
| `pg_upsert_many` / `pg_upsert_many_dict` | Batch upsert via `executemany` |
|
|
168
|
+
| `pg_insert_many` | Batch insert via `executemany` |
|
|
169
|
+
| `pg_delete` / `pg_delete_dict` | Delete by PK, returns deleted row |
|
|
170
|
+
|
|
171
|
+
Use these for simple CRUD. For custom `WHERE` clauses, joins, aggregations, or ordering, write a dedicated `.sql` file and a repository method.
|
|
172
|
+
|
|
173
|
+
### Loading `.sql` files
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
# db/loader.py
|
|
177
|
+
from pathlib import Path
|
|
178
|
+
from pgdevkit.db import SqlLoader
|
|
179
|
+
|
|
180
|
+
sql = SqlLoader(Path(__file__).parent / "queries")
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
# db/repositories/user_repository.py
|
|
185
|
+
from psycopg.rows import dict_row
|
|
186
|
+
from pgdevkit.db import pg_retrieve, pg_delete
|
|
187
|
+
from db.connection import pool
|
|
188
|
+
from db.loader import sql
|
|
189
|
+
from models.user_models import UserRow, UserSummary
|
|
190
|
+
|
|
191
|
+
class UserRepository:
|
|
192
|
+
async def get_by_id(self, user_id: int) -> UserRow | None:
|
|
193
|
+
async with pool.connection() as conn:
|
|
194
|
+
return await pg_retrieve(conn, UserRow, {"id": user_id})
|
|
195
|
+
|
|
196
|
+
async def list_active(self, limit: int = 100) -> list[UserSummary]:
|
|
197
|
+
async with pool.connection() as conn:
|
|
198
|
+
async with conn.cursor(row_factory=dict_row) as cur:
|
|
199
|
+
await cur.execute(sql.load_sql("users", "list_active_users"), {"limit": limit})
|
|
200
|
+
rows = await cur.fetchall()
|
|
201
|
+
return [UserSummary.model_validate(r) for r in rows]
|
|
202
|
+
|
|
203
|
+
async def delete(self, user: UserRow) -> UserRow | None:
|
|
204
|
+
async with pool.connection() as conn:
|
|
205
|
+
return await pg_delete(conn, user, UserRow)
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
Named parameters, `%(name)s` style, dict argument — never positional `%s`, never f-strings or `str.format()` for SQL text.
|
|
209
|
+
|
|
210
|
+
### Dynamic SQL
|
|
211
|
+
|
|
212
|
+
Avoid it whenever possible — a static `.sql` file is always clearer. See [`references/dynamic-sql.md`](references/dynamic-sql.md) for t-string templates (3.14+) and `psycopg.sql` (< 3.14) when column/table names genuinely vary at runtime.
|
|
213
|
+
|
|
214
|
+
### Avoid `LATERAL JOIN` — use a CTE instead
|
|
215
|
+
|
|
216
|
+
```sql
|
|
217
|
+
with latest_order as (
|
|
218
|
+
select
|
|
219
|
+
o.user_id,
|
|
220
|
+
o.total,
|
|
221
|
+
row_number() over (partition by o.user_id order by o.created_at desc) as rn
|
|
222
|
+
from orders as o
|
|
223
|
+
)
|
|
224
|
+
select u.id, u.email, lo.total
|
|
225
|
+
from users as u
|
|
226
|
+
join latest_order as lo on lo.user_id = u.id and lo.rn = 1
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
### Temporal tables
|
|
230
|
+
|
|
231
|
+
See [`references/temporal-tables.md`](references/temporal-tables.md) for row-level history via `nearform/temporal_tables`.
|
|
232
|
+
|
|
233
|
+
### Custom Postgres types in test data
|
|
234
|
+
|
|
235
|
+
`pgdevkit.testdb.schema`'s test-data seeding handles plain columns and JSONB, but not composite types or enums directly. See [`references/complex_helper.py`](references/complex_helper.py) for a psycopg adapter (`ComplexHelper`) if a project needs that.
|
|
236
|
+
|
|
237
|
+
### SQL formatting
|
|
238
|
+
|
|
239
|
+
```bash
|
|
240
|
+
uv add --dev shandy-sqlfmt[jinjafmt]
|
|
241
|
+
sqlfmt db/queries/ # format
|
|
242
|
+
sqlfmt --check db/queries/ # CI check
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
---
|
|
246
|
+
|
|
247
|
+
## The `database/` folder & backfilling untracked objects
|
|
248
|
+
|
|
249
|
+
See [docs/database-layout.md](../../docs/database-layout.md) for the full convention: layer directories, object-type subfolders and their apply order, file-naming rules (`.test_data.json`, `.init.sql`, `.prod`), and how migrations are organised.
|
|
250
|
+
|
|
251
|
+
If a table, view, or function was created directly on the database and never got a `.sql` file:
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
pgdb fetch-missing database/ --url postgresql://... # dry run, lists what's missing
|
|
255
|
+
pgdb fetch-missing database/ --url postgresql://... --write
|
|
256
|
+
```
|
|
257
|
+
|
|
258
|
+
It diffs the live schema against `database/`, reverse-engineers DDL for anything untracked, and writes it into the matching layer folder's `tables/`, `views/`, `scalar_functions/`, or `table_functions/` subfolder (matched by schema name against existing top-level directories, ignoring their leading sort number).
|
|
259
|
+
|
|
260
|
+
---
|
|
261
|
+
|
|
262
|
+
## Comparing scripts to a live database
|
|
263
|
+
|
|
264
|
+
```bash
|
|
265
|
+
pgdb compare --url postgresql://... database/
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
Reports drift between the `database/` `.sql` files and the actual schema — tables, views, functions, enums, composite types, and indexes. Pass `--report-extra-db` to also flag objects that exist in the database but aren't tracked (this is what `pgdb fetch-missing` uses internally).
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
## Quick checklist
|
|
273
|
+
|
|
274
|
+
- [ ] `[tool.pgdevkit]` configured in `pyproject.toml`; `tests/conftest.py` calls `ensure_testdb()`
|
|
275
|
+
- [ ] New table/view/function/type gets its own `.sql` file under the right layer + object-type folder (see [docs/database-layout.md](../../docs/database-layout.md))
|
|
276
|
+
- [ ] Simple CRUD uses `pgdevkit.db`'s `pg_*` helpers; custom queries use `.sql` files loaded via `SqlLoader`
|
|
277
|
+
- [ ] Inline SQL only for trivial queries ≤ 4 lines; anything with JOINs/CTEs/aggregations/subqueries uses a `.sql` file
|
|
278
|
+
- [ ] All parameters use `%(name)s` style with a dict argument
|
|
279
|
+
- [ ] Results mapped to a Pydantic model; table-mapped models extend `PostgresTableModel`
|
|
280
|
+
- [ ] No `LATERAL JOIN` — use a CTE that groups/aggregates first, then joins it
|
|
281
|
+
- [ ] `.prod` files are production-only and skipped by `pgdb testdb`
|
|
282
|
+
- [ ] Every table (and non-obvious column) has a `COMMENT ON`, placed in the object's own `.sql` file
|
|
283
|
+
- [ ] Untracked DB objects backfilled via `pgdb fetch-missing`, not left undocumented
|