pgdevkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pgdevkit/parser.py ADDED
@@ -0,0 +1,336 @@
1
+ from __future__ import annotations
2
+
3
+ import re
4
+ import logging
5
+ from pathlib import Path
6
+
7
+ import sqlglot
8
+ import sqlglot.expressions as exp
9
+
10
+ from .models import (
11
+ ColumnDef, ConstraintDef, CompositeTypeDef, DatabaseSchema,
12
+ EnumDef, FunctionDef, IndexDef, TableDef, ViewDef,
13
+ )
14
+
15
+ logger = logging.getLogger(__name__)
16
+
17
+ # Regex to extract dollar-quoted body
18
+ _DOLLAR_BODY = re.compile(r'\$(\w*)\$(.*?)\$\1\$', re.DOTALL | re.IGNORECASE)
19
+
20
+ # Regex for CREATE TYPE AS ENUM inside DO blocks
21
+ _DO_ENUM = re.compile(
22
+ r'CREATE\s+TYPE\s+(\w+(?:\.\w+)?)\s+AS\s+ENUM\s*\(([^)]+)\)',
23
+ re.IGNORECASE | re.DOTALL,
24
+ )
25
+
26
+ # Regex for CREATE TYPE AS composite inside DO blocks
27
+ _DO_COMPOSITE = re.compile(
28
+ r'CREATE\s+TYPE\s+(\w+(?:\.\w+)?)\s+AS\s*\(([^)]+)\)',
29
+ re.IGNORECASE | re.DOTALL,
30
+ )
31
+
32
+
33
+ def parse_directory(scripts_dir: Path) -> DatabaseSchema:
34
+ db_schema = DatabaseSchema()
35
+ for sql_file in sorted(scripts_dir.rglob("*.sql")):
36
+ _parse_file(sql_file, db_schema)
37
+ return db_schema
38
+
39
+
40
+ def _parse_file(path: Path, db_schema: DatabaseSchema) -> None:
41
+ content = path.read_text(encoding="utf-8")
42
+ try:
43
+ exprs = sqlglot.parse(content, dialect="postgres", error_level=sqlglot.ErrorLevel.WARN)
44
+ except Exception as e:
45
+ logger.warning("sqlglot failed on %s: %s", path.name, e)
46
+ exprs = []
47
+
48
+ for expr in exprs:
49
+ if expr is None:
50
+ continue
51
+ try:
52
+ _handle_expr(expr, content, db_schema)
53
+ except Exception as e:
54
+ logger.debug("Skipping expression in %s: %s", path.name, e)
55
+
56
+ _extract_do_block_objects(content, db_schema)
57
+
58
+
59
+ def _handle_expr(expr: exp.Expression, raw: str, db_schema: DatabaseSchema) -> None:
60
+ if not isinstance(expr, exp.Create):
61
+ return
62
+ kind = (expr.args.get("kind") or "").upper()
63
+ if kind == "TABLE":
64
+ _handle_table(expr, db_schema)
65
+ elif kind == "VIEW":
66
+ _handle_view(expr, db_schema)
67
+ elif kind in ("FUNCTION", "PROCEDURE"):
68
+ _handle_function(expr, raw, db_schema, kind.lower())
69
+ elif kind == "TYPE":
70
+ _handle_type(expr, db_schema)
71
+ elif kind == "SCHEMA":
72
+ _handle_schema_create(expr, db_schema)
73
+ elif kind == "INDEX":
74
+ _handle_index(expr, db_schema)
75
+
76
+
77
+ def _resolve_name(expr: exp.Create) -> tuple[str, str] | None:
78
+ """Return (schema, name) from a CREATE expression."""
79
+ this = expr.this
80
+ if isinstance(this, exp.Schema):
81
+ table_node = this.this
82
+ else:
83
+ table_node = this
84
+
85
+ if isinstance(table_node, exp.Table):
86
+ db_node = table_node.args.get("db")
87
+ schema = db_node.name if db_node else "public"
88
+ return schema, table_node.name
89
+ return None
90
+
91
+
92
+ def _handle_table(expr: exp.Create, db_schema: DatabaseSchema) -> None:
93
+ result = _resolve_name(expr)
94
+ if not result:
95
+ return
96
+ tschema, tname = result
97
+ properties = expr.args.get("properties")
98
+ is_partition = any(
99
+ isinstance(p, exp.PartitionedOfProperty) for p in (properties.expressions if properties else [])
100
+ )
101
+ table = TableDef(schema=tschema, name=tname, is_partition=is_partition)
102
+
103
+ this = expr.this
104
+ items = this.expressions if isinstance(this, exp.Schema) else []
105
+
106
+ pk_columns: set[str] = set()
107
+ for item in items:
108
+ if isinstance(item, exp.ColumnDef):
109
+ col = _parse_column_def(item)
110
+ if col:
111
+ table.columns.append(col)
112
+ else:
113
+ constr = _parse_table_constraint(item)
114
+ if constr:
115
+ table.constraints.append(constr)
116
+ pk_columns |= _extract_primary_key_columns(item)
117
+
118
+ for col in table.columns:
119
+ if col.name in pk_columns:
120
+ col.is_nullable = False
121
+
122
+ db_schema.tables[table.qualified_name] = table
123
+
124
+
125
+ def _extract_primary_key_columns(item: exp.Expression) -> set[str]:
126
+ inner = item
127
+ if isinstance(item, exp.Constraint):
128
+ inner = item.args.get("kind") or (item.expressions[0] if item.expressions else None)
129
+ if isinstance(inner, exp.PrimaryKey):
130
+ return {e.name for e in inner.expressions if hasattr(e, "name")}
131
+ return set()
132
+
133
+
134
+ def _parse_column_def(col: exp.ColumnDef) -> ColumnDef | None:
135
+ name = col.name
136
+ if not name or col.kind is None:
137
+ return None
138
+ data_type = col.kind.sql(dialect="postgres").lower()
139
+ is_serial = col.kind.this in (
140
+ exp.DataType.Type.SERIAL, exp.DataType.Type.SMALLSERIAL, exp.DataType.Type.BIGSERIAL,
141
+ )
142
+ is_nullable = not is_serial
143
+ default = None
144
+ is_generated = False
145
+
146
+ for c in col.constraints:
147
+ ck = c.kind
148
+ if isinstance(ck, (exp.NotNullColumnConstraint, exp.PrimaryKeyColumnConstraint)):
149
+ is_nullable = False
150
+ elif isinstance(ck, exp.DefaultColumnConstraint):
151
+ default = ck.this.sql(dialect="postgres") if ck.this else None
152
+ elif isinstance(ck, exp.GeneratedAsIdentityColumnConstraint):
153
+ is_generated = True
154
+ is_nullable = False
155
+ elif isinstance(ck, exp.ComputedColumnConstraint):
156
+ is_generated = True
157
+
158
+ return ColumnDef(name=name, data_type=data_type, is_nullable=is_nullable, default=default, is_generated=is_generated)
159
+
160
+
161
+ def _parse_table_constraint(item: exp.Expression) -> ConstraintDef | None:
162
+ name = None
163
+ kind = "UNKNOWN"
164
+
165
+ if isinstance(item, exp.Constraint):
166
+ name = item.name or None
167
+ inner = item.args.get("kind") or (item.expressions[0] if item.expressions else None)
168
+ else:
169
+ inner = item
170
+
171
+ if isinstance(inner, exp.PrimaryKey):
172
+ kind = "PRIMARY KEY"
173
+ elif isinstance(inner, exp.UniqueColumnConstraint):
174
+ kind = "UNIQUE"
175
+ elif isinstance(inner, exp.ForeignKey):
176
+ kind = "FOREIGN KEY"
177
+ elif isinstance(inner, exp.Check):
178
+ kind = "CHECK"
179
+ else:
180
+ return None
181
+
182
+ definition = item.sql(dialect="postgres").lower()
183
+ return ConstraintDef(name=name, kind=kind, definition=definition)
184
+
185
+
186
+ def _handle_view(expr: exp.Create, db_schema: DatabaseSchema) -> None:
187
+ result = _resolve_name(expr)
188
+ if not result:
189
+ return
190
+ vschema, vname = result
191
+ query = expr.expression
192
+ definition = query.sql(dialect="postgres").lower() if query else ""
193
+ view = ViewDef(schema=vschema, name=vname, definition=definition)
194
+ db_schema.views[view.qualified_name] = view
195
+
196
+
197
+ def _handle_function(expr: exp.Create, raw: str, db_schema: DatabaseSchema, kind: str) -> None:
198
+ # Get name/schema from sqlglot
199
+ func_node = expr.this
200
+ fname = func_node.name if hasattr(func_node, "name") else ""
201
+ if fname:
202
+ db_node = func_node.args.get("db") if hasattr(func_node, "args") else None
203
+ fschema = db_node.name if db_node else "public"
204
+ else:
205
+ # sqlglot (30.11.0) parses "CREATE FUNCTION myapp.greet(...)" as a
206
+ # UserDefinedFunction wrapping a Table (this=Identifier(greet),
207
+ # db=Identifier(myapp)). UserDefinedFunction.name doesn't unwrap
208
+ # that nested Table, so pull the name/schema from it directly.
209
+ inner = func_node.this if hasattr(func_node, "this") else None
210
+ if isinstance(inner, exp.Table) and inner.name:
211
+ fname = inner.name
212
+ db_node = inner.args.get("db")
213
+ fschema = db_node.name if db_node else "public"
214
+ else:
215
+ result = _resolve_name(expr)
216
+ if not result:
217
+ return
218
+ fschema, fname = result
219
+
220
+ # Extract args, return type, language, body with regex on raw SQL
221
+ args, return_type, language, body = _parse_function_details(raw)
222
+
223
+ func = FunctionDef(
224
+ schema=fschema,
225
+ name=fname,
226
+ args=args,
227
+ return_type=return_type,
228
+ language=language,
229
+ body=body,
230
+ kind=kind,
231
+ )
232
+ db_schema.functions[func.qualified_name] = func
233
+
234
+
235
+ _FUNC_SIG = re.compile(
236
+ r'CREATE\s+(?:OR\s+REPLACE\s+)?(?:FUNCTION|PROCEDURE)\s+'
237
+ r'(?:\w+\.)?(\w+)\s*\(([^)]*(?:\([^)]*\)[^)]*)*)\)\s*'
238
+ r'(?:RETURNS\s+((?:TABLE\s*\([^)]+\)|SETOF\s+\S+|\S+)))?\s*'
239
+ r'LANGUAGE\s+(\w+)',
240
+ re.IGNORECASE | re.DOTALL,
241
+ )
242
+
243
+
244
+ def _parse_function_details(sql: str) -> tuple[str, str, str, str]:
245
+ args, return_type, language, body = "", "", "", ""
246
+
247
+ m = _FUNC_SIG.search(sql)
248
+ if m:
249
+ args = re.sub(r'\s+', ' ', m.group(2) or "").strip().lower()
250
+ return_type = (m.group(3) or "").strip().lower()
251
+ language = (m.group(4) or "").strip().lower()
252
+
253
+ dm = _DOLLAR_BODY.search(sql)
254
+ if dm:
255
+ raw_body = dm.group(2)
256
+ lines = [l.strip() for l in raw_body.splitlines()]
257
+ body = "\n".join(l.lower() for l in lines if l)
258
+
259
+ return args, return_type, language, body
260
+
261
+
262
+ def _handle_type(expr: exp.Create, db_schema: DatabaseSchema) -> None:
263
+ result = _resolve_name(expr)
264
+ if not result:
265
+ return
266
+ tschema, tname = result
267
+
268
+ expression = expr.expression
269
+ if expression is None:
270
+ return
271
+
272
+ if isinstance(expression, exp.DataType) and expression.this == exp.DataType.Type.ENUM:
273
+ values = [lit.name for lit in expression.expressions if isinstance(lit, exp.Literal)]
274
+ enum = EnumDef(schema=tschema, name=tname, values=values)
275
+ db_schema.enums[enum.qualified_name] = enum
276
+ elif isinstance(expression, exp.Schema):
277
+ # Composite type: fields are ColumnDef-like
278
+ fields = []
279
+ for col in expression.expressions:
280
+ if isinstance(col, exp.ColumnDef) and col.kind:
281
+ fields.append((col.name, col.kind.sql(dialect="postgres").lower()))
282
+ comp = CompositeTypeDef(schema=tschema, name=tname, fields=fields)
283
+ db_schema.composites[comp.qualified_name] = comp
284
+
285
+
286
+ def _handle_schema_create(expr: exp.Create, db_schema: DatabaseSchema) -> None:
287
+ this = expr.this
288
+ name = this.name if hasattr(this, "name") else ""
289
+ if not name:
290
+ # sqlglot (30.11.0) parses "CREATE SCHEMA myapp" as a Table node
291
+ # whose `db` arg holds the schema name and `.name` (the table
292
+ # identifier) is empty.
293
+ db_node = this.args.get("db") if hasattr(this, "args") else None
294
+ name = db_node.name if db_node else ""
295
+ if name:
296
+ db_schema.schemas.add(name)
297
+
298
+
299
+ def _handle_index(expr: exp.Create, db_schema: DatabaseSchema) -> None:
300
+ this = expr.this
301
+ index_name = this.name if hasattr(this, "name") else ""
302
+ table_node = expr.find(exp.Table)
303
+ if not table_node:
304
+ return
305
+ db_node = table_node.args.get("db")
306
+ tschema = db_node.name if db_node else "public"
307
+ tname = table_node.name
308
+ definition = expr.sql(dialect="postgres").lower()
309
+ idx = IndexDef(schema=tschema, table=tname, name=index_name, definition=definition)
310
+ db_schema.indexes[idx.qualified_name] = idx
311
+
312
+
313
+ def _extract_do_block_objects(sql: str, db_schema: DatabaseSchema) -> None:
314
+ """Extract CREATE TYPE statements from DO $$ ... $$ blocks using regex."""
315
+ for dm in _DOLLAR_BODY.finditer(sql):
316
+ block = dm.group(2)
317
+ for m in _DO_ENUM.finditer(block):
318
+ qualified = m.group(1)
319
+ parts = qualified.split(".")
320
+ tschema, tname = (parts[0], parts[1]) if len(parts) == 2 else ("public", parts[0])
321
+ values_raw = m.group(2)
322
+ values = [v.strip().strip("'\"") for v in values_raw.split(",") if v.strip()]
323
+ enum = EnumDef(schema=tschema, name=tname, values=values)
324
+ db_schema.enums.setdefault(enum.qualified_name, enum)
325
+ for m in _DO_COMPOSITE.finditer(block):
326
+ qualified = m.group(1)
327
+ parts = qualified.split(".")
328
+ tschema, tname = (parts[0], parts[1]) if len(parts) == 2 else ("public", parts[0])
329
+ fields_raw = m.group(2)
330
+ fields = []
331
+ for field_def in fields_raw.split(","):
332
+ parts2 = field_def.strip().split()
333
+ if len(parts2) >= 2:
334
+ fields.append((parts2[0], " ".join(parts2[1:]).lower()))
335
+ comp = CompositeTypeDef(schema=tschema, name=tname, fields=fields)
336
+ db_schema.composites.setdefault(comp.qualified_name, comp)
@@ -0,0 +1,283 @@
1
+ ---
2
+ name: pgdevkit
3
+ plugin: coding
4
+ description: >
5
+ Use pgdevkit for any PostgreSQL work in a Python project: a local
6
+ Docker/Podman test database (`pgdb testdb`), importable ORM-free CRUD
7
+ helpers (`pgdevkit.db`), and the `database/`-folder schema-as-code
8
+ convention. Supersedes the old postgres-test-setup, postgres-best-practices,
9
+ and database-in-source skills — pgdevkit is a real dependency now, not
10
+ copy-pasted reference files. Use whenever the user wants to set up a local
11
+ test Postgres, write or review psycopg code, add a table/view/function to
12
+ a `database/` folder, or asks things like "add a database query", "create
13
+ a repository", "set up a test database", "reset the database", "add a
14
+ table", "migration script", "backfill missing objects", or mentions
15
+ psycopg/psycopg2/asyncpg/SQLAlchemy where the user seems open to a
16
+ different approach.
17
+ ---
18
+
19
+ # pgdevkit — Postgres for Python projects
20
+
21
+ One dependency covers three things that used to be three separate skills
22
+ with copy-pasted reference files:
23
+
24
+ | Old skill | Replaced by |
25
+ |---|---|
26
+ | `postgres-test-setup` | `pgdb testdb` (container + schema apply) |
27
+ | `postgres-best-practices` | `pgdevkit.db` (importable CRUD helpers) |
28
+ | `database-in-source` | The `database/` folder convention — see [docs/database-layout.md](../../docs/database-layout.md) |
29
+
30
+ Core rules for every piece of database code in a project using pgdevkit:
31
+
32
+ - **No ORM** — use [psycopg](https://www.psycopg.org/psycopg3/) directly, via `pgdevkit.db`'s helpers or hand-written queries.
33
+ - **Inline SQL** — trivial queries of **4 lines or fewer** may be written inline in Python. Anything with JOINs, subqueries, CTEs, aggregations, or multiple conditions lives in its own `.sql` file.
34
+ - **Named parameters** — always `%(name)s` style, never positional `%s`.
35
+ - **The `database/` folder is the source of truth for the schema** — see [docs/database-layout.md](../../docs/database-layout.md) for the layer/object-type/file-naming conventions.
36
+ - **Result mapping** — every query result maps to a Pydantic model; table-mapped models extend `pgdevkit.db.PostgresTableModel`.
37
+
38
+ ---
39
+
40
+ ## Install
41
+
42
+ ```bash
43
+ uv add pgdevkit[cli,db]
44
+ ```
45
+
46
+ `cli` pulls in `typer`/`rich` for the `pgdb` command; `db` pulls in `pydantic`/`psycopg-pool` for the importable CRUD helpers. Skip either extra if the project doesn't need it (e.g. a project using only `pgdb testdb` doesn't need `db`).
47
+
48
+ ---
49
+
50
+ ## Local test database — `pgdb testdb`
51
+
52
+ Add to `pyproject.toml`:
53
+
54
+ ```toml
55
+ [tool.pgdevkit]
56
+ database_dir = "database" # optional, defaults to "database"
57
+ env_prefix = "MDM_" # optional, defaults to "{name.upper()}_"
58
+ extensions = ["vector"] # optional, CREATE EXTENSION IF NOT EXISTS
59
+ ```
60
+
61
+ `name` is optional too — it falls back to the repo directory name.
62
+
63
+ In `tests/conftest.py`:
64
+
65
+ ```python
66
+ import pytest
67
+ from pgdevkit.testdb import ensure_testdb
68
+
69
+ @pytest.fixture(scope="session", autouse=True)
70
+ def _testdb_env():
71
+ env = ensure_testdb()
72
+ for key, value in env.items():
73
+ os.environ[key] = value
74
+ ```
75
+
76
+ `ensure_testdb()` starts the shared `pgdevkit-postgres` container if needed (via `podman`), creates a database scoped to this project+branch (so different worktrees/branches never collide), and applies every `.sql` file under `database_dir` in dependency order, seeding any `.test_data.json` sidecar files.
77
+
78
+ | What do you need? | Command |
79
+ |---|---|
80
+ | First-time setup / apply new files | `pgdb testdb up` |
81
+ | Breaking change (rename/drop column) | `pgdb testdb reset` |
82
+ | Inspect test DB data | `pgdb testdb run-sql --sql "SELECT ..." --results` |
83
+ | Re-apply one file (e.g. a function/view) | `pgdb testdb run-sql database/path/to/file.sql` |
84
+ | Drop this workspace's database | `pgdb testdb clean` |
85
+ | Drop every database for this project (all branches) | `pgdb testdb clean --all` |
86
+
87
+ ### CI
88
+
89
+ Podman needs to be available on the runner (`apt-get install -y podman` on `ubuntu-latest` if not preinstalled) — `ensure_testdb()`/`ensure_container()` shell out to it directly. There's no `PGDEVKIT_SKIP_CONTAINER` + service-container escape hatch wired through every fixture yet — if a project's CI can't run podman, it needs its own workaround for now.
90
+
91
+ ---
92
+
93
+ ## Application-side DB code — `pgdevkit.db`
94
+
95
+ ```
96
+ app/
97
+ ├── db/
98
+ │ ├── queries/
99
+ │ │ ├── users/
100
+ │ │ │ ├── get_user_by_id.sql
101
+ │ │ │ └── list_active_users.sql
102
+ │ └── repositories/
103
+ │ └── user_repository.py
104
+ ├── models/
105
+ │ └── user_models.py # Pydantic models for the user domain
106
+ ```
107
+
108
+ SQL files live under `db/queries/<topic>/`. Every custom query gets its own file — no multi-statement files that lump unrelated queries together.
109
+
110
+ ### Connection pool
111
+
112
+ ```python
113
+ # db/connection.py
114
+ from pgdevkit.db import PgPool
115
+
116
+ pool = PgPool(env_prefix="APP_POSTGRES_") # matches ensure_testdb()'s {env_prefix}POSTGRES_* vars
117
+
118
+ async def startup():
119
+ await pool.open()
120
+ ```
121
+
122
+ ### Models
123
+
124
+ ```python
125
+ # models/user_models.py
126
+ from __future__ import annotations
127
+ from datetime import datetime
128
+ from pydantic import BaseModel, ConfigDict
129
+ from pgdevkit.db import PostgresTableModel
130
+
131
+ class UserRow(PostgresTableModel):
132
+ model_config = ConfigDict(from_attributes=True)
133
+ id: int
134
+ email: str
135
+ display_name: str
136
+ created_at: datetime
137
+
138
+ @staticmethod
139
+ def get_table_name() -> tuple[str, str]:
140
+ return ("public", "users")
141
+
142
+ @staticmethod
143
+ def get_primary_key() -> list[str]:
144
+ return ["id"]
145
+
146
+ class UserSummary(BaseModel):
147
+ model_config = ConfigDict(from_attributes=True)
148
+ id: int
149
+ display_name: str
150
+ ```
151
+
152
+ Models that represent partial results (joins, aggregations, partial selects) extend `BaseModel` directly instead of `PostgresTableModel`.
153
+
154
+ ### CRUD helpers
155
+
156
+ ```python
157
+ from pgdevkit.db import pg_retrieve, pg_insert, pg_upsert, pg_delete
158
+ ```
159
+
160
+ | Helper | Purpose |
161
+ |--------|---------|
162
+ | `pg_retrieve` | Fetch single row by PK |
163
+ | `pg_retrieve_many` | Fetch rows matching filter dict |
164
+ | `pg_insert` | Insert one row, `RETURNING *` |
165
+ | `pg_update` / `pg_update_dict` | Update by PK |
166
+ | `pg_upsert` / `pg_upsert_dict` | `INSERT ... ON CONFLICT ... DO UPDATE` |
167
+ | `pg_upsert_many` / `pg_upsert_many_dict` | Batch upsert via `executemany` |
168
+ | `pg_insert_many` | Batch insert via `executemany` |
169
+ | `pg_delete` / `pg_delete_dict` | Delete by PK, returns deleted row |
170
+
171
+ Use these for simple CRUD. For custom `WHERE` clauses, joins, aggregations, or ordering, write a dedicated `.sql` file and a repository method.
172
+
173
+ ### Loading `.sql` files
174
+
175
+ ```python
176
+ # db/loader.py
177
+ from pathlib import Path
178
+ from pgdevkit.db import SqlLoader
179
+
180
+ sql = SqlLoader(Path(__file__).parent / "queries")
181
+ ```
182
+
183
+ ```python
184
+ # db/repositories/user_repository.py
185
+ from psycopg.rows import dict_row
186
+ from pgdevkit.db import pg_retrieve, pg_delete
187
+ from db.connection import pool
188
+ from db.loader import sql
189
+ from models.user_models import UserRow, UserSummary
190
+
191
+ class UserRepository:
192
+ async def get_by_id(self, user_id: int) -> UserRow | None:
193
+ async with pool.connection() as conn:
194
+ return await pg_retrieve(conn, UserRow, {"id": user_id})
195
+
196
+ async def list_active(self, limit: int = 100) -> list[UserSummary]:
197
+ async with pool.connection() as conn:
198
+ async with conn.cursor(row_factory=dict_row) as cur:
199
+ await cur.execute(sql.load_sql("users", "list_active_users"), {"limit": limit})
200
+ rows = await cur.fetchall()
201
+ return [UserSummary.model_validate(r) for r in rows]
202
+
203
+ async def delete(self, user: UserRow) -> UserRow | None:
204
+ async with pool.connection() as conn:
205
+ return await pg_delete(conn, user, UserRow)
206
+ ```
207
+
208
+ Named parameters, `%(name)s` style, dict argument — never positional `%s`, never f-strings or `str.format()` for SQL text.
209
+
210
+ ### Dynamic SQL
211
+
212
+ Avoid it whenever possible — a static `.sql` file is always clearer. See [`references/dynamic-sql.md`](references/dynamic-sql.md) for t-string templates (3.14+) and `psycopg.sql` (< 3.14) when column/table names genuinely vary at runtime.
213
+
214
+ ### Avoid `LATERAL JOIN` — use a CTE instead
215
+
216
+ ```sql
217
+ with latest_order as (
218
+ select
219
+ o.user_id,
220
+ o.total,
221
+ row_number() over (partition by o.user_id order by o.created_at desc) as rn
222
+ from orders as o
223
+ )
224
+ select u.id, u.email, lo.total
225
+ from users as u
226
+ join latest_order as lo on lo.user_id = u.id and lo.rn = 1
227
+ ```
228
+
229
+ ### Temporal tables
230
+
231
+ See [`references/temporal-tables.md`](references/temporal-tables.md) for row-level history via `nearform/temporal_tables`.
232
+
233
+ ### Custom Postgres types in test data
234
+
235
+ `pgdevkit.testdb.schema`'s test-data seeding handles plain columns and JSONB, but not composite types or enums directly. See [`references/complex_helper.py`](references/complex_helper.py) for a psycopg adapter (`ComplexHelper`) if a project needs that.
236
+
237
+ ### SQL formatting
238
+
239
+ ```bash
240
+ uv add --dev shandy-sqlfmt[jinjafmt]
241
+ sqlfmt db/queries/ # format
242
+ sqlfmt --check db/queries/ # CI check
243
+ ```
244
+
245
+ ---
246
+
247
+ ## The `database/` folder & backfilling untracked objects
248
+
249
+ See [docs/database-layout.md](../../docs/database-layout.md) for the full convention: layer directories, object-type subfolders and their apply order, file-naming rules (`.test_data.json`, `.init.sql`, `.prod`), and how migrations are organised.
250
+
251
+ If a table, view, or function was created directly on the database and never got a `.sql` file:
252
+
253
+ ```bash
254
+ pgdb fetch-missing database/ --url postgresql://... # dry run, lists what's missing
255
+ pgdb fetch-missing database/ --url postgresql://... --write
256
+ ```
257
+
258
+ It diffs the live schema against `database/`, reverse-engineers DDL for anything untracked, and writes it into the matching layer folder's `tables/`, `views/`, `scalar_functions/`, or `table_functions/` subfolder (matched by schema name against existing top-level directories, ignoring their leading sort number).
259
+
260
+ ---
261
+
262
+ ## Comparing scripts to a live database
263
+
264
+ ```bash
265
+ pgdb compare --url postgresql://... database/
266
+ ```
267
+
268
+ Reports drift between the `database/` `.sql` files and the actual schema — tables, views, functions, enums, composite types, and indexes. Pass `--report-extra-db` to also flag objects that exist in the database but aren't tracked (this is what `pgdb fetch-missing` uses internally).
269
+
270
+ ---
271
+
272
+ ## Quick checklist
273
+
274
+ - [ ] `[tool.pgdevkit]` configured in `pyproject.toml`; `tests/conftest.py` calls `ensure_testdb()`
275
+ - [ ] New table/view/function/type gets its own `.sql` file under the right layer + object-type folder (see [docs/database-layout.md](../../docs/database-layout.md))
276
+ - [ ] Simple CRUD uses `pgdevkit.db`'s `pg_*` helpers; custom queries use `.sql` files loaded via `SqlLoader`
277
+ - [ ] Inline SQL only for trivial queries ≤ 4 lines; anything with JOINs/CTEs/aggregations/subqueries uses a `.sql` file
278
+ - [ ] All parameters use `%(name)s` style with a dict argument
279
+ - [ ] Results mapped to a Pydantic model; table-mapped models extend `PostgresTableModel`
280
+ - [ ] No `LATERAL JOIN` — use a CTE that groups/aggregates first, then joins it
281
+ - [ ] `.prod` files are production-only and skipped by `pgdb testdb`
282
+ - [ ] Every table (and non-obvious column) has a `COMMENT ON`, placed in the object's own `.sql` file
283
+ - [ ] Untracked DB objects backfilled via `pgdb fetch-missing`, not left undocumented