pgdevkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pgdevkit/__init__.py +0 -0
- pgdevkit/cli.py +217 -0
- pgdevkit/connection.py +56 -0
- pgdevkit/db/__init__.py +37 -0
- pgdevkit/db/connection.py +74 -0
- pgdevkit/db/crud.py +184 -0
- pgdevkit/db/loader.py +19 -0
- pgdevkit/db/model.py +23 -0
- pgdevkit/diff.py +253 -0
- pgdevkit/docs/database-layout.md +145 -0
- pgdevkit/fetch_missing.py +229 -0
- pgdevkit/introspect.py +190 -0
- pgdevkit/lakebase.py +90 -0
- pgdevkit/models.py +102 -0
- pgdevkit/parser.py +336 -0
- pgdevkit/skills/pgdevkit/SKILL.md +283 -0
- pgdevkit/skills/pgdevkit/references/complex_helper.py +234 -0
- pgdevkit/skills/pgdevkit/references/dynamic-sql.md +92 -0
- pgdevkit/skills/pgdevkit/references/temporal-tables.md +33 -0
- pgdevkit/testdb/__init__.py +3 -0
- pgdevkit/testdb/api.py +137 -0
- pgdevkit/testdb/config.py +51 -0
- pgdevkit/testdb/constants.py +11 -0
- pgdevkit/testdb/container.py +95 -0
- pgdevkit/testdb/naming.py +41 -0
- pgdevkit/testdb/query.py +65 -0
- pgdevkit/testdb/schema.py +188 -0
- pgdevkit-0.1.0.dist-info/METADATA +140 -0
- pgdevkit-0.1.0.dist-info/RECORD +31 -0
- pgdevkit-0.1.0.dist-info/WHEEL +4 -0
- pgdevkit-0.1.0.dist-info/entry_points.txt +2 -0
pgdevkit/diff.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import re
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from enum import Enum
|
|
5
|
+
|
|
6
|
+
import sqlglot
|
|
7
|
+
import sqlglot.expressions as exp
|
|
8
|
+
|
|
9
|
+
from .models import DatabaseSchema, FunctionDef, IndexDef, TableDef
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DiffKind(str, Enum):
|
|
13
|
+
MISSING_IN_DB = "missing_in_db"
|
|
14
|
+
MISSING_IN_SCRIPTS = "missing_in_scripts"
|
|
15
|
+
MISMATCH = "mismatch"
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class DiffEntry:
|
|
20
|
+
kind: DiffKind
|
|
21
|
+
object_type: str
|
|
22
|
+
object_name: str
|
|
23
|
+
detail: str = ""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def compute_diff(scripts: DatabaseSchema, db: DatabaseSchema, report_extra_db: bool = False) -> list[DiffEntry]:
|
|
27
|
+
diffs: list[DiffEntry] = []
|
|
28
|
+
|
|
29
|
+
_diff_set("schema", scripts.schemas, db.schemas, diffs, report_extra_db)
|
|
30
|
+
|
|
31
|
+
tables_missing_in_db: set[str] = set()
|
|
32
|
+
tables_missing_in_scripts: set[str] = set()
|
|
33
|
+
|
|
34
|
+
for name, obj in scripts.tables.items():
|
|
35
|
+
if name not in db.tables:
|
|
36
|
+
tables_missing_in_db.add(name)
|
|
37
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_DB, "table", name))
|
|
38
|
+
else:
|
|
39
|
+
_diff_table(name, obj, db.tables[name], diffs)
|
|
40
|
+
if report_extra_db:
|
|
41
|
+
for name in db.tables:
|
|
42
|
+
if name not in scripts.tables:
|
|
43
|
+
tables_missing_in_scripts.add(name)
|
|
44
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_SCRIPTS, "table", name))
|
|
45
|
+
|
|
46
|
+
for name, obj in scripts.views.items():
|
|
47
|
+
if name not in db.views:
|
|
48
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_DB, "view", name))
|
|
49
|
+
else:
|
|
50
|
+
s = _norm_sql(obj.definition)
|
|
51
|
+
d = _norm_sql(db.views[name].definition)
|
|
52
|
+
if s != d:
|
|
53
|
+
diffs.append(DiffEntry(DiffKind.MISMATCH, "view", name, "definition differs"))
|
|
54
|
+
if report_extra_db:
|
|
55
|
+
for name in db.views:
|
|
56
|
+
if name not in scripts.views:
|
|
57
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_SCRIPTS, "view", name))
|
|
58
|
+
|
|
59
|
+
for name, obj in scripts.functions.items():
|
|
60
|
+
if name not in db.functions:
|
|
61
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_DB, "function", name))
|
|
62
|
+
else:
|
|
63
|
+
_diff_function(name, obj, db.functions[name], diffs)
|
|
64
|
+
if report_extra_db:
|
|
65
|
+
for name in db.functions:
|
|
66
|
+
if name not in scripts.functions:
|
|
67
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_SCRIPTS, "function", name))
|
|
68
|
+
|
|
69
|
+
for name, obj in scripts.enums.items():
|
|
70
|
+
if name not in db.enums:
|
|
71
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_DB, "enum", name))
|
|
72
|
+
elif obj.values != db.enums[name].values:
|
|
73
|
+
diffs.append(DiffEntry(DiffKind.MISMATCH, "enum", name, f"values: {obj.values} vs {db.enums[name].values}"))
|
|
74
|
+
if report_extra_db:
|
|
75
|
+
for name in db.enums:
|
|
76
|
+
if name not in scripts.enums:
|
|
77
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_SCRIPTS, "enum", name))
|
|
78
|
+
|
|
79
|
+
for name, obj in scripts.composites.items():
|
|
80
|
+
if name not in db.composites:
|
|
81
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_DB, "composite_type", name))
|
|
82
|
+
elif obj.fields != db.composites[name].fields:
|
|
83
|
+
diffs.append(DiffEntry(DiffKind.MISMATCH, "composite_type", name, "fields differ"))
|
|
84
|
+
if report_extra_db:
|
|
85
|
+
for name in db.composites:
|
|
86
|
+
if name not in scripts.composites:
|
|
87
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_SCRIPTS, "composite_type", name))
|
|
88
|
+
|
|
89
|
+
for name, obj in scripts.indexes.items():
|
|
90
|
+
if name not in db.indexes:
|
|
91
|
+
if f"{obj.schema}.{obj.table}" not in tables_missing_in_db:
|
|
92
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_DB, "index", name))
|
|
93
|
+
else:
|
|
94
|
+
_diff_index(name, obj, db.indexes[name], diffs)
|
|
95
|
+
if report_extra_db:
|
|
96
|
+
for name, obj in db.indexes.items():
|
|
97
|
+
if name not in scripts.indexes:
|
|
98
|
+
if f"{obj.schema}.{obj.table}" not in tables_missing_in_scripts:
|
|
99
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_SCRIPTS, "index", name))
|
|
100
|
+
|
|
101
|
+
return diffs
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _diff_set(obj_type: str, scripts_set: set, db_set: set, diffs: list[DiffEntry], report_extra: bool) -> None:
|
|
105
|
+
for item in scripts_set:
|
|
106
|
+
if item not in db_set:
|
|
107
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_DB, obj_type, item))
|
|
108
|
+
if report_extra:
|
|
109
|
+
for item in db_set:
|
|
110
|
+
if item not in scripts_set:
|
|
111
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_SCRIPTS, obj_type, item))
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _diff_table(name: str, s: TableDef, d: TableDef, diffs: list[DiffEntry]) -> None:
|
|
115
|
+
if s.is_partition or d.is_partition:
|
|
116
|
+
return
|
|
117
|
+
|
|
118
|
+
scols = {c.name: c for c in s.columns}
|
|
119
|
+
dcols = {c.name: c for c in d.columns}
|
|
120
|
+
for cname, sc in scols.items():
|
|
121
|
+
if cname not in dcols:
|
|
122
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_DB, "column", f"{name}.{cname}"))
|
|
123
|
+
else:
|
|
124
|
+
dc = dcols[cname]
|
|
125
|
+
issues = []
|
|
126
|
+
if _norm_type(sc.data_type) != _norm_type(dc.data_type):
|
|
127
|
+
issues.append(f"type: {sc.data_type!r} vs {dc.data_type!r}")
|
|
128
|
+
if sc.is_nullable != dc.is_nullable:
|
|
129
|
+
issues.append(f"nullable: {sc.is_nullable} vs {dc.is_nullable}")
|
|
130
|
+
if issues:
|
|
131
|
+
diffs.append(DiffEntry(DiffKind.MISMATCH, "column", f"{name}.{cname}", "; ".join(issues)))
|
|
132
|
+
for cname in dcols:
|
|
133
|
+
if cname not in scols:
|
|
134
|
+
diffs.append(DiffEntry(DiffKind.MISSING_IN_SCRIPTS, "column", f"{name}.{cname}"))
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _diff_function(name: str, s: FunctionDef, d: FunctionDef, diffs: list[DiffEntry]) -> None:
|
|
138
|
+
issues = []
|
|
139
|
+
if _norm_type(s.return_type) != _norm_type(d.return_type):
|
|
140
|
+
issues.append(f"return_type: {s.return_type!r} vs {d.return_type!r}")
|
|
141
|
+
if _norm_body(s.body) != _norm_body(d.body):
|
|
142
|
+
issues.append("body differs")
|
|
143
|
+
if issues:
|
|
144
|
+
diffs.append(DiffEntry(DiffKind.MISMATCH, "function", name, "; ".join(issues)))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _diff_index(name: str, s: IndexDef, d: IndexDef, diffs: list[DiffEntry]) -> None:
|
|
148
|
+
s_info = _parse_index_def(s.definition)
|
|
149
|
+
d_info = _parse_index_def(d.definition)
|
|
150
|
+
|
|
151
|
+
if s_info is None or d_info is None:
|
|
152
|
+
if _norm_sql(s.definition) != _norm_sql(d.definition):
|
|
153
|
+
diffs.append(DiffEntry(DiffKind.MISMATCH, "index", name, "definition differs"))
|
|
154
|
+
return
|
|
155
|
+
|
|
156
|
+
issues = []
|
|
157
|
+
if s_info["unique"] != d_info["unique"]:
|
|
158
|
+
issues.append(f"unique: {s_info['unique']} vs {d_info['unique']}")
|
|
159
|
+
if s_info["using"] != d_info["using"]:
|
|
160
|
+
issues.append(f"using: {s_info['using']} vs {d_info['using']}")
|
|
161
|
+
if s_info["columns"] != d_info["columns"]:
|
|
162
|
+
issues.append(f"columns: ({', '.join(s_info['columns'])}) vs ({', '.join(d_info['columns'])})")
|
|
163
|
+
if s_info["where_ast"] != d_info["where_ast"]:
|
|
164
|
+
issues.append(f"where: {s_info['where_display']!r} vs {d_info['where_display']!r}")
|
|
165
|
+
|
|
166
|
+
if issues:
|
|
167
|
+
diffs.append(DiffEntry(DiffKind.MISMATCH, "index", name, "; ".join(issues)))
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _parse_index_def(definition: str) -> dict | None:
|
|
171
|
+
try:
|
|
172
|
+
parsed = sqlglot.parse_one(definition, dialect="postgres")
|
|
173
|
+
except Exception:
|
|
174
|
+
return None
|
|
175
|
+
if not isinstance(parsed, exp.Create):
|
|
176
|
+
return None
|
|
177
|
+
index_node = parsed.this
|
|
178
|
+
if not isinstance(index_node, exp.Index):
|
|
179
|
+
return None
|
|
180
|
+
|
|
181
|
+
params = index_node.args.get("params")
|
|
182
|
+
using_node = params.args.get("using") if params else None
|
|
183
|
+
using = using_node.name.lower() if using_node else "btree"
|
|
184
|
+
|
|
185
|
+
columns = []
|
|
186
|
+
for col in (params.args.get("columns") if params else None) or []:
|
|
187
|
+
columns.append(_norm_sql(col.sql(dialect="postgres")))
|
|
188
|
+
|
|
189
|
+
where_node = params.args.get("where") if params else None
|
|
190
|
+
where_ast = _unwrap_paren(where_node.this) if where_node else None
|
|
191
|
+
where_display = _norm_sql(where_node.this.sql(dialect="postgres")) if where_node else None
|
|
192
|
+
|
|
193
|
+
return {
|
|
194
|
+
"unique": bool(parsed.args.get("unique")),
|
|
195
|
+
"using": using,
|
|
196
|
+
"columns": columns,
|
|
197
|
+
"where_ast": where_ast,
|
|
198
|
+
"where_display": where_display,
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def _unwrap_paren(node):
|
|
203
|
+
return node.transform(lambda n: n.this if isinstance(n, exp.Paren) else n)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _norm_sql(s: str) -> str:
|
|
207
|
+
# Postgres's pg_get_viewdef()/pg_get_indexdef() always append a trailing
|
|
208
|
+
# ";", while sqlglot's Expression.sql() rendering never does; strip it so
|
|
209
|
+
# semantically-identical definitions compare equal.
|
|
210
|
+
return " ".join(s.lower().split()).rstrip(";")
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _norm_body(s: str) -> str:
|
|
214
|
+
lines = [l.strip() for l in s.splitlines()]
|
|
215
|
+
return "\n".join(l.lower() for l in lines if l)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
_TYPE_SYNONYMS = {
|
|
219
|
+
"int": "integer", "int4": "integer",
|
|
220
|
+
"int2": "smallint",
|
|
221
|
+
"int8": "bigint",
|
|
222
|
+
"float4": "real",
|
|
223
|
+
"float8": "double precision",
|
|
224
|
+
"bool": "boolean",
|
|
225
|
+
"decimal": "numeric",
|
|
226
|
+
"varchar": "character varying",
|
|
227
|
+
"char": "character", "bpchar": "character",
|
|
228
|
+
"timestamptz": "timestamp with time zone",
|
|
229
|
+
"timestamp": "timestamp without time zone",
|
|
230
|
+
"timetz": "time with time zone",
|
|
231
|
+
"time": "time without time zone",
|
|
232
|
+
"varbit": "bit varying",
|
|
233
|
+
"serial": "integer", "serial4": "integer",
|
|
234
|
+
"smallserial": "smallint", "serial2": "smallint",
|
|
235
|
+
"bigserial": "bigint", "serial8": "bigint",
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _norm_type(t: str) -> str:
|
|
240
|
+
s = " ".join(t.lower().split())
|
|
241
|
+
|
|
242
|
+
array_suffix = ""
|
|
243
|
+
while s.endswith("[]"):
|
|
244
|
+
array_suffix += "[]"
|
|
245
|
+
s = s[:-2].strip()
|
|
246
|
+
|
|
247
|
+
match = re.search(r"\(([^)]*)\)", s)
|
|
248
|
+
params = f"({match.group(1)})" if match else ""
|
|
249
|
+
base = (s[: match.start()] + s[match.end() :]).strip() if match else s
|
|
250
|
+
base = " ".join(base.split())
|
|
251
|
+
|
|
252
|
+
base = _TYPE_SYNONYMS.get(base, base)
|
|
253
|
+
return f"{base}{params}{array_suffix}"
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# `database/` folder layout
|
|
2
|
+
|
|
3
|
+
The Postgres schema is versioned as plain `.sql` files under a `database/`
|
|
4
|
+
folder in the repo — that folder *is* the source of truth for the schema.
|
|
5
|
+
`pgdb testdb` (see [`pgdevkit/testdb/schema.py`](../pgdevkit/testdb/schema.py))
|
|
6
|
+
applies it to a local test database; a human applies the same files to
|
|
7
|
+
production.
|
|
8
|
+
|
|
9
|
+
This is the single source for the convention — don't duplicate this table or
|
|
10
|
+
these rules elsewhere; link here instead.
|
|
11
|
+
|
|
12
|
+
---
|
|
13
|
+
|
|
14
|
+
## Layer directories
|
|
15
|
+
|
|
16
|
+
Top-level directories group tables/objects by conceptual layer — one
|
|
17
|
+
directory per Postgres schema, or per logical grouping within one schema.
|
|
18
|
+
Names and count are entirely per-project; a generic example:
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
database/
|
|
22
|
+
├── schema.sql # CREATE SCHEMA statements
|
|
23
|
+
├── 0_public/ # shared types/functions usable from anywhere
|
|
24
|
+
├── 1_reference_data/ # dimension / reference tables
|
|
25
|
+
├── 2_transactional/ # fact / transactional tables
|
|
26
|
+
├── 3_app/ # user-editable, app-owned tables
|
|
27
|
+
├── 4_reporting/ # aggregated / statistics tables
|
|
28
|
+
├── permissions.sql # grants
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
The leading number is a **display/sort aid only** — it groups related
|
|
32
|
+
folders together in a file listing so a human can scan them top-to-bottom in
|
|
33
|
+
a sensible order. It does not control apply order: `pgdb testdb` applies
|
|
34
|
+
files by object-type priority (below) and resolves cross-file dependencies
|
|
35
|
+
itself, regardless of which numbered folder a file sits in. Feel free to
|
|
36
|
+
renumber layers for readability without worrying about breaking anything.
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## Object-type subfolders, in apply order
|
|
41
|
+
|
|
42
|
+
Within a layer directory, group files by object type. This is what actually
|
|
43
|
+
controls apply order, across all layer directories — cross-file dependencies
|
|
44
|
+
between objects of the same type (e.g. one view selecting from another) are
|
|
45
|
+
resolved automatically; this table only fixes the order *between* types.
|
|
46
|
+
This table must match `_TYPE_ORDER` / `_SCHEMA_QUALIFIED_TYPES` in
|
|
47
|
+
[`pgdevkit/testdb/schema.py`](../pgdevkit/testdb/schema.py) exactly — update
|
|
48
|
+
both together.
|
|
49
|
+
|
|
50
|
+
| Priority | Directory | Object type |
|
|
51
|
+
|---|---|---|
|
|
52
|
+
| 1 | `schema` | `CREATE SCHEMA` |
|
|
53
|
+
| 2 | `types` | Custom types / enums |
|
|
54
|
+
| 3 | `tables` | Tables |
|
|
55
|
+
| 4 | `scalar_functions` | Scalar functions |
|
|
56
|
+
| 5 | `functions` | Functions |
|
|
57
|
+
| 6 | `views` | Views |
|
|
58
|
+
| 7 | `table_functions` | Table functions |
|
|
59
|
+
| 8 | `procedures` | Procedures |
|
|
60
|
+
| 100 | `permissions` | Grants |
|
|
61
|
+
| 101 | `indexes` | Indexes |
|
|
62
|
+
|
|
63
|
+
One object per file: `tables/user.sql`, `views/all_edits.sql`,
|
|
64
|
+
`types/measurement_unit.sql`.
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
## File-naming conventions
|
|
69
|
+
|
|
70
|
+
| Suffix | Meaning |
|
|
71
|
+
|---|---|
|
|
72
|
+
| `<name>.sql` | The object's live definition (`CREATE TABLE`, `CREATE OR REPLACE VIEW`, ...) |
|
|
73
|
+
| `<name>.test_data.json` | Seed rows for a table — a JSON array of row objects, loaded after the table is created |
|
|
74
|
+
| `<name>.init.sql` | One-time setup for an object (e.g. a backfill), run once, kept separate from the reusable definition |
|
|
75
|
+
| `<name>.prod.sql` / `.prod` anywhere in the name | Production-only (real permission grants, real user accounts) — skipped by `pgdb testdb` |
|
|
76
|
+
| `all.sql` | Generated concatenation of the whole tree — not hand-edited, not committed |
|
|
77
|
+
|
|
78
|
+
---
|
|
79
|
+
|
|
80
|
+
## Migrations
|
|
81
|
+
|
|
82
|
+
One-off, non-idempotent changes (rename/drop column, backfill, data fix) go
|
|
83
|
+
in a `migrations/` (or `_migration_scripts/`) folder — one file per change,
|
|
84
|
+
named by date:
|
|
85
|
+
|
|
86
|
+
```
|
|
87
|
+
database/migrations/2026-07-10_customer_geocode.sql
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Rules:
|
|
91
|
+
|
|
92
|
+
- Skipped by `pgdb testdb` — it only applies the layer directories above.
|
|
93
|
+
- Update the corresponding table/view `.sql` file in the same change so its
|
|
94
|
+
definition already reflects the new shape — the migration and the
|
|
95
|
+
source-of-truth file must never drift apart.
|
|
96
|
+
- Applied to production manually, once, by a human, after being verified
|
|
97
|
+
locally.
|
|
98
|
+
- Never edited after being applied — a further change gets a new dated file.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## `COMMENT ON` — document schema in the object's own file
|
|
103
|
+
|
|
104
|
+
Add a `COMMENT ON` for every table, and for any column whose purpose isn't
|
|
105
|
+
obvious from its name and type (flags, status codes, denormalized fields,
|
|
106
|
+
units). Put it directly in the table's `.sql` file, right after the
|
|
107
|
+
`CREATE TABLE` — not in a migration, wiki, or README. It lives with the
|
|
108
|
+
definition it describes and survives `\d+` / `pg_catalog` inspection.
|
|
109
|
+
|
|
110
|
+
```sql
|
|
111
|
+
-- database/1_reference_data/tables/user.sql
|
|
112
|
+
create table dim.user (
|
|
113
|
+
id bigint generated always as identity primary key,
|
|
114
|
+
email text not null,
|
|
115
|
+
is_active boolean not null default true
|
|
116
|
+
);
|
|
117
|
+
|
|
118
|
+
comment on table dim.user is 'End-user accounts; one row per registered person.';
|
|
119
|
+
comment on column dim.user.is_active is 'False once a user is soft-deleted; keep for audit trail.';
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## Backfilling untracked objects
|
|
125
|
+
|
|
126
|
+
If a table, scalar function, or table function was created directly on the
|
|
127
|
+
database and never got a `.sql` file, use `pgdb fetch-missing` to find it and
|
|
128
|
+
generate one — see `pgdb fetch-missing --help`. It connects to Postgres,
|
|
129
|
+
diffs the live schema against what's already tracked under `database/`, and
|
|
130
|
+
for each object you select, reverse-engineers its DDL into the matching
|
|
131
|
+
layer folder's `tables/`, `views/`, `scalar_functions/`, or
|
|
132
|
+
`table_functions/` subfolder. The layer folder is matched by schema name
|
|
133
|
+
against the existing top-level directories under `database/`, ignoring their
|
|
134
|
+
leading sort number.
|
|
135
|
+
|
|
136
|
+
---
|
|
137
|
+
|
|
138
|
+
## Quick checklist
|
|
139
|
+
|
|
140
|
+
- [ ] New table/view/function/type gets its own `.sql` file under the right layer + object-type folder
|
|
141
|
+
- [ ] Object-type folder (`tables`, `views`, ...) matches the apply-order table above — that's what governs ordering, not the layer's leading number
|
|
142
|
+
- [ ] One-off changes go in `migrations/`, dated, never edited after applying
|
|
143
|
+
- [ ] The live `.sql` file is updated in the same change as any migration touching that object
|
|
144
|
+
- [ ] `.prod` files are production-only and skipped by `pgdb testdb`
|
|
145
|
+
- [ ] Every table (and non-obvious column) has a `COMMENT ON`, placed in the object's own `.sql` file
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import psycopg
|
|
9
|
+
|
|
10
|
+
from .diff import DiffKind, compute_diff
|
|
11
|
+
from .introspect import introspect_db
|
|
12
|
+
from .parser import parse_directory
|
|
13
|
+
|
|
14
|
+
_LAYER_PREFIX_RE = re.compile(r"^\d+_?")
|
|
15
|
+
|
|
16
|
+
# Matches the `tables`/`views`/`scalar_functions`/`table_functions` object-type
|
|
17
|
+
# subfolders documented in docs/database-layout.md.
|
|
18
|
+
SUBFOLDER = {
|
|
19
|
+
"table": "tables",
|
|
20
|
+
"view": "views",
|
|
21
|
+
"scalar_function": "scalar_functions",
|
|
22
|
+
"table_function": "table_functions",
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class MissingObject:
|
|
28
|
+
object_type: str # "table" | "view" | "scalar_function" | "table_function"
|
|
29
|
+
schema: str
|
|
30
|
+
name: str
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def qualified_name(self) -> str:
|
|
34
|
+
return f"{self.schema}.{self.name}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def layer_folder_for(scripts_dir: Path, schema: str) -> Path:
|
|
38
|
+
"""Map a schema name to its existing layer directory under scripts_dir,
|
|
39
|
+
matched by stripping each top-level folder's leading sort number (e.g.
|
|
40
|
+
"1_reference_data" -> "reference_data"). Falls back to scripts_dir/schema
|
|
41
|
+
if no existing folder matches."""
|
|
42
|
+
if scripts_dir.is_dir():
|
|
43
|
+
for entry in scripts_dir.iterdir():
|
|
44
|
+
if not entry.is_dir() or entry.name in ("migrations", "_migration_scripts"):
|
|
45
|
+
continue
|
|
46
|
+
if _LAYER_PREFIX_RE.sub("", entry.name).lower() == schema.lower():
|
|
47
|
+
return entry
|
|
48
|
+
return scripts_dir / schema
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def find_missing_objects(scripts_dir: Path, conninfo: str) -> list[MissingObject]:
|
|
52
|
+
"""Tables, views, and functions that exist in the live database but
|
|
53
|
+
aren't tracked as .sql files under scripts_dir."""
|
|
54
|
+
scripts = parse_directory(scripts_dir)
|
|
55
|
+
db = introspect_db(conninfo)
|
|
56
|
+
diffs = compute_diff(scripts, db, report_extra_db=True)
|
|
57
|
+
|
|
58
|
+
missing: list[MissingObject] = []
|
|
59
|
+
with psycopg.connect(conninfo) as conn:
|
|
60
|
+
for d in diffs:
|
|
61
|
+
if d.kind != DiffKind.MISSING_IN_SCRIPTS or d.object_type not in ("table", "view", "function"):
|
|
62
|
+
continue
|
|
63
|
+
schema, name = d.object_name.split(".", 1)
|
|
64
|
+
object_type = d.object_type
|
|
65
|
+
if object_type == "function":
|
|
66
|
+
object_type = "table_function" if _returns_set(conn, schema, name) else "scalar_function"
|
|
67
|
+
missing.append(MissingObject(object_type, schema, name))
|
|
68
|
+
return missing
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _returns_set(conn: Any, schema: str, name: str) -> bool:
|
|
72
|
+
with conn.cursor() as cur:
|
|
73
|
+
cur.execute(
|
|
74
|
+
"""
|
|
75
|
+
SELECT p.proretset
|
|
76
|
+
FROM pg_catalog.pg_proc p
|
|
77
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
|
|
78
|
+
WHERE n.nspname = %s AND p.proname = %s AND p.prokind = 'f'
|
|
79
|
+
ORDER BY p.oid LIMIT 1
|
|
80
|
+
""",
|
|
81
|
+
(schema, name),
|
|
82
|
+
)
|
|
83
|
+
row = cur.fetchone()
|
|
84
|
+
return bool(row and row[0])
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def reconstruct_ddl(conn: Any, obj: MissingObject) -> str:
|
|
88
|
+
"""Reverse-engineer the DDL for a missing object from pg_catalog."""
|
|
89
|
+
if obj.object_type == "table":
|
|
90
|
+
return _table_ddl(conn, obj.schema, obj.name)
|
|
91
|
+
if obj.object_type == "view":
|
|
92
|
+
return _view_ddl(conn, obj.schema, obj.name)
|
|
93
|
+
return _function_ddl(conn, obj.schema, obj.name)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _table_ddl(conn: Any, schema: str, table: str) -> str:
|
|
97
|
+
with conn.cursor() as cur:
|
|
98
|
+
cur.execute(
|
|
99
|
+
"""
|
|
100
|
+
SELECT c.oid
|
|
101
|
+
FROM pg_catalog.pg_class c
|
|
102
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
103
|
+
WHERE n.nspname = %s AND c.relname = %s AND c.relkind = 'r'
|
|
104
|
+
""",
|
|
105
|
+
(schema, table),
|
|
106
|
+
)
|
|
107
|
+
row = cur.fetchone()
|
|
108
|
+
if not row:
|
|
109
|
+
raise ValueError(f"Table {schema}.{table} not found in pg_catalog")
|
|
110
|
+
oid = row[0]
|
|
111
|
+
|
|
112
|
+
cur.execute(
|
|
113
|
+
"""
|
|
114
|
+
SELECT
|
|
115
|
+
a.attname,
|
|
116
|
+
pg_catalog.format_type(a.atttypid, a.atttypmod),
|
|
117
|
+
a.attnotnull,
|
|
118
|
+
a.attidentity,
|
|
119
|
+
a.attgenerated,
|
|
120
|
+
CASE
|
|
121
|
+
WHEN a.attidentity = 'a' THEN 'GENERATED ALWAYS AS IDENTITY'
|
|
122
|
+
WHEN a.attidentity = 'd' THEN 'GENERATED BY DEFAULT AS IDENTITY'
|
|
123
|
+
WHEN a.attgenerated = 's' THEN
|
|
124
|
+
'GENERATED ALWAYS AS (' || pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) || ') STORED'
|
|
125
|
+
WHEN ad.adbin IS NOT NULL THEN
|
|
126
|
+
'DEFAULT ' || pg_catalog.pg_get_expr(ad.adbin, ad.adrelid)
|
|
127
|
+
ELSE NULL
|
|
128
|
+
END
|
|
129
|
+
FROM pg_catalog.pg_attribute a
|
|
130
|
+
LEFT JOIN pg_catalog.pg_attrdef ad
|
|
131
|
+
ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
|
|
132
|
+
WHERE a.attrelid = %s AND a.attnum > 0 AND NOT a.attisdropped
|
|
133
|
+
ORDER BY a.attnum
|
|
134
|
+
""",
|
|
135
|
+
(oid,),
|
|
136
|
+
)
|
|
137
|
+
columns = cur.fetchall()
|
|
138
|
+
|
|
139
|
+
cur.execute(
|
|
140
|
+
"""
|
|
141
|
+
SELECT conname, pg_catalog.pg_get_constraintdef(oid, true)
|
|
142
|
+
FROM pg_catalog.pg_constraint
|
|
143
|
+
WHERE conrelid = %s
|
|
144
|
+
ORDER BY contype, conname
|
|
145
|
+
""",
|
|
146
|
+
(oid,),
|
|
147
|
+
)
|
|
148
|
+
constraints = cur.fetchall()
|
|
149
|
+
|
|
150
|
+
cur.execute(
|
|
151
|
+
"""
|
|
152
|
+
SELECT indexname, indexdef
|
|
153
|
+
FROM pg_indexes
|
|
154
|
+
WHERE schemaname = %s AND tablename = %s
|
|
155
|
+
AND indexname NOT IN (
|
|
156
|
+
SELECT conname FROM pg_catalog.pg_constraint WHERE conrelid = %s
|
|
157
|
+
)
|
|
158
|
+
ORDER BY indexname
|
|
159
|
+
""",
|
|
160
|
+
(schema, table, oid),
|
|
161
|
+
)
|
|
162
|
+
indexes = cur.fetchall()
|
|
163
|
+
|
|
164
|
+
col_defs: list[str] = []
|
|
165
|
+
for attname, data_type, not_null, identity, _generated, extra in columns:
|
|
166
|
+
col_def = f"{attname} {data_type}"
|
|
167
|
+
if not_null and not identity:
|
|
168
|
+
col_def += " NOT NULL"
|
|
169
|
+
if extra:
|
|
170
|
+
col_def += f" {extra}"
|
|
171
|
+
col_defs.append(col_def)
|
|
172
|
+
|
|
173
|
+
for conname, condef in constraints:
|
|
174
|
+
col_defs.append(f"CONSTRAINT {conname} {condef}")
|
|
175
|
+
|
|
176
|
+
ddl = f"CREATE TABLE IF NOT EXISTS {schema}.{table} (\n"
|
|
177
|
+
ddl += ",\n".join(f" {c}" for c in col_defs)
|
|
178
|
+
ddl += "\n);\n"
|
|
179
|
+
|
|
180
|
+
for _, indexdef in indexes:
|
|
181
|
+
indexdef = re.sub(r"^CREATE INDEX\b", "CREATE INDEX IF NOT EXISTS", indexdef)
|
|
182
|
+
indexdef = re.sub(r"^CREATE UNIQUE INDEX\b", "CREATE UNIQUE INDEX IF NOT EXISTS", indexdef)
|
|
183
|
+
ddl += f"\n{indexdef};\n"
|
|
184
|
+
|
|
185
|
+
return ddl
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _view_ddl(conn: Any, schema: str, name: str) -> str:
|
|
189
|
+
with conn.cursor() as cur:
|
|
190
|
+
cur.execute(
|
|
191
|
+
"""
|
|
192
|
+
SELECT pg_get_viewdef(c.oid, true)
|
|
193
|
+
FROM pg_catalog.pg_class c
|
|
194
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
|
195
|
+
WHERE n.nspname = %s AND c.relname = %s AND c.relkind = 'v'
|
|
196
|
+
""",
|
|
197
|
+
(schema, name),
|
|
198
|
+
)
|
|
199
|
+
row = cur.fetchone()
|
|
200
|
+
if not row or row[0] is None:
|
|
201
|
+
raise ValueError(f"View {schema}.{name} not found in pg_catalog")
|
|
202
|
+
return f"CREATE OR REPLACE VIEW {schema}.{name} AS\n{row[0].rstrip()};\n"
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _function_ddl(conn: Any, schema: str, name: str) -> str:
|
|
206
|
+
"""Fetch a function's DDL via pg_get_functiondef.
|
|
207
|
+
|
|
208
|
+
If the function is overloaded, this uses the lowest-oid (oldest) match —
|
|
209
|
+
good enough to seed a file; resolve manually if there's more than one."""
|
|
210
|
+
with conn.cursor() as cur:
|
|
211
|
+
cur.execute(
|
|
212
|
+
"""
|
|
213
|
+
SELECT p.oid
|
|
214
|
+
FROM pg_catalog.pg_proc p
|
|
215
|
+
JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
|
|
216
|
+
WHERE n.nspname = %s AND p.proname = %s AND p.prokind = 'f'
|
|
217
|
+
ORDER BY p.oid LIMIT 1
|
|
218
|
+
""",
|
|
219
|
+
(schema, name),
|
|
220
|
+
)
|
|
221
|
+
row = cur.fetchone()
|
|
222
|
+
if not row:
|
|
223
|
+
raise ValueError(f"Function {schema}.{name} not found in pg_catalog")
|
|
224
|
+
|
|
225
|
+
cur.execute("SELECT pg_catalog.pg_get_functiondef(%s)", (row[0],))
|
|
226
|
+
ddl_row = cur.fetchone()
|
|
227
|
+
if not ddl_row or ddl_row[0] is None:
|
|
228
|
+
raise ValueError(f"Could not reconstruct DDL for function {schema}.{name}")
|
|
229
|
+
return ddl_row[0].rstrip() + ";\n"
|