postgresledger 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- postgresledger-0.0.1/LICENSE +21 -0
- postgresledger-0.0.1/PKG-INFO +289 -0
- postgresledger-0.0.1/README.md +270 -0
- postgresledger-0.0.1/pyproject.toml +38 -0
- postgresledger-0.0.1/pyproject.toml.orig +37 -0
- postgresledger-0.0.1/src/postgresledger/__init__.py +6 -0
- postgresledger-0.0.1/src/postgresledger/cli.py +205 -0
- postgresledger-0.0.1/src/postgresledger/commands.py +285 -0
- postgresledger-0.0.1/src/postgresledger/compare.py +116 -0
- postgresledger-0.0.1/src/postgresledger/config.py +82 -0
- postgresledger-0.0.1/src/postgresledger/core.py +288 -0
- postgresledger-0.0.1/src/postgresledger/pgschema.py +112 -0
- postgresledger-0.0.1/src/postgresledger/scratch.py +156 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nebojša Mrkić
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: postgresledger
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Versioned PostgreSQL migrations from plain SQL, without an ORM: generated from schema.sql, tracked per database, with drift detection
|
|
5
|
+
Author: Nebojša Mrkić
|
|
6
|
+
Author-email: Nebojša Mrkić <mrkic.nebojsa@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Topic :: Database
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Requires-Dist: python-dotenv>=1.2.4
|
|
13
|
+
Requires-Python: >=3.11
|
|
14
|
+
Project-URL: Homepage, https://github.com/nmrkic/postgresledger
|
|
15
|
+
Project-URL: Repository, https://github.com/nmrkic/postgresledger
|
|
16
|
+
Project-URL: Issues, https://github.com/nmrkic/postgresledger/issues
|
|
17
|
+
Project-URL: Changelog, https://github.com/nmrkic/postgresledger/releases
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
|
|
20
|
+
# postgresledger
|
|
21
|
+
|
|
22
|
+
Versioned migrations for PostgreSQL from plain SQL, without an ORM (tables,
|
|
23
|
+
views, materialized views, functions, triggers), built on
|
|
24
|
+
[pgschema](https://www.pgschema.com). The command is `pgl`.
|
|
25
|
+
|
|
26
|
+
`pgl` works on **one database per run**: the `DATABASE_*` variables from `.env`
|
|
27
|
+
(or the environment). Every database records which migrations it has; the
|
|
28
|
+
tool never needs to know about the others.
|
|
29
|
+
|
|
30
|
+
How it works, layout and rules:
|
|
31
|
+
[`docs/architecture.md`](https://github.com/nmrkic/postgresledger/blob/main/docs/architecture.md).
|
|
32
|
+
|
|
33
|
+
## What it does
|
|
34
|
+
|
|
35
|
+
- **Baseline** — dumps an existing database into `database/schema/schema.sql`
|
|
36
|
+
and `0001_baseline.sql`, so an empty database can be recreated from it.
|
|
37
|
+
A database that already has the same schema is recorded as up to date.
|
|
38
|
+
- **Migrations** — edit `schema.sql`, run `pgl new <name>`, get the next
|
|
39
|
+
`NNNN_name.sql` (verified before it is kept).
|
|
40
|
+
- **Apply** — runs the pending migrations and records them in
|
|
41
|
+
`postgresledger.schema_migrations` on the database.
|
|
42
|
+
- **Drift** — compares the database with what its migrations say it should
|
|
43
|
+
be, and lists changes made by hand.
|
|
44
|
+
- **Capture** — keep a manual change: a new migration, recorded as already
|
|
45
|
+
there on that database. (There is no command that drops manual changes on
|
|
46
|
+
a database; that is done by hand or with a normal migration, on purpose.)
|
|
47
|
+
- **Snapshot / compare** — save a database's schema to a file and list the
|
|
48
|
+
differences between two databases, object by object.
|
|
49
|
+
|
|
50
|
+
On the server, postgresledger only ever creates its tracking schema. Everything
|
|
51
|
+
that needs a scratch database (building "all migrations applied",
|
|
52
|
+
generating, drift, capture) runs in a throwaway Postgres on the
|
|
53
|
+
machine running `pgl`, deleted after each command (see
|
|
54
|
+
[Scratch Postgres](#scratch-postgres)).
|
|
55
|
+
|
|
56
|
+
## How it works
|
|
57
|
+
|
|
58
|
+
### Components
|
|
59
|
+
|
|
60
|
+
```mermaid
|
|
61
|
+
flowchart LR
|
|
62
|
+
dev([developer / CI]) -- pgl --> pgl[postgresledger]
|
|
63
|
+
pgl -- reads --> toml[postgresledger.toml<br/>project settings]
|
|
64
|
+
pgl -- reads --> env[.env<br/>DATABASE_*]
|
|
65
|
+
pgl -- dump / plan / apply --> pgschema[pgschema]
|
|
66
|
+
pgl -- run migrations --> psql[psql]
|
|
67
|
+
|
|
68
|
+
subgraph repo[Repository: database/]
|
|
69
|
+
schema[schema/schema.sql<br/>desired state]
|
|
70
|
+
migs[migrations/NNNN_name.sql]
|
|
71
|
+
snaps[snapshots/name.sql]
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
subgraph target[The database in .env]
|
|
75
|
+
db[(schema public<br/>+ postgresledger.schema_migrations)]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
subgraph local[This machine, per command]
|
|
79
|
+
scratch[(throwaway Postgres<br/>shadow + plan)]
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
pgl <--> schema
|
|
83
|
+
pgl --> migs
|
|
84
|
+
pgl --> snaps
|
|
85
|
+
psql --> db
|
|
86
|
+
pgschema --> db
|
|
87
|
+
psql --> scratch
|
|
88
|
+
pgschema --> scratch
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Scratch Postgres
|
|
92
|
+
|
|
93
|
+
Some commands need a database that can be dropped and recreated: `new`,
|
|
94
|
+
`drift`, `capture` and `baseline` build "every migration applied"
|
|
95
|
+
in one. That never happens on a real server, and it doesn't use Docker
|
|
96
|
+
either. pgl runs Postgres directly, as an ordinary process under your user:
|
|
97
|
+
|
|
98
|
+
- **Once per machine**, `pgl setup` downloads a ready-made Postgres 17 build
|
|
99
|
+
(the server programs and their libraries, from the
|
|
100
|
+
[zonky](https://github.com/zonkyio/embedded-postgres-binaries) project)
|
|
101
|
+
and unpacks it into `~/.cache/postgresledger/postgres-<version>/`. Nothing is
|
|
102
|
+
installed system-wide; no root, no `apt`, no Docker.
|
|
103
|
+
- **Per command**, pgl runs `initdb` into a fresh temp folder, starts the
|
|
104
|
+
server on `127.0.0.1` on a random free port (not reachable from outside
|
|
105
|
+
the machine), does its work in it, then stops it and deletes the folder,
|
|
106
|
+
also when the command fails.
|
|
107
|
+
|
|
108
|
+
```mermaid
|
|
109
|
+
flowchart LR
|
|
110
|
+
C([pgl drift]) --> I[initdb in /tmp/postgresledger-xxxx]
|
|
111
|
+
I --> S[start postgres<br/>127.0.0.1, free port]
|
|
112
|
+
S --> W[shadow: apply every migration<br/>compare with the database]
|
|
113
|
+
W --> X[stop postgres<br/>delete the temp folder]
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
It takes about a second; afterwards no process, file or port is left. `apply`,
|
|
117
|
+
`status`, `snapshot` and `compare` don't start it. It cannot run as root
|
|
118
|
+
(Postgres refuses to), and needs Linux or macOS on amd64 or arm64.
|
|
119
|
+
|
|
120
|
+
Why not Docker: it would have to be installed and running everywhere `pgl`
|
|
121
|
+
runs (CI runners, servers), many of those don't allow it, and a container
|
|
122
|
+
starts slower than a local process.
|
|
123
|
+
|
|
124
|
+
### Workflow
|
|
125
|
+
|
|
126
|
+
Each arrow runs with the `.env` of the database it acts on.
|
|
127
|
+
|
|
128
|
+
```mermaid
|
|
129
|
+
flowchart TD
|
|
130
|
+
A[Existing database, e.g. dev] -->|pgl baseline| B[schema.sql + 0001_baseline.sql<br/>recorded as baseline on dev]
|
|
131
|
+
B -->|pgl apply, on uat / prod| C[Every database up to date]
|
|
132
|
+
|
|
133
|
+
C -->|edit schema.sql<br/>pgl new name| D[NNNN_name.sql]
|
|
134
|
+
D -->|pgl apply, on each database| C
|
|
135
|
+
|
|
136
|
+
C -->|pgl drift| E{Manual changes?}
|
|
137
|
+
E -->|no, exit 0| C
|
|
138
|
+
E -->|yes, exit 1| F{Keep them?}
|
|
139
|
+
F -->|pgl capture name, on prod| G[NNNN_name.sql<br/>recorded as captured on prod,<br/>not executed there]
|
|
140
|
+
G -->|pgl apply on the other databases| C
|
|
141
|
+
F -->|no: undo by hand,<br/>then pgl drift| C
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### Generating a migration (`baseline`, `new`, `capture`)
|
|
145
|
+
|
|
146
|
+
```mermaid
|
|
147
|
+
flowchart TD
|
|
148
|
+
S([start]) --> R[Start a throwaway Postgres on this machine]
|
|
149
|
+
R --> A[Apply every existing migration to it]
|
|
150
|
+
A --> P[pgschema plan<br/>schema.sql vs that state]
|
|
151
|
+
P --> Q{Plan empty?}
|
|
152
|
+
Q -->|yes| N([no changes, nothing written])
|
|
153
|
+
Q -->|no| W[Write next NNNN_name.sql]
|
|
154
|
+
W --> V[Apply it there too, plan again]
|
|
155
|
+
V --> Q2{Plan empty?}
|
|
156
|
+
Q2 -->|yes| OK([migration kept, throwaway Postgres deleted])
|
|
157
|
+
Q2 -->|no| X([file deleted, command fails])
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
`baseline` dumps the database into `schema.sql` first and records `0001` as
|
|
161
|
+
`baseline` there; `capture` dumps the database into `schema.sql` first and
|
|
162
|
+
records the new migration as `captured` there. `new` does not touch the
|
|
163
|
+
database at all.
|
|
164
|
+
|
|
165
|
+
### Drift and capture
|
|
166
|
+
|
|
167
|
+
```mermaid
|
|
168
|
+
sequenceDiagram
|
|
169
|
+
actor U as developer
|
|
170
|
+
participant T as pgl
|
|
171
|
+
participant S as throwaway Postgres
|
|
172
|
+
participant P as prod (.env)
|
|
173
|
+
|
|
174
|
+
Note over P: someone changes prod by hand
|
|
175
|
+
U->>T: pgl drift
|
|
176
|
+
T->>P: all migrations applied?
|
|
177
|
+
T->>S: start, apply all migrations (expected state)
|
|
178
|
+
T->>P: pgschema dump (actual state)
|
|
179
|
+
T->>T: pgschema plan actual vs expected
|
|
180
|
+
T-->>U: DRIFT: list of manual changes, exit 1
|
|
181
|
+
|
|
182
|
+
U->>T: pgl capture orders_notes
|
|
183
|
+
T->>P: pgschema dump -> schema.sql
|
|
184
|
+
T->>S: generate + verify NNNN_orders_notes.sql
|
|
185
|
+
T->>P: record NNNN as captured (not executed)
|
|
186
|
+
Note over U: with the uat / dev .env: pgl apply
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
### Apply
|
|
190
|
+
|
|
191
|
+
```mermaid
|
|
192
|
+
flowchart LR
|
|
193
|
+
S([pgl apply]) --> C{already applied files<br/>unchanged? checksum}
|
|
194
|
+
C -->|no| E([fail: file was edited])
|
|
195
|
+
C -->|yes| F[next pending NNNN_name.sql]
|
|
196
|
+
F --> K{contains CONCURRENTLY?}
|
|
197
|
+
K -->|no| T[BEGIN<br/>pg_advisory_xact_lock<br/>migration SQL<br/>INSERT schema_migrations<br/>COMMIT]
|
|
198
|
+
K -->|yes| N[statements one by one<br/>then INSERT schema_migrations]
|
|
199
|
+
T -->|more pending| F
|
|
200
|
+
N -->|more pending| F
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
## Install
|
|
204
|
+
|
|
205
|
+
Requires Python 3.11+ and `psql`. `pgl init` / `pgl setup` download the rest
|
|
206
|
+
once per machine into `~/.cache/postgresledger/`: pgschema and a Postgres
|
|
207
|
+
server for the throwaway scratch databases (both pinned, SHA256 checked).
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
uv tool install postgresledger # or: pipx install postgresledger
|
|
211
|
+
pgl --version
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
| OS | Install the prerequisites |
|
|
215
|
+
|---|---|
|
|
216
|
+
| Linux (Debian/Ubuntu) | `sudo apt install python3 postgresql-client` |
|
|
217
|
+
| macOS | `brew install python libpq && brew link --force libpq` |
|
|
218
|
+
| Windows | no pgschema build — use [WSL2](https://learn.microsoft.com/windows/wsl/install) and follow the Linux row |
|
|
219
|
+
|
|
220
|
+
Supported: Linux and macOS, amd64 and arm64. The scratch Postgres cannot
|
|
221
|
+
run as root (run `pgl` as a normal user, also in CI containers). To bump
|
|
222
|
+
pgschema or the scratch Postgres, change `VERSION` and the hashes in
|
|
223
|
+
`src/postgresledger/pgschema.py` / `src/postgresledger/scratch.py`.
|
|
224
|
+
|
|
225
|
+
## Usage
|
|
226
|
+
|
|
227
|
+
In the project that owns the database:
|
|
228
|
+
|
|
229
|
+
```bash
|
|
230
|
+
pgl init # postgresledger.toml, .env (git-ignored), database/schema/, database/migrations/
|
|
231
|
+
# set the DATABASE_* variables in .env (see below)
|
|
232
|
+
|
|
233
|
+
pgl baseline # 1. snapshot the existing database as 0001_baseline
|
|
234
|
+
|
|
235
|
+
# 2. edit database/schema/schema.sql, then
|
|
236
|
+
pgl new add_loyalty_and_coupons
|
|
237
|
+
pgl apply # --dry-run to preview
|
|
238
|
+
pgl status
|
|
239
|
+
|
|
240
|
+
# 3. someone changed the database by hand
|
|
241
|
+
pgl drift # exit code 1 on drift
|
|
242
|
+
pgl capture orders_notes # keep it: other databases get it with `pgl apply`
|
|
243
|
+
# or undo it by hand, then `pgl drift` again
|
|
244
|
+
|
|
245
|
+
# 4. how do two databases differ?
|
|
246
|
+
pgl snapshot prod # with prod's .env -> database/snapshots/prod.sql
|
|
247
|
+
pgl snapshot uat # with uat's .env
|
|
248
|
+
pgl compare uat prod # or `pgl compare prod`: the snapshot vs the database itself
|
|
249
|
+
```
|
|
250
|
+
|
|
251
|
+
The database comes from these variables in `.env`:
|
|
252
|
+
|
|
253
|
+
```bash
|
|
254
|
+
DATABASE_HOST=localhost # default: localhost
|
|
255
|
+
DATABASE_PORT=5432 # default: 5432
|
|
256
|
+
DATABASE_USER=postgres # default: postgres
|
|
257
|
+
DATABASE_PASSWORD=secret # default: empty
|
|
258
|
+
DATABASE_NAME=app # required
|
|
259
|
+
DATABASE_SSLMODE=require # default: prefer
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
Each server or CI job has its own `.env`, or sets the variables in its
|
|
263
|
+
environment; a variable set there wins over `.env`. `pgl --env-file .env.uat …`
|
|
264
|
+
(or `PGL_ENV_FILE=.env.uat`) picks another file for one run.
|
|
265
|
+
Run `pgl` in the folder with `postgresledger.toml`, or point to it with
|
|
266
|
+
`--config PATH`. `pgl <command> -h` describes each command.
|
|
267
|
+
|
|
268
|
+
## Developing postgresledger
|
|
269
|
+
|
|
270
|
+
Needs [uv](https://docs.astral.sh/uv/) and Docker. This repository is also
|
|
271
|
+
the tool's test bed: a local Postgres in `docker/`
|
|
272
|
+
with `dev`, `stage` and `prod`, and demo data in `database/seed/`. `.env`
|
|
273
|
+
points at `dev`; `make <target> ENV=stage` runs one command on another
|
|
274
|
+
database, and `make apply-all` / `drift-all` / `status-all` /
|
|
275
|
+
`snapshot-all` loop over the three.
|
|
276
|
+
|
|
277
|
+
```bash
|
|
278
|
+
make setup # uv sync + pgl setup
|
|
279
|
+
make up # postgres:17 on localhost:55432
|
|
280
|
+
make seed # demo schema + data into dev
|
|
281
|
+
make test # unit tests + the full flow against the local Postgres
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
`make help` lists every target; `make reset` starts from scratch (empty
|
|
285
|
+
Postgres, seeded dev, no schema, migrations or snapshots).
|
|
286
|
+
|
|
287
|
+
## License
|
|
288
|
+
|
|
289
|
+
MIT, see [LICENSE](https://github.com/nmrkic/postgresledger/blob/main/LICENSE).
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# postgresledger
|
|
2
|
+
|
|
3
|
+
Versioned migrations for PostgreSQL from plain SQL, without an ORM (tables,
|
|
4
|
+
views, materialized views, functions, triggers), built on
|
|
5
|
+
[pgschema](https://www.pgschema.com). The command is `pgl`.
|
|
6
|
+
|
|
7
|
+
`pgl` works on **one database per run**: the `DATABASE_*` variables from `.env`
|
|
8
|
+
(or the environment). Every database records which migrations it has; the
|
|
9
|
+
tool never needs to know about the others.
|
|
10
|
+
|
|
11
|
+
How it works, layout and rules:
|
|
12
|
+
[`docs/architecture.md`](https://github.com/nmrkic/postgresledger/blob/main/docs/architecture.md).
|
|
13
|
+
|
|
14
|
+
## What it does
|
|
15
|
+
|
|
16
|
+
- **Baseline** — dumps an existing database into `database/schema/schema.sql`
|
|
17
|
+
and `0001_baseline.sql`, so an empty database can be recreated from it.
|
|
18
|
+
A database that already has the same schema is recorded as up to date.
|
|
19
|
+
- **Migrations** — edit `schema.sql`, run `pgl new <name>`, get the next
|
|
20
|
+
`NNNN_name.sql` (verified before it is kept).
|
|
21
|
+
- **Apply** — runs the pending migrations and records them in
|
|
22
|
+
`postgresledger.schema_migrations` on the database.
|
|
23
|
+
- **Drift** — compares the database with what its migrations say it should
|
|
24
|
+
be, and lists changes made by hand.
|
|
25
|
+
- **Capture** — keep a manual change: a new migration, recorded as already
|
|
26
|
+
there on that database. (There is no command that drops manual changes on
|
|
27
|
+
a database; that is done by hand or with a normal migration, on purpose.)
|
|
28
|
+
- **Snapshot / compare** — save a database's schema to a file and list the
|
|
29
|
+
differences between two databases, object by object.
|
|
30
|
+
|
|
31
|
+
On the server, postgresledger only ever creates its tracking schema. Everything
|
|
32
|
+
that needs a scratch database (building "all migrations applied",
|
|
33
|
+
generating, drift, capture) runs in a throwaway Postgres on the
|
|
34
|
+
machine running `pgl`, deleted after each command (see
|
|
35
|
+
[Scratch Postgres](#scratch-postgres)).
|
|
36
|
+
|
|
37
|
+
## How it works
|
|
38
|
+
|
|
39
|
+
### Components
|
|
40
|
+
|
|
41
|
+
```mermaid
|
|
42
|
+
flowchart LR
|
|
43
|
+
dev([developer / CI]) -- pgl --> pgl[postgresledger]
|
|
44
|
+
pgl -- reads --> toml[postgresledger.toml<br/>project settings]
|
|
45
|
+
pgl -- reads --> env[.env<br/>DATABASE_*]
|
|
46
|
+
pgl -- dump / plan / apply --> pgschema[pgschema]
|
|
47
|
+
pgl -- run migrations --> psql[psql]
|
|
48
|
+
|
|
49
|
+
subgraph repo[Repository: database/]
|
|
50
|
+
schema[schema/schema.sql<br/>desired state]
|
|
51
|
+
migs[migrations/NNNN_name.sql]
|
|
52
|
+
snaps[snapshots/name.sql]
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
subgraph target[The database in .env]
|
|
56
|
+
db[(schema public<br/>+ postgresledger.schema_migrations)]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
subgraph local[This machine, per command]
|
|
60
|
+
scratch[(throwaway Postgres<br/>shadow + plan)]
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
pgl <--> schema
|
|
64
|
+
pgl --> migs
|
|
65
|
+
pgl --> snaps
|
|
66
|
+
psql --> db
|
|
67
|
+
pgschema --> db
|
|
68
|
+
psql --> scratch
|
|
69
|
+
pgschema --> scratch
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### Scratch Postgres
|
|
73
|
+
|
|
74
|
+
Some commands need a database that can be dropped and recreated: `new`,
|
|
75
|
+
`drift`, `capture` and `baseline` build "every migration applied"
|
|
76
|
+
in one. That never happens on a real server, and it doesn't use Docker
|
|
77
|
+
either. pgl runs Postgres directly, as an ordinary process under your user:
|
|
78
|
+
|
|
79
|
+
- **Once per machine**, `pgl setup` downloads a ready-made Postgres 17 build
|
|
80
|
+
(the server programs and their libraries, from the
|
|
81
|
+
[zonky](https://github.com/zonkyio/embedded-postgres-binaries) project)
|
|
82
|
+
and unpacks it into `~/.cache/postgresledger/postgres-<version>/`. Nothing is
|
|
83
|
+
installed system-wide; no root, no `apt`, no Docker.
|
|
84
|
+
- **Per command**, pgl runs `initdb` into a fresh temp folder, starts the
|
|
85
|
+
server on `127.0.0.1` on a random free port (not reachable from outside
|
|
86
|
+
the machine), does its work in it, then stops it and deletes the folder,
|
|
87
|
+
also when the command fails.
|
|
88
|
+
|
|
89
|
+
```mermaid
|
|
90
|
+
flowchart LR
|
|
91
|
+
C([pgl drift]) --> I[initdb in /tmp/postgresledger-xxxx]
|
|
92
|
+
I --> S[start postgres<br/>127.0.0.1, free port]
|
|
93
|
+
S --> W[shadow: apply every migration<br/>compare with the database]
|
|
94
|
+
W --> X[stop postgres<br/>delete the temp folder]
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
It takes about a second; afterwards no process, file or port is left. `apply`,
|
|
98
|
+
`status`, `snapshot` and `compare` don't start it. It cannot run as root
|
|
99
|
+
(Postgres refuses to), and needs Linux or macOS on amd64 or arm64.
|
|
100
|
+
|
|
101
|
+
Why not Docker: it would have to be installed and running everywhere `pgl`
|
|
102
|
+
runs (CI runners, servers), many of those don't allow it, and a container
|
|
103
|
+
starts slower than a local process.
|
|
104
|
+
|
|
105
|
+
### Workflow
|
|
106
|
+
|
|
107
|
+
Each arrow runs with the `.env` of the database it acts on.
|
|
108
|
+
|
|
109
|
+
```mermaid
|
|
110
|
+
flowchart TD
|
|
111
|
+
A[Existing database, e.g. dev] -->|pgl baseline| B[schema.sql + 0001_baseline.sql<br/>recorded as baseline on dev]
|
|
112
|
+
B -->|pgl apply, on uat / prod| C[Every database up to date]
|
|
113
|
+
|
|
114
|
+
C -->|edit schema.sql<br/>pgl new name| D[NNNN_name.sql]
|
|
115
|
+
D -->|pgl apply, on each database| C
|
|
116
|
+
|
|
117
|
+
C -->|pgl drift| E{Manual changes?}
|
|
118
|
+
E -->|no, exit 0| C
|
|
119
|
+
E -->|yes, exit 1| F{Keep them?}
|
|
120
|
+
F -->|pgl capture name, on prod| G[NNNN_name.sql<br/>recorded as captured on prod,<br/>not executed there]
|
|
121
|
+
G -->|pgl apply on the other databases| C
|
|
122
|
+
F -->|no: undo by hand,<br/>then pgl drift| C
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
### Generating a migration (`baseline`, `new`, `capture`)
|
|
126
|
+
|
|
127
|
+
```mermaid
|
|
128
|
+
flowchart TD
|
|
129
|
+
S([start]) --> R[Start a throwaway Postgres on this machine]
|
|
130
|
+
R --> A[Apply every existing migration to it]
|
|
131
|
+
A --> P[pgschema plan<br/>schema.sql vs that state]
|
|
132
|
+
P --> Q{Plan empty?}
|
|
133
|
+
Q -->|yes| N([no changes, nothing written])
|
|
134
|
+
Q -->|no| W[Write next NNNN_name.sql]
|
|
135
|
+
W --> V[Apply it there too, plan again]
|
|
136
|
+
V --> Q2{Plan empty?}
|
|
137
|
+
Q2 -->|yes| OK([migration kept, throwaway Postgres deleted])
|
|
138
|
+
Q2 -->|no| X([file deleted, command fails])
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
`baseline` dumps the database into `schema.sql` first and records `0001` as
|
|
142
|
+
`baseline` there; `capture` dumps the database into `schema.sql` first and
|
|
143
|
+
records the new migration as `captured` there. `new` does not touch the
|
|
144
|
+
database at all.
|
|
145
|
+
|
|
146
|
+
### Drift and capture
|
|
147
|
+
|
|
148
|
+
```mermaid
|
|
149
|
+
sequenceDiagram
|
|
150
|
+
actor U as developer
|
|
151
|
+
participant T as pgl
|
|
152
|
+
participant S as throwaway Postgres
|
|
153
|
+
participant P as prod (.env)
|
|
154
|
+
|
|
155
|
+
Note over P: someone changes prod by hand
|
|
156
|
+
U->>T: pgl drift
|
|
157
|
+
T->>P: all migrations applied?
|
|
158
|
+
T->>S: start, apply all migrations (expected state)
|
|
159
|
+
T->>P: pgschema dump (actual state)
|
|
160
|
+
T->>T: pgschema plan actual vs expected
|
|
161
|
+
T-->>U: DRIFT: list of manual changes, exit 1
|
|
162
|
+
|
|
163
|
+
U->>T: pgl capture orders_notes
|
|
164
|
+
T->>P: pgschema dump -> schema.sql
|
|
165
|
+
T->>S: generate + verify NNNN_orders_notes.sql
|
|
166
|
+
T->>P: record NNNN as captured (not executed)
|
|
167
|
+
Note over U: with the uat / dev .env: pgl apply
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
### Apply
|
|
171
|
+
|
|
172
|
+
```mermaid
|
|
173
|
+
flowchart LR
|
|
174
|
+
S([pgl apply]) --> C{already applied files<br/>unchanged? checksum}
|
|
175
|
+
C -->|no| E([fail: file was edited])
|
|
176
|
+
C -->|yes| F[next pending NNNN_name.sql]
|
|
177
|
+
F --> K{contains CONCURRENTLY?}
|
|
178
|
+
K -->|no| T[BEGIN<br/>pg_advisory_xact_lock<br/>migration SQL<br/>INSERT schema_migrations<br/>COMMIT]
|
|
179
|
+
K -->|yes| N[statements one by one<br/>then INSERT schema_migrations]
|
|
180
|
+
T -->|more pending| F
|
|
181
|
+
N -->|more pending| F
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
## Install
|
|
185
|
+
|
|
186
|
+
Requires Python 3.11+ and `psql`. `pgl init` / `pgl setup` download the rest
|
|
187
|
+
once per machine into `~/.cache/postgresledger/`: pgschema and a Postgres
|
|
188
|
+
server for the throwaway scratch databases (both pinned, SHA256 checked).
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
uv tool install postgresledger # or: pipx install postgresledger
|
|
192
|
+
pgl --version
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
| OS | Install the prerequisites |
|
|
196
|
+
|---|---|
|
|
197
|
+
| Linux (Debian/Ubuntu) | `sudo apt install python3 postgresql-client` |
|
|
198
|
+
| macOS | `brew install python libpq && brew link --force libpq` |
|
|
199
|
+
| Windows | no pgschema build — use [WSL2](https://learn.microsoft.com/windows/wsl/install) and follow the Linux row |
|
|
200
|
+
|
|
201
|
+
Supported: Linux and macOS, amd64 and arm64. The scratch Postgres cannot
|
|
202
|
+
run as root (run `pgl` as a normal user, also in CI containers). To bump
|
|
203
|
+
pgschema or the scratch Postgres, change `VERSION` and the hashes in
|
|
204
|
+
`src/postgresledger/pgschema.py` / `src/postgresledger/scratch.py`.
|
|
205
|
+
|
|
206
|
+
## Usage
|
|
207
|
+
|
|
208
|
+
In the project that owns the database:
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
pgl init # postgresledger.toml, .env (git-ignored), database/schema/, database/migrations/
|
|
212
|
+
# set the DATABASE_* variables in .env (see below)
|
|
213
|
+
|
|
214
|
+
pgl baseline # 1. snapshot the existing database as 0001_baseline
|
|
215
|
+
|
|
216
|
+
# 2. edit database/schema/schema.sql, then
|
|
217
|
+
pgl new add_loyalty_and_coupons
|
|
218
|
+
pgl apply # --dry-run to preview
|
|
219
|
+
pgl status
|
|
220
|
+
|
|
221
|
+
# 3. someone changed the database by hand
|
|
222
|
+
pgl drift # exit code 1 on drift
|
|
223
|
+
pgl capture orders_notes # keep it: other databases get it with `pgl apply`
|
|
224
|
+
# or undo it by hand, then `pgl drift` again
|
|
225
|
+
|
|
226
|
+
# 4. how do two databases differ?
|
|
227
|
+
pgl snapshot prod # with prod's .env -> database/snapshots/prod.sql
|
|
228
|
+
pgl snapshot uat # with uat's .env
|
|
229
|
+
pgl compare uat prod # or `pgl compare prod`: the snapshot vs the database itself
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
The database comes from these variables in `.env`:
|
|
233
|
+
|
|
234
|
+
```bash
|
|
235
|
+
DATABASE_HOST=localhost # default: localhost
|
|
236
|
+
DATABASE_PORT=5432 # default: 5432
|
|
237
|
+
DATABASE_USER=postgres # default: postgres
|
|
238
|
+
DATABASE_PASSWORD=secret # default: empty
|
|
239
|
+
DATABASE_NAME=app # required
|
|
240
|
+
DATABASE_SSLMODE=require # default: prefer
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
Each server or CI job has its own `.env`, or sets the variables in its
|
|
244
|
+
environment; a variable set there wins over `.env`. `pgl --env-file .env.uat …`
|
|
245
|
+
(or `PGL_ENV_FILE=.env.uat`) picks another file for one run.
|
|
246
|
+
Run `pgl` in the folder with `postgresledger.toml`, or point to it with
|
|
247
|
+
`--config PATH`. `pgl <command> -h` describes each command.
|
|
248
|
+
|
|
249
|
+
## Developing postgresledger
|
|
250
|
+
|
|
251
|
+
Needs [uv](https://docs.astral.sh/uv/) and Docker. This repository is also
|
|
252
|
+
the tool's test bed: a local Postgres in `docker/`
|
|
253
|
+
with `dev`, `stage` and `prod`, and demo data in `database/seed/`. `.env`
|
|
254
|
+
points at `dev`; `make <target> ENV=stage` runs one command on another
|
|
255
|
+
database, and `make apply-all` / `drift-all` / `status-all` /
|
|
256
|
+
`snapshot-all` loop over the three.
|
|
257
|
+
|
|
258
|
+
```bash
|
|
259
|
+
make setup # uv sync + pgl setup
|
|
260
|
+
make up # postgres:17 on localhost:55432
|
|
261
|
+
make seed # demo schema + data into dev
|
|
262
|
+
make test # unit tests + the full flow against the local Postgres
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
`make help` lists every target; `make reset` starts from scratch (empty
|
|
266
|
+
Postgres, seeded dev, no schema, migrations or snapshots).
|
|
267
|
+
|
|
268
|
+
## License
|
|
269
|
+
|
|
270
|
+
MIT, see [LICENSE](https://github.com/nmrkic/postgresledger/blob/main/LICENSE).
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "postgresledger"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "Versioned PostgreSQL migrations from plain SQL, without an ORM: generated from schema.sql, tracked per database, with drift detection"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.11"
|
|
9
|
+
dependencies = ["python-dotenv>=1.2.4"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Topic :: Database",
|
|
13
|
+
"Environment :: Console",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[[project.authors]]
|
|
17
|
+
name = "Nebojša Mrkić"
|
|
18
|
+
email = "mrkic.nebojsa@gmail.com"
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Homepage = "https://github.com/nmrkic/postgresledger"
|
|
22
|
+
Repository = "https://github.com/nmrkic/postgresledger"
|
|
23
|
+
Issues = "https://github.com/nmrkic/postgresledger/issues"
|
|
24
|
+
Changelog = "https://github.com/nmrkic/postgresledger/releases"
|
|
25
|
+
|
|
26
|
+
[project.scripts]
|
|
27
|
+
pgl = "postgresledger.cli:main"
|
|
28
|
+
postgresledger = "postgresledger.cli:main"
|
|
29
|
+
|
|
30
|
+
[dependency-groups]
|
|
31
|
+
dev = ["pytest>=8"]
|
|
32
|
+
|
|
33
|
+
[build-system]
|
|
34
|
+
requires = ["uv_build>=0.9,<0.12"]
|
|
35
|
+
build-backend = "uv_build"
|
|
36
|
+
|
|
37
|
+
[tool.pytest.ini_options]
|
|
38
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "postgresledger"
|
|
3
|
+
version = "0.0.1" # set from the release tag by .github/workflows/publish.yml, not here
|
|
4
|
+
description = "Versioned PostgreSQL migrations from plain SQL, without an ORM: generated from schema.sql, tracked per database, with drift detection"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
requires-python = ">=3.11"
|
|
9
|
+
authors = [{ name = "Nebojša Mrkić", email = "mrkic.nebojsa@gmail.com" }]
|
|
10
|
+
dependencies = [
|
|
11
|
+
"python-dotenv>=1.2.4",
|
|
12
|
+
]
|
|
13
|
+
classifiers = [
|
|
14
|
+
"Programming Language :: Python :: 3",
|
|
15
|
+
"Topic :: Database",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[project.urls]
|
|
20
|
+
Homepage = "https://github.com/nmrkic/postgresledger"
|
|
21
|
+
Repository = "https://github.com/nmrkic/postgresledger"
|
|
22
|
+
Issues = "https://github.com/nmrkic/postgresledger/issues"
|
|
23
|
+
Changelog = "https://github.com/nmrkic/postgresledger/releases"
|
|
24
|
+
|
|
25
|
+
[project.scripts]
|
|
26
|
+
pgl = "postgresledger.cli:main"
|
|
27
|
+
postgresledger = "postgresledger.cli:main"
|
|
28
|
+
|
|
29
|
+
[dependency-groups]
|
|
30
|
+
dev = ["pytest>=8"]
|
|
31
|
+
|
|
32
|
+
[build-system]
|
|
33
|
+
requires = ["uv_build>=0.9,<0.12"]
|
|
34
|
+
build-backend = "uv_build"
|
|
35
|
+
|
|
36
|
+
[tool.pytest.ini_options]
|
|
37
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
"""postgresledger - versioned PostgreSQL migrations from plain SQL, on top of pgschema."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import version
|
|
4
|
+
|
|
5
|
+
# the version of the installed package: the release tag it was built from (0.0.0 in a local checkout)
|
|
6
|
+
__version__ = version("postgresledger")
|