pgsesame 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pgsesame-0.1.0/.dockerignore +10 -0
- pgsesame-0.1.0/.github/workflows/ci.yml +99 -0
- pgsesame-0.1.0/.github/workflows/release.yml +101 -0
- pgsesame-0.1.0/.gitignore +11 -0
- pgsesame-0.1.0/.pre-commit-config.yaml +32 -0
- pgsesame-0.1.0/.python-version +1 -0
- pgsesame-0.1.0/CHANGELOG.md +30 -0
- pgsesame-0.1.0/DESIGN.md +272 -0
- pgsesame-0.1.0/Dockerfile +19 -0
- pgsesame-0.1.0/LICENSE +202 -0
- pgsesame-0.1.0/PKG-INFO +136 -0
- pgsesame-0.1.0/README.md +110 -0
- pgsesame-0.1.0/docs/images/change-set.png +0 -0
- pgsesame-0.1.0/docs/images/drift.png +0 -0
- pgsesame-0.1.0/docs/images/plan.png +0 -0
- pgsesame-0.1.0/examples/redshift.yaml +40 -0
- pgsesame-0.1.0/pyproject.toml +120 -0
- pgsesame-0.1.0/src/pgsesame/__init__.py +8 -0
- pgsesame-0.1.0/src/pgsesame/aws.py +182 -0
- pgsesame-0.1.0/src/pgsesame/changeset.py +121 -0
- pgsesame-0.1.0/src/pgsesame/cli.py +345 -0
- pgsesame-0.1.0/src/pgsesame/console.py +41 -0
- pgsesame-0.1.0/src/pgsesame/db.py +90 -0
- pgsesame-0.1.0/src/pgsesame/ops.py +269 -0
- pgsesame-0.1.0/src/pgsesame/planner.py +215 -0
- pgsesame-0.1.0/src/pgsesame/postgres.py +89 -0
- pgsesame-0.1.0/src/pgsesame/redshift.py +100 -0
- pgsesame-0.1.0/src/pgsesame/spec.py +254 -0
- pgsesame-0.1.0/src/pgsesame/state.py +54 -0
- pgsesame-0.1.0/tests/aws/README.md +32 -0
- pgsesame-0.1.0/tests/aws/serverless.yaml +46 -0
- pgsesame-0.1.0/tests/test_aws.py +188 -0
- pgsesame-0.1.0/tests/test_data_api.py +82 -0
- pgsesame-0.1.0/tests/test_ops.py +82 -0
- pgsesame-0.1.0/tests/test_planner.py +105 -0
- pgsesame-0.1.0/tests/test_postgres.py +267 -0
- pgsesame-0.1.0/tests/test_redshift.py +194 -0
- pgsesame-0.1.0/tests/test_spec.py +158 -0
- pgsesame-0.1.0/uv.lock +1217 -0
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# Lint, types and tests on every pull request and on main. The PostgreSQL tests
|
|
2
|
+
# run against a real server (a service container); the Redshift tests need
|
|
3
|
+
# oblako's redshift-local and run locally (see tests/test_redshift.py).
|
|
4
|
+
name: ci
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
pull_request:
|
|
8
|
+
push:
|
|
9
|
+
branches: [main]
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
|
|
14
|
+
concurrency:
|
|
15
|
+
group: ci-${{ github.ref }}
|
|
16
|
+
cancel-in-progress: true
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
check:
|
|
20
|
+
name: tests (python ${{ matrix.python }}, postgres ${{ matrix.postgres }})
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
strategy:
|
|
23
|
+
fail-fast: false
|
|
24
|
+
matrix:
|
|
25
|
+
# every PostgreSQL still supported (the access catalogs change between
|
|
26
|
+
# versions: pg_auth_members in 16, MAINTAIN in 17) on the hooks' Python, and
|
|
27
|
+
# the oldest and newest Python pyproject.toml allows on one PostgreSQL
|
|
28
|
+
include:
|
|
29
|
+
- {python: "3.12", postgres: "14"}
|
|
30
|
+
- {python: "3.12", postgres: "15"}
|
|
31
|
+
- {python: "3.12", postgres: "16"}
|
|
32
|
+
- {python: "3.12", postgres: "17"}
|
|
33
|
+
- {python: "3.12", postgres: "18"}
|
|
34
|
+
- {python: "3.10", postgres: "16"}
|
|
35
|
+
- {python: "3.13", postgres: "16"}
|
|
36
|
+
services:
|
|
37
|
+
postgres:
|
|
38
|
+
image: postgres:${{ matrix.postgres }}
|
|
39
|
+
env:
|
|
40
|
+
POSTGRES_USER: sesame
|
|
41
|
+
POSTGRES_PASSWORD: sesame
|
|
42
|
+
POSTGRES_DB: sesame
|
|
43
|
+
ports: ["5432:5432"]
|
|
44
|
+
options: >-
|
|
45
|
+
--health-cmd "pg_isready -U sesame" --health-interval 2s
|
|
46
|
+
--health-timeout 5s --health-retries 30
|
|
47
|
+
env:
|
|
48
|
+
PGSESAME_TEST_DSN: postgresql://sesame:sesame@localhost:5432/sesame
|
|
49
|
+
steps:
|
|
50
|
+
- uses: actions/checkout@v4
|
|
51
|
+
|
|
52
|
+
- name: Install uv
|
|
53
|
+
uses: astral-sh/setup-uv@v5
|
|
54
|
+
|
|
55
|
+
- name: Install the project
|
|
56
|
+
run: uv sync --locked --python ${{ matrix.python }}
|
|
57
|
+
|
|
58
|
+
- name: Hooks (ruff, pydocstyle, ty)
|
|
59
|
+
if: matrix.python == '3.12' && matrix.postgres == '16'
|
|
60
|
+
run: uvx prek run --all-files --show-diff-on-failure
|
|
61
|
+
|
|
62
|
+
- name: Tests
|
|
63
|
+
run: uv run pytest -q
|
|
64
|
+
|
|
65
|
+
redshift:
|
|
66
|
+
name: redshift (oblako's redshift-local)
|
|
67
|
+
runs-on: ubuntu-latest
|
|
68
|
+
services:
|
|
69
|
+
redshift:
|
|
70
|
+
# pinned by digest: :16 moves with every oblako image change. To update,
|
|
71
|
+
# docker buildx imagetools inspect public.ecr.aws/oblako/redshift-local:16
|
|
72
|
+
# (oblako d56528d: users, groups, roles and the SVV privilege views)
|
|
73
|
+
image: public.ecr.aws/oblako/redshift-local:16@sha256:3d2618b23e25ab76d4a2dbfb716f5066e8983d0865182eacec6ea215bb958f8b
|
|
74
|
+
env:
|
|
75
|
+
POSTGRES_USER: oblako
|
|
76
|
+
POSTGRES_PASSWORD: oblako
|
|
77
|
+
POSTGRES_DB: oblako
|
|
78
|
+
ports: ["5439:5439"]
|
|
79
|
+
env:
|
|
80
|
+
PGSESAME_TEST_REDSHIFT_DSN: host=localhost port=5439 user=oblako password=oblako dbname=oblako sslmode=require
|
|
81
|
+
steps:
|
|
82
|
+
- uses: actions/checkout@v4
|
|
83
|
+
|
|
84
|
+
- name: Install uv
|
|
85
|
+
uses: astral-sh/setup-uv@v5
|
|
86
|
+
|
|
87
|
+
- name: Install the project
|
|
88
|
+
run: uv sync --locked
|
|
89
|
+
|
|
90
|
+
- name: Wait for redshift-local (the proxy starts after its first-run setup)
|
|
91
|
+
run: |
|
|
92
|
+
for i in $(seq 1 90); do
|
|
93
|
+
uv run python -c "import os, psycopg; psycopg.connect(os.environ['PGSESAME_TEST_REDSHIFT_DSN'], connect_timeout=3).execute('select 1 from svv_roles limit 1')" 2>/dev/null && exit 0
|
|
94
|
+
sleep 2
|
|
95
|
+
done
|
|
96
|
+
echo "redshift-local did not come up"; exit 1
|
|
97
|
+
|
|
98
|
+
- name: Redshift tests
|
|
99
|
+
run: uv run pytest -q tests/test_redshift.py
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
# Publish pgsesame to PyPI when a version tag is pushed (git tag v0.1.0 && git push
|
|
2
|
+
# origin v0.1.0). Trusted publishing: PyPI trusts this workflow through GitHub's
|
|
3
|
+
# OIDC token, so no API token is stored anywhere. The publish job runs in the
|
|
4
|
+
# `pypi` environment, which can require a reviewer's approval before it starts.
|
|
5
|
+
# Then the same version goes to ghcr.io/almostly/pgsesame as a container image.
|
|
6
|
+
name: release
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
tags: ["v*"]
|
|
11
|
+
|
|
12
|
+
permissions:
|
|
13
|
+
contents: read
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
build:
|
|
17
|
+
name: build and check
|
|
18
|
+
runs-on: ubuntu-latest
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
|
|
22
|
+
- name: Install uv
|
|
23
|
+
uses: astral-sh/setup-uv@v5
|
|
24
|
+
|
|
25
|
+
- name: The tag matches the package version
|
|
26
|
+
run: |
|
|
27
|
+
version=$(uv run --no-project python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
28
|
+
if [ "v${version}" != "${GITHUB_REF_NAME}" ]; then
|
|
29
|
+
echo "tag ${GITHUB_REF_NAME} does not match pyproject.toml version ${version}"
|
|
30
|
+
exit 1
|
|
31
|
+
fi
|
|
32
|
+
|
|
33
|
+
- name: Build the wheel and the source archive
|
|
34
|
+
run: uv build
|
|
35
|
+
|
|
36
|
+
- name: Check the distributions
|
|
37
|
+
run: uvx twine check --strict dist/*
|
|
38
|
+
|
|
39
|
+
- uses: actions/upload-artifact@v4
|
|
40
|
+
with:
|
|
41
|
+
name: dist
|
|
42
|
+
path: dist/
|
|
43
|
+
|
|
44
|
+
publish:
|
|
45
|
+
name: publish to PyPI
|
|
46
|
+
needs: build
|
|
47
|
+
runs-on: ubuntu-latest
|
|
48
|
+
environment:
|
|
49
|
+
name: pypi
|
|
50
|
+
url: https://pypi.org/project/pgsesame/
|
|
51
|
+
permissions:
|
|
52
|
+
id-token: write # trusted publishing
|
|
53
|
+
steps:
|
|
54
|
+
- uses: actions/download-artifact@v4
|
|
55
|
+
with:
|
|
56
|
+
name: dist
|
|
57
|
+
path: dist/
|
|
58
|
+
|
|
59
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
60
|
+
|
|
61
|
+
image:
|
|
62
|
+
name: image to ghcr.io
|
|
63
|
+
# after PyPI, so an image never carries a version PyPI doesn't have
|
|
64
|
+
needs: publish
|
|
65
|
+
runs-on: ubuntu-latest
|
|
66
|
+
permissions:
|
|
67
|
+
contents: read
|
|
68
|
+
packages: write # push to ghcr.io with the workflow's own token
|
|
69
|
+
steps:
|
|
70
|
+
- uses: actions/checkout@v4
|
|
71
|
+
|
|
72
|
+
- uses: docker/setup-qemu-action@v3 # arm64 on an amd64 runner
|
|
73
|
+
|
|
74
|
+
- uses: docker/setup-buildx-action@v3
|
|
75
|
+
|
|
76
|
+
- uses: docker/login-action@v3
|
|
77
|
+
with:
|
|
78
|
+
registry: ghcr.io
|
|
79
|
+
username: ${{ github.actor }}
|
|
80
|
+
password: ${{ secrets.GITHUB_TOKEN }}
|
|
81
|
+
|
|
82
|
+
- name: Tags (v0.1.0 -> 0.1.0, 0.1, latest) and labels
|
|
83
|
+
id: meta
|
|
84
|
+
uses: docker/metadata-action@v5
|
|
85
|
+
with:
|
|
86
|
+
images: ghcr.io/${{ github.repository }}
|
|
87
|
+
tags: |
|
|
88
|
+
type=semver,pattern={{version}}
|
|
89
|
+
type=semver,pattern={{major}}.{{minor}}
|
|
90
|
+
type=raw,value=latest
|
|
91
|
+
labels: |
|
|
92
|
+
org.opencontainers.image.title=pgsesame
|
|
93
|
+
org.opencontainers.image.description=Permissions as code for PostgreSQL and Amazon Redshift
|
|
94
|
+
|
|
95
|
+
- uses: docker/build-push-action@v6
|
|
96
|
+
with:
|
|
97
|
+
context: .
|
|
98
|
+
platforms: linux/amd64,linux/arm64
|
|
99
|
+
push: true
|
|
100
|
+
tags: ${{ steps.meta.outputs.tags }}
|
|
101
|
+
labels: ${{ steps.meta.outputs.labels }}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Hooks for prek (or pre-commit, which reads the same file).
|
|
2
|
+
# Run on demand: uvx prek run --all-files
|
|
3
|
+
# Install the git hooks: uvx prek install --hook-type pre-commit --hook-type commit-msg
|
|
4
|
+
repos:
|
|
5
|
+
- repo: https://github.com/commitizen-tools/commitizen
|
|
6
|
+
rev: v4.16.4
|
|
7
|
+
hooks:
|
|
8
|
+
- id: commitizen # validate the message format: Area(+|~|-): description
|
|
9
|
+
stages: [commit-msg]
|
|
10
|
+
|
|
11
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
12
|
+
rev: v0.15.14
|
|
13
|
+
hooks:
|
|
14
|
+
- id: ruff-check
|
|
15
|
+
args: [--fix]
|
|
16
|
+
- id: ruff-format
|
|
17
|
+
|
|
18
|
+
- repo: https://github.com/PyCQA/pydocstyle
|
|
19
|
+
rev: 6.3.0
|
|
20
|
+
hooks:
|
|
21
|
+
- id: pydocstyle
|
|
22
|
+
files: ^src/pgsesame/.*\.py$
|
|
23
|
+
|
|
24
|
+
- repo: local
|
|
25
|
+
hooks:
|
|
26
|
+
- id: ty
|
|
27
|
+
name: ty check
|
|
28
|
+
# through uv, so ty resolves imports against the project's environment
|
|
29
|
+
entry: uv run --frozen ty check src tests
|
|
30
|
+
language: system
|
|
31
|
+
types: [python]
|
|
32
|
+
pass_filenames: false
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
Commits follow `Area(<+|~|->): description`, where `+` = **Added**, `~` =
|
|
9
|
+
**Changed**, `-` = **Removed**. Running `cz bump` turns those commits into the
|
|
10
|
+
versioned entries below.
|
|
11
|
+
|
|
12
|
+
## Unreleased
|
|
13
|
+
|
|
14
|
+
## v0.1.0 (2026-10-06)
|
|
15
|
+
|
|
16
|
+
The first release.
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- **CLI**: the `sesame` command (also installed as `pgsesame`, so `uvx pgsesame` works): `sesame validate`, `sesame plan` (Terraform's exit codes: 0 nothing to do, 2 changes, 1 error), `sesame apply`, `sesame show` and `sesame schema`, in pgcli's green
|
|
21
|
+
- **Spec**: principal-centric YAML for roles, users, Redshift groups, memberships, privileges on databases, schemas, tables, views and sequences, ownership and default privileges; parsed into pydantic models with constrained types, every problem reported with its YAML path; a JSON Schema for editors
|
|
22
|
+
- **Plan**: only the spec's principals are managed; `schema.*` expands to the schema's objects; revokes and membership removals are planned but applied only with `--allow-revoke`
|
|
23
|
+
- **Plan**: change sets: `sesame plan -o` saves the plan, `sesame apply changes.json` runs exactly it, or refuses when the database changed in a way that changes the plan, when it was planned against another database, or when its spec was edited
|
|
24
|
+
- **Postgres**: read through the catalog's ACLs; plan and apply in one transaction
|
|
25
|
+
- **Redshift**: users, groups and RBAC roles, read through Redshift's SVV views (no ACL parsing); Redshift's DDL and `GROUP` / `ROLE` grantees
|
|
26
|
+
- **Redshift**: connect with a password, with temporary IAM credentials (`--iam`, a cluster or a Serverless workgroup), or through the Data API (`--data-api`, one transaction per apply)
|
|
27
|
+
- **Release**: on PyPI, and as a container image for CI without Python: `ghcr.io/almostly/pgsesame` (amd64, arm64; tags `0.1.0`, `0.1`, `latest`)
|
|
28
|
+
- **Security**: passwords never go in the spec, a plan, a change set or a log: SecretStr from the environment variable the spec names; the DSN is a SecretStr
|
|
29
|
+
|
|
30
|
+
Ownership and default privileges are validated but not yet planned.
|
pgsesame-0.1.0/DESIGN.md
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
# pgsesame design
|
|
2
|
+
|
|
3
|
+
pgsesame manages database permissions from a YAML file. You declare roles, users,
|
|
4
|
+
groups, memberships, privileges and ownership; `sesame plan` reads what the
|
|
5
|
+
database currently grants, compares it with the file, and prints the SQL that
|
|
6
|
+
would close the difference; `sesame apply` runs it. The workflow follows Terraform:
|
|
7
|
+
the file is reviewed in a pull request, the plan is reviewed before anything runs,
|
|
8
|
+
and a second plan after apply should be empty.
|
|
9
|
+
|
|
10
|
+
It targets PostgreSQL and Amazon Redshift. They share most of the GRANT
|
|
11
|
+
vocabulary but differ in principals (Redshift has groups next to users and roles),
|
|
12
|
+
in privileges (Redshift adds ALTER, TRUNCATE, DROP and ASSUMEROLE) and in how the
|
|
13
|
+
catalog exposes them, so each engine has its own reader and SQL renderer behind a
|
|
14
|
+
common model.
|
|
15
|
+
|
|
16
|
+
## Lessons this design starts from
|
|
17
|
+
|
|
18
|
+
redtape (Redshift) and pgbedrock (PostgreSQL) both did this job and are both
|
|
19
|
+
unmaintained. Running redtape 0.4.2 against a production Redshift cluster showed
|
|
20
|
+
what an access manager has to get right:
|
|
21
|
+
|
|
22
|
+
- **Read the real catalog without crashing.** redtape's ACL parser rejects
|
|
23
|
+
privilege letters it does not know (Redshift's `A` and `P`) and the extra columns
|
|
24
|
+
newer Redshift functions return. pgsesame reads Redshift through its `SVV_*`
|
|
25
|
+
privilege views, which spell privileges out, and treats anything it does not
|
|
26
|
+
model as "unmanaged", reported but never fatal.
|
|
27
|
+
- **Diff, do not re-emit.** redtape re-issued every declared grant on every run.
|
|
28
|
+
pgsesame compares desired and current state as sets keyed by name, so a run
|
|
29
|
+
issues only the change, and a converged database plans nothing.
|
|
30
|
+
- **Revoke, but only when asked.** A tool that cannot revoke cannot converge; a
|
|
31
|
+
tool that revokes by default is dangerous. redtape's unfiltered plan contained 46
|
|
32
|
+
DROP USER/GROUP statements. pgsesame plans revokes and drops but applies them only
|
|
33
|
+
with explicit flags, and never touches principals outside its scope.
|
|
34
|
+
- **Cover what people actually declare.** Default privileges, ownership and
|
|
35
|
+
Redshift roles are first-class, not afterthoughts.
|
|
36
|
+
|
|
37
|
+
## Workflow
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
spec.yaml ──load + validate──▶ desired state ─┐
|
|
41
|
+
├─ diff ─▶ plan (ordered SQL) ─▶ apply
|
|
42
|
+
database ──read catalog──────▶ current state ─┘
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
- `sesame validate spec.yaml`: parse and check the file; no database needed.
|
|
46
|
+
- `sesame plan spec.yaml`: connect, read, diff, print the plan. The exit code
|
|
47
|
+
follows `terraform plan -detailed-exitcode`: 0 nothing to do, 2 changes planned,
|
|
48
|
+
1 error, so CI can tell "drift" from "failure".
|
|
49
|
+
- `sesame apply spec.yaml`: plan, then run the plan.
|
|
50
|
+
|
|
51
|
+
### Change sets
|
|
52
|
+
|
|
53
|
+
For review before anything runs, as CloudFormation's change sets and
|
|
54
|
+
`terraform plan -out` do: `sesame plan spec.yaml -o changes.json` saves the plan,
|
|
55
|
+
`sesame show changes.json` prints it again without a database, and
|
|
56
|
+
`sesame apply changes.json` runs exactly those statements.
|
|
57
|
+
|
|
58
|
+
A change set embeds the validated spec (with its SHA-256, so an edited file is
|
|
59
|
+
refused), the target it was planned against (`user@host:db`; applying it anywhere
|
|
60
|
+
else is refused), when it was made, and the typed operations. Apply plans again
|
|
61
|
+
from the embedded spec against the database as it is now and runs the saved
|
|
62
|
+
statements only if the new plan is identical; otherwise it refuses and shows the
|
|
63
|
+
new plan. So a change that matters (a grant made by hand, a new table under
|
|
64
|
+
`schema.*`) stops a stale change set, and a change elsewhere does not. No secret
|
|
65
|
+
is written: a new role's password stays out of the file and is read again from
|
|
66
|
+
its environment variable at apply.
|
|
67
|
+
|
|
68
|
+
In a pull request: CI runs `sesame plan -o`, the plan is posted for review, and
|
|
69
|
+
the merge job runs `sesame apply` on that file.
|
|
70
|
+
|
|
71
|
+
## The CLI's look
|
|
72
|
+
|
|
73
|
+
The command is `sesame`, styled after pgcli: green is the accent colour (the
|
|
74
|
+
header, the `user@host:db` connection line, help headings), and the plan colours
|
|
75
|
+
its operations the way Terraform does: green `+` for creates and grants, red `-`
|
|
76
|
+
for revokes and drops, yellow `~` for changes such as ownership. Colour turns off
|
|
77
|
+
when the output is not a terminal or `NO_COLOR` is set, so CI logs stay plain. An
|
|
78
|
+
interactive `sesame shell` in pgcli's style (prompt_toolkit, completion of
|
|
79
|
+
principal and object names) is a possible later addition, not part of the first
|
|
80
|
+
milestones.
|
|
81
|
+
|
|
82
|
+
## The spec
|
|
83
|
+
|
|
84
|
+
Principal-centric, so a reviewer reads one block to see everything a role can do.
|
|
85
|
+
|
|
86
|
+
```yaml
|
|
87
|
+
version: 1
|
|
88
|
+
engine: redshift # or postgres
|
|
89
|
+
|
|
90
|
+
principals:
|
|
91
|
+
etl:
|
|
92
|
+
type: user
|
|
93
|
+
login: true
|
|
94
|
+
owns:
|
|
95
|
+
schemas: [analytics]
|
|
96
|
+
|
|
97
|
+
analyst:
|
|
98
|
+
type: role
|
|
99
|
+
member_of: [reader]
|
|
100
|
+
privileges:
|
|
101
|
+
schemas:
|
|
102
|
+
usage: [analytics, marts]
|
|
103
|
+
tables:
|
|
104
|
+
select: [analytics.*, marts.daily_sales]
|
|
105
|
+
|
|
106
|
+
reader:
|
|
107
|
+
type: role
|
|
108
|
+
|
|
109
|
+
alice:
|
|
110
|
+
type: user
|
|
111
|
+
login: true
|
|
112
|
+
groups: [analysts] # Redshift only
|
|
113
|
+
member_of: [analyst]
|
|
114
|
+
|
|
115
|
+
analysts:
|
|
116
|
+
type: group # Redshift only
|
|
117
|
+
|
|
118
|
+
default_privileges:
|
|
119
|
+
- owner: etl # objects etl creates ...
|
|
120
|
+
schema: analytics # ... in this schema ...
|
|
121
|
+
grantee: analyst # ... are readable by analyst
|
|
122
|
+
tables: [select]
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Rules:
|
|
126
|
+
|
|
127
|
+
- **Principal types**: `role` and `user` on both engines (a PostgreSQL user is a
|
|
128
|
+
role that can log in); `group` on Redshift only.
|
|
129
|
+
- **Privileges by object type**: `databases`, `schemas`, `tables`, `views`,
|
|
130
|
+
`sequences` (PostgreSQL), `functions`; each maps privilege names to object
|
|
131
|
+
patterns. `schema.*` means every existing object of that type in the schema,
|
|
132
|
+
expanded when the plan is made; future objects are covered by
|
|
133
|
+
`default_privileges`, as in the database itself.
|
|
134
|
+
- **Ownership** (`owns`) is declared on the owner. Changing it plans
|
|
135
|
+
`ALTER ... OWNER TO`.
|
|
136
|
+
- **No secrets.** Passwords never appear in the spec. A login user either gets its
|
|
137
|
+
password from an environment variable named in the spec
|
|
138
|
+
(`password_env: ALICE_PASSWORD`), authenticates through IAM (Redshift
|
|
139
|
+
`IAM:` users), or has `password: disabled`.
|
|
140
|
+
- Unknown keys, unknown principals in references and engine-specific keys on the
|
|
141
|
+
wrong engine are validation errors, reported with the YAML path.
|
|
142
|
+
|
|
143
|
+
## Scope: what pgsesame manages
|
|
144
|
+
|
|
145
|
+
The spec's principals are managed. Everything else in the database is **observed
|
|
146
|
+
but untouched**: reported in the plan as unmanaged, never altered. A spec can widen
|
|
147
|
+
its scope with `manage: {prefixes: [app_, svc_]}` to take over principals by name,
|
|
148
|
+
so a team can adopt pgsesame one area at a time. Built-in principals (`PUBLIC`
|
|
149
|
+
aside), superusers, the connecting user and Redshift's `rdsdb` are never managed.
|
|
150
|
+
|
|
151
|
+
## Reading the current state
|
|
152
|
+
|
|
153
|
+
**PostgreSQL**: `pg_roles` and `pg_auth_members` for principals and memberships;
|
|
154
|
+
`aclexplode()` over `pg_database`, `pg_namespace`, `pg_class` and `pg_proc` ACLs
|
|
155
|
+
for privileges; `pg_default_acl` for default privileges; object owners from the
|
|
156
|
+
same catalogs.
|
|
157
|
+
|
|
158
|
+
**Redshift**: `SVV_USER_INFO` and `pg_group` for users and groups,
|
|
159
|
+
`SVV_ROLES`, `SVV_USER_GRANTS` and `SVV_ROLE_GRANTS` for roles and role membership,
|
|
160
|
+
`SVV_RELATION_PRIVILEGES`, `SVV_SCHEMA_PRIVILEGES`, `SVV_DATABASE_PRIVILEGES`,
|
|
161
|
+
`SVV_FUNCTION_PRIVILEGES` and `SVV_DEFAULT_PRIVILEGES` for privileges. These views
|
|
162
|
+
name each privilege, so there is no ACL string to parse. Where a view is missing,
|
|
163
|
+
the reader falls back to ACL strings with a parser that keeps letters it does not
|
|
164
|
+
know as unmanaged privileges.
|
|
165
|
+
|
|
166
|
+
Both readers return the same model: principals, memberships, a set of
|
|
167
|
+
`(grantee, privilege, object)` triples, owners and default-privilege entries.
|
|
168
|
+
|
|
169
|
+
## Diff and plan
|
|
170
|
+
|
|
171
|
+
Desired and current state are compared as sets. Each difference becomes one
|
|
172
|
+
operation (create principal, grant, revoke, add member, change owner, ...), and
|
|
173
|
+
the plan orders them so every statement can succeed: create principals first,
|
|
174
|
+
then ownership, then grants and memberships, then revokes, then drops last.
|
|
175
|
+
Operations render to engine-specific SQL with identifiers quoted on both sides
|
|
176
|
+
(including `IAM:` user names), `PUBLIC` kept as a keyword.
|
|
177
|
+
|
|
178
|
+
Revokes and drops are always planned and shown, but marked; `apply` runs them only
|
|
179
|
+
with `--allow-revoke` and `--allow-drop`. Without the flags, apply runs the
|
|
180
|
+
additive part and reports what it skipped.
|
|
181
|
+
|
|
182
|
+
## Applying
|
|
183
|
+
|
|
184
|
+
PostgreSQL runs the plan in one transaction: all of it or none. Redshift runs it in
|
|
185
|
+
one transaction where its statements allow, and otherwise statement by statement,
|
|
186
|
+
stopping at the first error and reporting what ran. After apply, `sesame plan`
|
|
187
|
+
should be empty; the integration tests assert exactly that.
|
|
188
|
+
|
|
189
|
+
## Connecting
|
|
190
|
+
|
|
191
|
+
Connection settings are not part of the spec: the same spec is planned against
|
|
192
|
+
development, staging and production, so where to connect comes from the command
|
|
193
|
+
line or the environment. Redshift speaks PostgreSQL's wire protocol, so a direct
|
|
194
|
+
connection is the same psycopg connection for both engines; only how the
|
|
195
|
+
credentials are obtained differs.
|
|
196
|
+
|
|
197
|
+
- **Credentials**: `--dsn`, or the standard libpq variables (`PGHOST`, `PGPORT`,
|
|
198
|
+
`PGDATABASE`, `PGUSER`, `PGPASSWORD`) and `~/.pgpass`. Works for PostgreSQL and
|
|
199
|
+
for Redshift with a database user's password.
|
|
200
|
+
- **Redshift with IAM** (`pip install "pgsesame[redshift]"`): `--cluster ID` or
|
|
201
|
+
`--workgroup NAME` with `--iam`. pgsesame asks AWS for temporary database
|
|
202
|
+
credentials (`GetClusterCredentialsWithIAM` or `GetClusterCredentials` for a
|
|
203
|
+
provisioned cluster, `redshift-serverless GetCredentials` for a workgroup) with
|
|
204
|
+
the usual AWS credential chain, then connects directly with them. No password is
|
|
205
|
+
stored anywhere; the machine still needs a network path to the database.
|
|
206
|
+
- **Redshift Data API** (same extra): `--data-api` with `--cluster ID` or
|
|
207
|
+
`--workgroup NAME`, authenticated with `--secret-arn` or IAM. Statements go over
|
|
208
|
+
AWS's HTTPS API, so no network path to the database is needed, which suits CI
|
|
209
|
+
runners outside the VPC. The Data API runs one statement (or one batch) per call,
|
|
210
|
+
so apply sends the plan as a batch where Redshift allows it.
|
|
211
|
+
|
|
212
|
+
All three produce the same connection interface inside pgsesame (run a query,
|
|
213
|
+
run statements), so the reader, planner and applier do not know which one is in
|
|
214
|
+
use, and the integration tests can run the same scenarios over each.
|
|
215
|
+
|
|
216
|
+
## Implementation choices
|
|
217
|
+
|
|
218
|
+
- **The spec is pydantic models.** Pydantic checks the structure (types, unknown
|
|
219
|
+
keys, allowed values); a second pass checks what needs the whole spec
|
|
220
|
+
(engine-specific privileges, references between principals). Both report every
|
|
221
|
+
problem with its YAML path. `sesame schema` prints the spec's JSON Schema, so
|
|
222
|
+
editors can complete and check the YAML as it's written.
|
|
223
|
+
- **Statements are typed objects, not strings.** The planner produces frozen
|
|
224
|
+
pydantic models (`CreateRole`, `Grant`, `Revoke`, `AddMember`, `AlterOwner`,
|
|
225
|
+
...); each renders itself for PostgreSQL or Redshift in one place, and golden
|
|
226
|
+
tests pin the SQL for both engines.
|
|
227
|
+
- **SQL is composed with `psycopg.sql`** (`SQL`, `Identifier`, `Literal`), so
|
|
228
|
+
names from the spec are always quoted correctly (`IAM:alice`, mixed case,
|
|
229
|
+
reserved words) and never concatenated into SQL. The Data API path renders the
|
|
230
|
+
same objects to text with the same quoting rules.
|
|
231
|
+
- **Catalog queries are named constants**, one module per engine, each covered by
|
|
232
|
+
the integration tests against a real server.
|
|
233
|
+
- **No SQLAlchemy.** Its Core has no constructs for GRANT, REVOKE, roles, default
|
|
234
|
+
privileges or ownership, its reflection does not cover roles or ACLs, and it
|
|
235
|
+
needs a DBAPI driver, which the Data API does not have. It would add weight
|
|
236
|
+
without removing any SQL. An adapter that accepts a SQLAlchemy engine as a
|
|
237
|
+
connection can come later if users ask for it.
|
|
238
|
+
|
|
239
|
+
## Testing
|
|
240
|
+
|
|
241
|
+
- Unit tests: spec validation, diff and plan ordering, SQL rendering, the ACL
|
|
242
|
+
fallback parser.
|
|
243
|
+
- Integration tests: `plan`, `apply`, then an empty `plan`, against PostgreSQL in
|
|
244
|
+
Docker and against oblako's redshift-local. redshift-local does not yet provide the
|
|
245
|
+
`SVV_*` privilege views; adding them to oblako is a prerequisite for the Redshift
|
|
246
|
+
integration tests, and also a parity gain for oblako.
|
|
247
|
+
|
|
248
|
+
## Roadmap
|
|
249
|
+
|
|
250
|
+
Done in 0.1: the spec and `sesame validate`; plan and apply for roles, users,
|
|
251
|
+
groups, memberships and privileges on PostgreSQL (14 to 18) and Redshift (read
|
|
252
|
+
through its SVV views); change sets; Redshift over IAM credentials and the Data
|
|
253
|
+
API, each tested against oblako and Redshift Serverless.
|
|
254
|
+
|
|
255
|
+
0.2:
|
|
256
|
+
|
|
257
|
+
- Ownership (`owns`) and default privileges, planned and applied.
|
|
258
|
+
- Built-in roles a spec can refer to without managing them: Redshift Serverless's
|
|
259
|
+
`sys:*`, Supabase's `anon`, `authenticated`, `service_role`, AlloyDB's
|
|
260
|
+
`alloydbsuperuser`, RDS's `rds_superuser`, Cloud SQL's `cloudsqlsuperuser`.
|
|
261
|
+
- Managed PostgreSQL in CI: Supabase (`supabase start`) and AlloyDB Omni, run as
|
|
262
|
+
the platform's admin role, which is not a superuser there (on PostgreSQL 16+
|
|
263
|
+
such a role manages only the roles it created).
|
|
264
|
+
- A GitHub Action: plan on a pull request with the plan as a comment, apply the
|
|
265
|
+
reviewed change set on merge.
|
|
266
|
+
- `manage.prefixes`.
|
|
267
|
+
|
|
268
|
+
0.3:
|
|
269
|
+
|
|
270
|
+
- Row-level security: `ENABLE ROW LEVEL SECURITY` and `CREATE POLICY` in the
|
|
271
|
+
spec, planned and diffed like grants. On Supabase, policies are how data access
|
|
272
|
+
is controlled, more than grants.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# pgsesame for CI systems without Python: docker run ghcr.io/almostly/pgsesame plan ...
|
|
2
|
+
# Published by .github/workflows/release.yml with each version tag (amd64, arm64).
|
|
3
|
+
FROM python:3.12-slim
|
|
4
|
+
|
|
5
|
+
COPY --from=ghcr.io/astral-sh/uv:0.10 /uv /usr/local/bin/uv
|
|
6
|
+
|
|
7
|
+
WORKDIR /src
|
|
8
|
+
COPY pyproject.toml uv.lock README.md LICENSE ./
|
|
9
|
+
COPY src ./src
|
|
10
|
+
# the Redshift extra too (IAM credentials, the Data API): the image is for any target
|
|
11
|
+
RUN uv pip install --system --no-cache ".[redshift]" && rm -rf /src /usr/local/bin/uv
|
|
12
|
+
|
|
13
|
+
# no root: pgsesame only reads a spec and talks to a database
|
|
14
|
+
RUN useradd --create-home --uid 10001 sesame
|
|
15
|
+
USER sesame
|
|
16
|
+
WORKDIR /work
|
|
17
|
+
|
|
18
|
+
ENTRYPOINT ["sesame"]
|
|
19
|
+
CMD ["--help"]
|