pgdevkit 0.1.0__tar.gz

pgdevkit-0.1.0/.github/workflows/python-publish.yml

@@ -0,0 +1,39 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/pgdevkit
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.14"
+      - name: Install uv
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install dependencies
+        run: uv sync --all-extras --all-groups
+      - name: Build package
+        run: uv build
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

pgdevkit-0.1.0/.github/workflows/python-test.yml

@@ -0,0 +1,36 @@
+name: Python Test
+
+on:
+  push:
+    branches: ["main"]
+    paths-ignore: ["README.md", "docs", ".github"]
+  pull_request:
+    branches: ["main"]
+    paths-ignore: ["README.md", "docs", ".github"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.14"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: "recursive"
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Ensure podman is installed
+        run: command -v podman || (sudo apt-get update && sudo apt-get install -y podman)
+      - name: Install uv
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install project dependencies
+        run: uv sync --all-extras --all-groups
+      - name: ty check
+        run: uv run ty check pgdevkit
+      - name: Test with pytest
+        run: uv run -m pytest  --capture=tee-sys --maxfail=3 tests

pgdevkit-0.1.0/.gitignore

@@ -0,0 +1,22 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# Local planning docs (not shipped with the package)
+docs/superpowers/
+
+# Local migration checklist (not shipped with the package)
+/MIGRATION_NOTES.md
+
+# Local follow-up tracking (not shipped with the package)
+/FOLLOWUP.md
+
+# Git worktrees
+/.worktrees/

pgdevkit-0.1.0/.python-version

@@ -0,0 +1 @@
+3.14

pgdevkit-0.1.0/PKG-INFO

@@ -0,0 +1,140 @@
+Metadata-Version: 2.4
+Name: pgdevkit
+Version: 0.1.0
+Summary: A helper for developing with Postgres
+Requires-Python: >=3.14
+Requires-Dist: psycopg[binary]>=3.2.0
+Requires-Dist: sqlglot[c]>=30.11.0
+Provides-Extra: azure
+Requires-Dist: azure-identity>=1.19.0; extra == 'azure'
+Provides-Extra: cli
+Requires-Dist: rich>=13.0.0; extra == 'cli'
+Requires-Dist: typer>=0.26.7; extra == 'cli'
+Provides-Extra: db
+Requires-Dist: psycopg-pool>=3.3.0; extra == 'db'
+Requires-Dist: pydantic>=2.0; extra == 'db'
+Description-Content-Type: text/markdown
+
+# pgdevkit
+
+A helper for developing with Postgres.
+
+## `pgdb compare`
+
+Compare a directory of SQL scripts (see the `database-in-source` layout
+convention) against a live database and report differences:
+
+```bash
+pgdb compare --url postgresql://user:pass@host:port/db path/to/database/
+```
+
+### Entra ID auth (Azure Postgres / Databricks Lakebase)
+
+Pass `--entra-user <identity>` to `pgdb compare` to authenticate with an
+Entra ID token instead of a static password. Which token flow is used is
+auto-detected from the database hostname:
+
+- **Azure Database for PostgreSQL** (`*.postgres.database.azure.com`,
+  `*.postgres.cosmos.azure.com`) — the default: fetches a token via
+  `DefaultAzureCredential` and uses it directly as the password. Requires
+  the `azure` extra: `pip install pgdevkit[azure]`.
+- **Databricks Lakebase** (`*.database.azuredatabricks.net`,
+  `*.database.cloud.databricks.com`) — fetches a Databricks-scoped Entra
+  token, then exchanges it for a short-lived Postgres credential via the
+  Databricks workspace API. Also requires `--databricks-workspace-host`
+  and `--databricks-instance`:
+
+```bash
+pgdb compare --url postgresql://instance-abc.database.azuredatabricks.net:5432/databricks_postgres \
+  --entra-user alice@example.com \
+  --databricks-workspace-host https://adb-123456789.azuredatabricks.net \
+  --databricks-instance myinstance \
+  path/to/database/
+```
+
+(`--url`'s own user/password, if any, are discarded and replaced — `--entra-user`
+plus the fetched token become the connection's actual credentials.)
+
+## `pgdb testdb`
+
+Manages a single shared, Podman-backed Postgres container for local tests
+across all your projects — no more one-container-per-project-per-worktree.
+Isolation between projects and worktrees is per-database, inside one
+container.
+
+Add to `pyproject.toml`:
+
+```toml
+[tool.pgdevkit]
+name = "myproject"        # optional; defaults to the repo directory name
+database_dir = "database" # optional; defaults to "database"
+```
+
+Add to `conftest.py`:
+
+```python
+import os
+import pytest
+from pgdevkit.testdb import ensure_testdb
+
+@pytest.fixture(scope="session", autouse=True)
+def ensure_test_postgres():
+    for k, v in ensure_testdb().items():
+        os.environ[k] = v
+```
+
+CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`.
+
+Container connection defaults (`localhost:54322`, `postgres`/`testpwd`) can
+be overridden with `PGDEVKIT_TESTDB_HOST`, `PGDEVKIT_TESTDB_PORT`,
+`PGDEVKIT_TESTDB_USER`, `PGDEVKIT_TESTDB_PASSWORD`. Before touching
+podman/docker, pgdevkit first checks (with a short timeout) whether Postgres
+is already reachable at that address and skips container management if so.
+Set `PGDEVKIT_SKIP_CONTAINER=1` to always assume it's already there and skip
+that check too.
+
+## `pgdevkit.db` — helpers for application code
+
+Install with the `db` extra: `pip install pgdevkit[db]`.
+
+- **`PostgresTableModel`** — a `pydantic.BaseModel` base class for models
+  that map 1:1 to a table row. Implement `get_table_name()` (returns
+  `(schema, table)`) and `get_primary_key()` on each model.
+- **`PgPool`** — an async connection pool keyed off
+  `{env_prefix}HOST/PORT/DB/USER/PASSWORD` env vars. Call `await pool.open()`
+  once at startup, then use `async with pool.connection() as con:`.
+  Pass `entra_user` to authenticate via Entra ID instead of a static
+  password — same host-based auto-detection as `pgdb compare`'s
+  `--entra-user`. For Lakebase hosts, also set the
+  `{env_prefix}DATABRICKS_WORKSPACE_HOST` and `{env_prefix}DATABRICKS_INSTANCE`
+  env vars.
+- **CRUD functions** — `pg_retrieve`, `pg_retrieve_many`, `pg_insert`,
+  `pg_insert_many`, `pg_update`, `pg_update_dict`, `pg_upsert`,
+  `pg_upsert_dict`, `pg_upsert_many`, `pg_upsert_many_dict`, `pg_delete`,
+  `pg_delete_dict` — typed (`PostgresTableModel`-based) or dict-based CRUD
+  against a table, built on `psycopg` for safe identifier/value handling.
+- **`SqlLoader`** — loads and caches `.sql` files from
+  `{root}/<topic>/<name>.sql`, for keeping hand-written queries out of
+  Python source.
+
+```python
+from pgdevkit.db import PgPool, PostgresTableModel, pg_retrieve, pg_upsert
+
+class Widget(PostgresTableModel):
+    id: int
+    name: str
+
+    @staticmethod
+    def get_table_name() -> tuple[str, str]:
+        return ("public", "widget")
+
+    @staticmethod
+    def get_primary_key() -> list[str]:
+        return ["id"]
+
+pool = PgPool(env_prefix="POSTGRES_")
+await pool.open()
+async with pool.connection() as con:
+    widget = await pg_retrieve(con, Widget, {"id": 1})
+    await pg_upsert(con, Widget(id=1, name="thing"), Widget)
+```

pgdevkit-0.1.0/README.md

@@ -0,0 +1,123 @@
+# pgdevkit
+
+A helper for developing with Postgres.
+
+## `pgdb compare`
+
+Compare a directory of SQL scripts (see the `database-in-source` layout
+convention) against a live database and report differences:
+
+```bash
+pgdb compare --url postgresql://user:pass@host:port/db path/to/database/
+```
+
+### Entra ID auth (Azure Postgres / Databricks Lakebase)
+
+Pass `--entra-user <identity>` to `pgdb compare` to authenticate with an
+Entra ID token instead of a static password. Which token flow is used is
+auto-detected from the database hostname:
+
+- **Azure Database for PostgreSQL** (`*.postgres.database.azure.com`,
+  `*.postgres.cosmos.azure.com`) — the default: fetches a token via
+  `DefaultAzureCredential` and uses it directly as the password. Requires
+  the `azure` extra: `pip install pgdevkit[azure]`.
+- **Databricks Lakebase** (`*.database.azuredatabricks.net`,
+  `*.database.cloud.databricks.com`) — fetches a Databricks-scoped Entra
+  token, then exchanges it for a short-lived Postgres credential via the
+  Databricks workspace API. Also requires `--databricks-workspace-host`
+  and `--databricks-instance`:
+
+```bash
+pgdb compare --url postgresql://instance-abc.database.azuredatabricks.net:5432/databricks_postgres \
+  --entra-user alice@example.com \
+  --databricks-workspace-host https://adb-123456789.azuredatabricks.net \
+  --databricks-instance myinstance \
+  path/to/database/
+```
+
+(`--url`'s own user/password, if any, are discarded and replaced — `--entra-user`
+plus the fetched token become the connection's actual credentials.)
+
+## `pgdb testdb`
+
+Manages a single shared, Podman-backed Postgres container for local tests
+across all your projects — no more one-container-per-project-per-worktree.
+Isolation between projects and worktrees is per-database, inside one
+container.
+
+Add to `pyproject.toml`:
+
+```toml
+[tool.pgdevkit]
+name = "myproject"        # optional; defaults to the repo directory name
+database_dir = "database" # optional; defaults to "database"
+```
+
+Add to `conftest.py`:
+
+```python
+import os
+import pytest
+from pgdevkit.testdb import ensure_testdb
+
+@pytest.fixture(scope="session", autouse=True)
+def ensure_test_postgres():
+    for k, v in ensure_testdb().items():
+        os.environ[k] = v
+```
+
+CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`.
+
+Container connection defaults (`localhost:54322`, `postgres`/`testpwd`) can
+be overridden with `PGDEVKIT_TESTDB_HOST`, `PGDEVKIT_TESTDB_PORT`,
+`PGDEVKIT_TESTDB_USER`, `PGDEVKIT_TESTDB_PASSWORD`. Before touching
+podman/docker, pgdevkit first checks (with a short timeout) whether Postgres
+is already reachable at that address and skips container management if so.
+Set `PGDEVKIT_SKIP_CONTAINER=1` to always assume it's already there and skip
+that check too.
+
+## `pgdevkit.db` — helpers for application code
+
+Install with the `db` extra: `pip install pgdevkit[db]`.
+
+- **`PostgresTableModel`** — a `pydantic.BaseModel` base class for models
+  that map 1:1 to a table row. Implement `get_table_name()` (returns
+  `(schema, table)`) and `get_primary_key()` on each model.
+- **`PgPool`** — an async connection pool keyed off
+  `{env_prefix}HOST/PORT/DB/USER/PASSWORD` env vars. Call `await pool.open()`
+  once at startup, then use `async with pool.connection() as con:`.
+  Pass `entra_user` to authenticate via Entra ID instead of a static
+  password — same host-based auto-detection as `pgdb compare`'s
+  `--entra-user`. For Lakebase hosts, also set the
+  `{env_prefix}DATABRICKS_WORKSPACE_HOST` and `{env_prefix}DATABRICKS_INSTANCE`
+  env vars.
+- **CRUD functions** — `pg_retrieve`, `pg_retrieve_many`, `pg_insert`,
+  `pg_insert_many`, `pg_update`, `pg_update_dict`, `pg_upsert`,
+  `pg_upsert_dict`, `pg_upsert_many`, `pg_upsert_many_dict`, `pg_delete`,
+  `pg_delete_dict` — typed (`PostgresTableModel`-based) or dict-based CRUD
+  against a table, built on `psycopg` for safe identifier/value handling.
+- **`SqlLoader`** — loads and caches `.sql` files from
+  `{root}/<topic>/<name>.sql`, for keeping hand-written queries out of
+  Python source.
+
+```python
+from pgdevkit.db import PgPool, PostgresTableModel, pg_retrieve, pg_upsert
+
+class Widget(PostgresTableModel):
+    id: int
+    name: str
+
+    @staticmethod
+    def get_table_name() -> tuple[str, str]:
+        return ("public", "widget")
+
+    @staticmethod
+    def get_primary_key() -> list[str]:
+        return ["id"]
+
+pool = PgPool(env_prefix="POSTGRES_")
+await pool.open()
+async with pool.connection() as con:
+    widget = await pg_retrieve(con, Widget, {"id": 1})
+    await pg_upsert(con, Widget(id=1, name="thing"), Widget)
+```

pgdevkit-0.1.0/docs/database-layout.md

@@ -0,0 +1,145 @@
+# `database/` folder layout
+
+The Postgres schema is versioned as plain `.sql` files under a `database/`
+folder in the repo — that folder *is* the source of truth for the schema.
+`pgdb testdb` (see [`pgdevkit/testdb/schema.py`](../pgdevkit/testdb/schema.py))
+applies it to a local test database; a human applies the same files to
+production.
+
+This is the single source for the convention — don't duplicate this table or
+these rules elsewhere; link here instead.
+
+---
+
+## Layer directories
+
+Top-level directories group tables/objects by conceptual layer — one
+directory per Postgres schema, or per logical grouping within one schema.
+Names and count are entirely per-project; a generic example:
+
+```
+database/
+├── schema.sql            # CREATE SCHEMA statements
+├── 0_public/              # shared types/functions usable from anywhere
+├── 1_reference_data/      # dimension / reference tables
+├── 2_transactional/       # fact / transactional tables
+├── 3_app/                 # user-editable, app-owned tables
+├── 4_reporting/           # aggregated / statistics tables
+├── permissions.sql        # grants
+```
+
+The leading number is a **display/sort aid only** — it groups related
+folders together in a file listing so a human can scan them top-to-bottom in
+a sensible order. It does not control apply order: `pgdb testdb` applies
+files by object-type priority (below) and resolves cross-file dependencies
+itself, regardless of which numbered folder a file sits in. Feel free to
+renumber layers for readability without worrying about breaking anything.
+
+---
+
+## Object-type subfolders, in apply order
+
+Within a layer directory, group files by object type. This is what actually
+controls apply order, across all layer directories — cross-file dependencies
+between objects of the same type (e.g. one view selecting from another) are
+resolved automatically; this table only fixes the order *between* types.
+This table must match `_TYPE_ORDER` / `_SCHEMA_QUALIFIED_TYPES` in
+[`pgdevkit/testdb/schema.py`](../pgdevkit/testdb/schema.py) exactly — update
+both together.
+
+| Priority | Directory | Object type |
+|---|---|---|
+| 1 | `schema` | `CREATE SCHEMA` |
+| 2 | `types` | Custom types / enums |
+| 3 | `tables` | Tables |
+| 4 | `scalar_functions` | Scalar functions |
+| 5 | `functions` | Functions |
+| 6 | `views` | Views |
+| 7 | `table_functions` | Table functions |
+| 8 | `procedures` | Procedures |
+| 100 | `permissions` | Grants |
+| 101 | `indexes` | Indexes |
+
+One object per file: `tables/user.sql`, `views/all_edits.sql`,
+`types/measurement_unit.sql`.
+
+---
+
+## File-naming conventions
+
+| Suffix | Meaning |
+|---|---|
+| `<name>.sql` | The object's live definition (`CREATE TABLE`, `CREATE OR REPLACE VIEW`, ...) |
+| `<name>.test_data.json` | Seed rows for a table — a JSON array of row objects, loaded after the table is created |
+| `<name>.init.sql` | One-time setup for an object (e.g. a backfill), run once, kept separate from the reusable definition |
+| `<name>.prod.sql` / `.prod` anywhere in the name | Production-only (real permission grants, real user accounts) — skipped by `pgdb testdb` |
+| `all.sql` | Generated concatenation of the whole tree — not hand-edited, not committed |
+
+---
+
+## Migrations
+
+One-off, non-idempotent changes (rename/drop column, backfill, data fix) go
+in a `migrations/` (or `_migration_scripts/`) folder — one file per change,
+named by date:
+
+```
+database/migrations/2026-07-10_customer_geocode.sql
+```
+
+Rules:
+
+- Skipped by `pgdb testdb` — it only applies the layer directories above.
+- Update the corresponding table/view `.sql` file in the same change so its
+  definition already reflects the new shape — the migration and the
+  source-of-truth file must never drift apart.
+- Applied to production manually, once, by a human, after being verified
+  locally.
+- Never edited after being applied — a further change gets a new dated file.
+
+---
+
+## `COMMENT ON` — document schema in the object's own file
+
+Add a `COMMENT ON` for every table, and for any column whose purpose isn't
+obvious from its name and type (flags, status codes, denormalized fields,
+units). Put it directly in the table's `.sql` file, right after the
+`CREATE TABLE` — not in a migration, wiki, or README. It lives with the
+definition it describes and survives `\d+` / `pg_catalog` inspection.
+
+```sql
+-- database/1_reference_data/tables/user.sql
+create table dim.user (
+    id bigint generated always as identity primary key,
+    email text not null,
+    is_active boolean not null default true
+);
+
+comment on table dim.user is 'End-user accounts; one row per registered person.';
+comment on column dim.user.is_active is 'False once a user is soft-deleted; keep for audit trail.';
+```
+
+---
+
+## Backfilling untracked objects
+
+If a table, scalar function, or table function was created directly on the
+database and never got a `.sql` file, use `pgdb fetch-missing` to find it and
+generate one — see `pgdb fetch-missing --help`. It connects to Postgres,
+diffs the live schema against what's already tracked under `database/`, and
+for each object you select, reverse-engineers its DDL into the matching
+layer folder's `tables/`, `views/`, `scalar_functions/`, or
+`table_functions/` subfolder. The layer folder is matched by schema name
+against the existing top-level directories under `database/`, ignoring their
+leading sort number.
+
+---
+
+## Quick checklist
+
+- [ ] New table/view/function/type gets its own `.sql` file under the right layer + object-type folder
+- [ ] Object-type folder (`tables`, `views`, ...) matches the apply-order table above — that's what governs ordering, not the layer's leading number
+- [ ] One-off changes go in `migrations/`, dated, never edited after applying
+- [ ] The live `.sql` file is updated in the same change as any migration touching that object
+- [ ] `.prod` files are production-only and skipped by `pgdb testdb`
+- [ ] Every table (and non-obvious column) has a `COMMENT ON`, placed in the object's own `.sql` file