pgdevkit 0.1.0__py3-none-any.whl

pgdevkit/testdb/naming.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import hashlib
+import re
+import subprocess
+from pathlib import Path
+
+_INVALID_CHARS = re.compile(r"[^a-z0-9_]+")
+_MAX_SLUG_LEN = 30
+
+
+def slugify(value: str) -> str:
+    """Lowercase, replace invalid chars with '_', truncate+hash if too long."""
+    slug = _INVALID_CHARS.sub("_", value.lower()).strip("_")
+    if not slug:
+        slug = "x"
+    if len(slug) <= _MAX_SLUG_LEN:
+        return slug
+    digest = hashlib.sha256(slug.encode()).hexdigest()[:8]
+    return f"{slug[:_MAX_SLUG_LEN]}_{digest}"
+
+
+def current_branch(cwd: Path | None = None) -> str:
+    """Return the branch checked out in the git worktree rooted at cwd."""
+    result = subprocess.run(
+        ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+        cwd=cwd,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout.strip()
+
+
+def workspace_db_name(project_name: str, branch: str) -> str:
+    """Compute a Postgres-safe, collision-resistant database name for this
+    project+branch. A second slugify pass over the joined string guarantees
+    the result stays under Postgres's 63-byte identifier limit even when
+    both inputs are already at the per-component truncation limit."""
+    joined = f"{slugify(project_name)}_{slugify(branch)}"
+    return slugify(joined)

pgdevkit/testdb/query.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import re
+from typing import LiteralString, cast
+
+import psycopg
+from psycopg.rows import dict_row
+
+_DOLLAR_TAG = re.compile(r"\$[A-Za-z_]*\$")
+
+
+def _split_statements(sql: str) -> list[str]:
+    """Split on ';', but treat single/double-quoted strings and dollar-quoted
+    bodies (e.g. a plpgsql function's $$ ... $$) as opaque so an embedded ';'
+    inside them doesn't split the statement in two."""
+    statements: list[str] = []
+    buf: list[str] = []
+    i, n = 0, len(sql)
+    while i < n:
+        ch = sql[i]
+        if ch == "$" and (m := _DOLLAR_TAG.match(sql, i)):
+            tag = m.group()
+            end = sql.find(tag, m.end())
+            end = n if end == -1 else end + len(tag)
+            buf.append(sql[i:end])
+            i = end
+            continue
+        if ch in ("'", '"'):
+            end = i + 1
+            while end < n:
+                if sql[end] == ch:
+                    end += 1
+                    if sql[end : end + 1] == ch:  # doubled quote = escaped literal quote
+                        end += 1
+                        continue
+                    break
+                end += 1
+            buf.append(sql[i:end])
+            i = end
+            continue
+        if ch == ";":
+            statements.append("".join(buf))
+            buf = []
+            i += 1
+            continue
+        buf.append(ch)
+        i += 1
+    statements.append("".join(buf))
+    return [s.strip() for s in statements if s.strip()]
+
+
+async def execute(dsn: str, sql: str) -> list[dict] | None:
+    """Run one or more statements against dsn. Returns the rows of the final
+    statement if it produced any, else None."""
+    statements = _split_statements(sql)
+    last_rows: list[dict] | None = None
+    async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as con:
+        for stmt in statements:
+            async with con.cursor(row_factory=dict_row) as cur:
+                # stmt is arbitrary, caller-provided SQL text (a .sql file or
+                # --sql argument) — not a compile-time literal, but this
+                # function's entire purpose is to run it as-is.
+                await cur.execute(cast(LiteralString, stmt))
+                last_rows = await cur.fetchall() if cur.description else None
+    return last_rows

pgdevkit/testdb/schema.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from pathlib import Path
+from typing import Any, cast
+
+import psycopg
+import sqlglot
+import sqlglot.expressions as exp
+from psycopg.rows import dict_row
+from psycopg.sql import SQL, Identifier, Placeholder
+
+logger = logging.getLogger(__name__)
+logging.getLogger("sqlglot").setLevel(logging.ERROR)
+
+# The `database/` folder convention (layer dirs, object-type subfolders and
+# their apply order, file-naming rules) is documented in
+# docs/database-layout.md — keep that table in sync with this dict.
+_TYPE_ORDER = {
+    "schema": 1,
+    "types": 2,
+    "tables": 3,
+    "scalar_functions": 4,
+    "functions": 5,
+    "views": 6,
+    "table_functions": 7,
+    "procedures": 8,
+    "permissions": 100,
+    "indexes": 101,
+}
+
+
+def _get_type_order(path: Path) -> int:
+    filename = re.sub(r"^\d+(\.\d+)?", "", path.name).removeprefix("_").removesuffix(".sql")
+    if filename in _TYPE_ORDER:
+        return _TYPE_ORDER[filename]
+    if path.parent.name in _TYPE_ORDER:
+        return _TYPE_ORDER[path.parent.name]
+    raise ValueError(f"Unknown SQL type for {path.name} in {path.parent.name}")
+
+
+def _strip_layer_prefix(schema_name: str) -> str:
+    """Strip a leading numeric layer prefix (e.g. "1_dim" -> "dim") so it
+    matches the unprefixed schema name used in SQL identifiers."""
+    if re.match(r"^\d+_", schema_name):
+        return schema_name.split("_", 1)[1]
+    return schema_name
+
+
+_SCHEMA_QUALIFIED_TYPES = {
+    "types",
+    "tables",
+    "scalar_functions",
+    "functions",
+    "views",
+    "table_functions",
+    "procedures",
+}
+
+
+def _get_sql_deps(sql: str) -> set[str]:
+    exprs = sqlglot.parse(sql, dialect="postgres")
+    deps: set[str] = set()
+    for e in exprs:
+        if e is None:
+            continue
+        for t in e.find_all(exp.Table):
+            if t.args.get("this") is not None and t.args.get("db") is not None:
+                deps.add(str(t))
+    return deps
+
+
+def _iter_sql_files(database_dir: Path):
+    """Yield (Path, sql_content) pairs in dependency-safe execution order."""
+    files: list[Path] = []
+    for root, _, dbfiles in os.walk(database_dir):
+        if "_migration_scripts" in root or "migrations" in root:
+            continue
+        for file in dbfiles:
+            if file in ("all.sql", "100_permissions.sql"):
+                continue
+            if file.endswith(".sql") and ".prod" not in file:
+                files.append(Path(root) / file)
+
+    delivered: set[str] = set()
+    delayed: list[tuple[str | None, Path, str]] = []
+    all_declared: set[str] = set()
+
+    for file in sorted(files, key=lambda p: (_get_type_order(p), p.name)):
+        content = file.read_text(encoding="utf-8")
+        deps = _get_sql_deps(content)
+        if file.parent.name in _SCHEMA_QUALIFIED_TYPES:
+            schema = _strip_layer_prefix(file.parent.parent.name)
+            full_name = f"{schema}.{file.stem}"
+            deps.discard(full_name)  # the file's own CREATE target is not a real dependency
+            all_declared.add(full_name)
+            if not deps or all(d in delivered for d in deps):
+                delivered.add(full_name)
+                yield file, content
+            else:
+                delayed.append((full_name, file, content))
+            continue
+        if not deps or all(d in delivered for d in deps):
+            yield file, content
+        else:
+            delayed.append((None, file, content))
+
+    while delayed:
+        progressed = False
+        for i in range(len(delayed) - 1, -1, -1):
+            declared_name, file, content = delayed[i]
+            deps = _get_sql_deps(content)
+            if declared_name:
+                deps.discard(declared_name)
+            if all(d in delivered or d not in all_declared for d in deps):
+                if declared_name:
+                    delivered.add(declared_name)
+                yield file, content
+                delayed.pop(i)
+                progressed = True
+        if not progressed:
+            raise ValueError(f"Circular or missing SQL dependencies: {[f[1] for f in delayed]}")
+
+
+async def _insert_test_data(
+    json_file: Path, table: str, force_reset: bool, con: psycopg.AsyncConnection
+) -> None:
+    if not json_file.exists():
+        return
+    rows: list[dict[str, Any]] = json.loads(json_file.read_text(encoding="utf-8"))
+    if not rows:
+        return
+
+    schema, table_name = table.split(".")
+    async with con.cursor(row_factory=dict_row) as cur:
+        if not force_reset:
+            await cur.execute(SQL("SELECT count(*) AS cnt FROM {t}").format(t=Identifier(schema, table_name)))
+            row = await cur.fetchone()
+            if row and row["cnt"] == len(rows):
+                return
+
+        col_names = list(rows[0].keys())
+        for row in rows:
+            for col in col_names:
+                if isinstance(row[col], (dict, list)):
+                    row[col] = json.dumps(row[col])
+
+        await cur.execute(SQL("DELETE FROM {t}").format(t=Identifier(schema, table_name)))
+        insert_sql = SQL("INSERT INTO {t} ({cols}) VALUES ({vals})").format(
+            t=Identifier(schema, table_name),
+            cols=SQL(", ").join(Identifier(c) for c in col_names),
+            vals=SQL(", ").join(Placeholder(c) for c in col_names),
+        )
+        await cur.executemany(insert_sql, rows)
+
+
+async def apply_schema(
+    con: psycopg.AsyncConnection,
+    database_dir: Path,
+    extensions: tuple[str, ...] = (),
+    force_reset: bool = False,
+) -> None:
+    """Apply every .sql file under database_dir (in dependency-safe order)
+    and seed any matching .test_data.json files. Safe to call repeatedly."""
+    await con.set_autocommit(True)
+    for extension in extensions:
+        await con.execute(SQL("CREATE EXTENSION IF NOT EXISTS {e}").format(e=Identifier(extension)))
+
+    async def _apply(file: Path, sql: str) -> None:
+        await con.execute(cast(Any, sql))
+        json_file = file.with_suffix(".test_data.json")
+        if json_file.exists():
+            schema_name = _strip_layer_prefix(file.parent.parent.name)
+            await _insert_test_data(json_file, f"{schema_name}.{file.stem}", force_reset, con)
+
+    failures: list[tuple[Path, str]] = []
+    for file, sql in _iter_sql_files(database_dir):
+        try:
+            await _apply(file, sql)
+        except Exception as e:  # noqa: BLE001
+            logger.warning("Error executing %s (will retry): %s", file, e)
+            failures.append((file, sql))
+
+    for file, sql in failures:
+        await _apply(file, sql)

pgdevkit-0.1.0.dist-info/METADATA

@@ -0,0 +1,140 @@
+Metadata-Version: 2.4
+Name: pgdevkit
+Version: 0.1.0
+Summary: A helper for developing with Postgres
+Requires-Python: >=3.14
+Requires-Dist: psycopg[binary]>=3.2.0
+Requires-Dist: sqlglot[c]>=30.11.0
+Provides-Extra: azure
+Requires-Dist: azure-identity>=1.19.0; extra == 'azure'
+Provides-Extra: cli
+Requires-Dist: rich>=13.0.0; extra == 'cli'
+Requires-Dist: typer>=0.26.7; extra == 'cli'
+Provides-Extra: db
+Requires-Dist: psycopg-pool>=3.3.0; extra == 'db'
+Requires-Dist: pydantic>=2.0; extra == 'db'
+Description-Content-Type: text/markdown
+
+# pgdevkit
+
+A helper for developing with Postgres.
+
+## `pgdb compare`
+
+Compare a directory of SQL scripts (see the `database-in-source` layout
+convention) against a live database and report differences:
+
+```bash
+pgdb compare --url postgresql://user:pass@host:port/db path/to/database/
+```
+
+### Entra ID auth (Azure Postgres / Databricks Lakebase)
+
+Pass `--entra-user <identity>` to `pgdb compare` to authenticate with an
+Entra ID token instead of a static password. Which token flow is used is
+auto-detected from the database hostname:
+
+- **Azure Database for PostgreSQL** (`*.postgres.database.azure.com`,
+  `*.postgres.cosmos.azure.com`) — the default: fetches a token via
+  `DefaultAzureCredential` and uses it directly as the password. Requires
+  the `azure` extra: `pip install pgdevkit[azure]`.
+- **Databricks Lakebase** (`*.database.azuredatabricks.net`,
+  `*.database.cloud.databricks.com`) — fetches a Databricks-scoped Entra
+  token, then exchanges it for a short-lived Postgres credential via the
+  Databricks workspace API. Also requires `--databricks-workspace-host`
+  and `--databricks-instance`:
+
+```bash
+pgdb compare --url postgresql://instance-abc.database.azuredatabricks.net:5432/databricks_postgres \
+  --entra-user alice@example.com \
+  --databricks-workspace-host https://adb-123456789.azuredatabricks.net \
+  --databricks-instance myinstance \
+  path/to/database/
+```
+
+(`--url`'s own user/password, if any, are discarded and replaced — `--entra-user`
+plus the fetched token become the connection's actual credentials.)
+
+## `pgdb testdb`
+
+Manages a single shared, Podman-backed Postgres container for local tests
+across all your projects — no more one-container-per-project-per-worktree.
+Isolation between projects and worktrees is per-database, inside one
+container.
+
+Add to `pyproject.toml`:
+
+```toml
+[tool.pgdevkit]
+name = "myproject"        # optional; defaults to the repo directory name
+database_dir = "database" # optional; defaults to "database"
+```
+
+Add to `conftest.py`:
+
+```python
+import os
+import pytest
+from pgdevkit.testdb import ensure_testdb
+
+@pytest.fixture(scope="session", autouse=True)
+def ensure_test_postgres():
+    for k, v in ensure_testdb().items():
+        os.environ[k] = v
+```
+
+CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`.
+
+Container connection defaults (`localhost:54322`, `postgres`/`testpwd`) can
+be overridden with `PGDEVKIT_TESTDB_HOST`, `PGDEVKIT_TESTDB_PORT`,
+`PGDEVKIT_TESTDB_USER`, `PGDEVKIT_TESTDB_PASSWORD`. Before touching
+podman/docker, pgdevkit first checks (with a short timeout) whether Postgres
+is already reachable at that address and skips container management if so.
+Set `PGDEVKIT_SKIP_CONTAINER=1` to always assume it's already there and skip
+that check too.
+
+## `pgdevkit.db` — helpers for application code
+
+Install with the `db` extra: `pip install pgdevkit[db]`.
+
+- **`PostgresTableModel`** — a `pydantic.BaseModel` base class for models
+  that map 1:1 to a table row. Implement `get_table_name()` (returns
+  `(schema, table)`) and `get_primary_key()` on each model.
+- **`PgPool`** — an async connection pool keyed off
+  `{env_prefix}HOST/PORT/DB/USER/PASSWORD` env vars. Call `await pool.open()`
+  once at startup, then use `async with pool.connection() as con:`.
+  Pass `entra_user` to authenticate via Entra ID instead of a static
+  password — same host-based auto-detection as `pgdb compare`'s
+  `--entra-user`. For Lakebase hosts, also set the
+  `{env_prefix}DATABRICKS_WORKSPACE_HOST` and `{env_prefix}DATABRICKS_INSTANCE`
+  env vars.
+- **CRUD functions** — `pg_retrieve`, `pg_retrieve_many`, `pg_insert`,
+  `pg_insert_many`, `pg_update`, `pg_update_dict`, `pg_upsert`,
+  `pg_upsert_dict`, `pg_upsert_many`, `pg_upsert_many_dict`, `pg_delete`,
+  `pg_delete_dict` — typed (`PostgresTableModel`-based) or dict-based CRUD
+  against a table, built on `psycopg` for safe identifier/value handling.
+- **`SqlLoader`** — loads and caches `.sql` files from
+  `{root}/<topic>/<name>.sql`, for keeping hand-written queries out of
+  Python source.
+
+```python
+from pgdevkit.db import PgPool, PostgresTableModel, pg_retrieve, pg_upsert
+
+class Widget(PostgresTableModel):
+    id: int
+    name: str
+
+    @staticmethod
+    def get_table_name() -> tuple[str, str]:
+        return ("public", "widget")
+
+    @staticmethod
+    def get_primary_key() -> list[str]:
+        return ["id"]
+
+pool = PgPool(env_prefix="POSTGRES_")
+await pool.open()
+async with pool.connection() as con:
+    widget = await pg_retrieve(con, Widget, {"id": 1})
+    await pg_upsert(con, Widget(id=1, name="thing"), Widget)
+```

pgdevkit-0.1.0.dist-info/RECORD

@@ -0,0 +1,31 @@
+pgdevkit/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pgdevkit/cli.py,sha256=71z8wBVhj51N_ZC34S4NxVCyE58esXdVyarDphI_d4o,8309
+pgdevkit/connection.py,sha256=UaHzF4QaYTf7v5-LWscHJcTV11byfVGGlSdb5S4CvfM,1901
+pgdevkit/diff.py,sha256=oZhkMWWtsI1UAo9eJ7LJ-PbBGab01Rh1iTIA7iaTLXw,9550
+pgdevkit/fetch_missing.py,sha256=1-2NBDCQNtirywO9Ucm775UKSACfAzcnQlp-4Jh2weE,8136
+pgdevkit/introspect.py,sha256=QYIlRuZk4lBHGxaW7zb_kFRIuNfi3AxH0Q2F_GKzlVI,7870
+pgdevkit/lakebase.py,sha256=0PABOk4p-2klLTY8QAUibQf9QeyNHKywUWEWGdkUY8k,3165
+pgdevkit/models.py,sha256=dAZ8f7pgDsxoLVrDWsM4h0qwJ0pOzXDdWMGsnjrW80E,2103
+pgdevkit/parser.py,sha256=B3NRtcSBuEUcQNp5lxEMi6ApVedkbNUNdklCea-Ts40,12021
+pgdevkit/db/__init__.py,sha256=H8muiCuNr_ogJo4tg3iLxxfR2jTckSrxxrVQyTd_3I4,701
+pgdevkit/db/connection.py,sha256=yIj1pBS_vX4CFNZvidGf1AIIKlXblfQKejNYEO5KfVA,2606
+pgdevkit/db/crud.py,sha256=uWVTgHI-JJezXufP-WDHs6Q8LPtSw8YEk7IErVmyDe4,7223
+pgdevkit/db/loader.py,sha256=0Swmp0h0s7I4acG0Dv3cem2YD4lFvq8aFEbFQPPQrxg,616
+pgdevkit/db/model.py,sha256=sAE4KMrDApxCiDvIRz2spa4YNJYq-fxyb5R1Cxt19ro,663
+pgdevkit/testdb/__init__.py,sha256=-APwuluorU3iRCUsiJPK7Bc1AS5_IXxCdKB4zuaQ868,178
+pgdevkit/testdb/api.py,sha256=MEaH9zH542jWSm9w1SseTk8vNIqgI5fwRKd8iVMH3vM,4930
+pgdevkit/testdb/config.py,sha256=v0KFSkWhhZ8vy9167nqHqPsuCjilVrIQ6iqo8m_9-Hk,1545
+pgdevkit/testdb/constants.py,sha256=-pa05wdqXkSszl9xlO2wMLGpkGzGz9vrWFxLgs8Ep3Y,465
+pgdevkit/testdb/container.py,sha256=s3G8v6z_BMQF6SPtixUcbBC0xjMak9TzKt_EJWsUEg8,3162
+pgdevkit/testdb/naming.py,sha256=Kyk5qEfxUkbggUG5bD7U7CJ_kAET332VPQNB-O5kvTY,1318
+pgdevkit/testdb/query.py,sha256=e4UbgdCKKIot3a8Qixg-OX-Af0rlQ4a8ODhJBD8mtVM,2318
+pgdevkit/testdb/schema.py,sha256=WsA3-czg5DhyEbNj-TBTpU9w4MdgwFHb02TowuvYzJo,6572
+pgdevkit/docs/database-layout.md,sha256=dNvDH956xSXuAnLqkOopyOfaIm_A-RtUd57VTaZ1GQU,6035
+pgdevkit/skills/pgdevkit/SKILL.md,sha256=P6CojgqlF3oOEfcETLwN22YvyZCPK4ofPn2DMx70fFE,11324
+pgdevkit/skills/pgdevkit/references/complex_helper.py,sha256=0qQ-35ae2gUGdUXM8vL5mHRkVph-NNOBaJbQQvU8Stk,10062
+pgdevkit/skills/pgdevkit/references/dynamic-sql.md,sha256=aTR1sx81moiTwaMaiOecaqk-9rT8y5Zxz7W1BoC3vK0,2774
+pgdevkit/skills/pgdevkit/references/temporal-tables.md,sha256=LwZnItvQWI7bB0oRHxIfUc0ynnq7N0V5Mnv-QHvFN3Q,1364
+pgdevkit-0.1.0.dist-info/METADATA,sha256=MhTnLQk9TdbMax7XXtS4njJjq1w4Yps95oIQ-HqcSNk,5128
+pgdevkit-0.1.0.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+pgdevkit-0.1.0.dist-info/entry_points.txt,sha256=FK2P55lf0WaGlOZJIINOGgCt8WB7ep0OnPsgnrp80bg,42
+pgdevkit-0.1.0.dist-info/RECORD,,

pgdevkit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.31.0
+Root-Is-Purelib: true
+Tag: py3-none-any

pgdevkit-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pgdb = pgdevkit.cli:app