sqlpush 0.3.0__tar.gz → 0.4.0__tar.gz

{sqlpush-0.3.0 → sqlpush-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlpush
-Version: 0.3.0
+Version: 0.4.0
 Summary: Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models
 Keywords: sqlalchemy,alembic,postgresql,timescaledb,prisma,schema,migrations,database,drift,cli
 Author: Juan Miguel Contreras

{sqlpush-0.3.0 → sqlpush-0.4.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sqlpush"
-version = "0.3.0"
+version = "0.4.0"
 description = "Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models"
 readme = "README.md"
 requires-python = ">=3.10"

{sqlpush-0.3.0 → sqlpush-0.4.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "sqlpush"
-version = "0.3.0"
+version = "0.4.0"
 description = "Schema lifecycle tool for production PostgreSQL/TimescaleDB on SQLAlchemy 2.0 (SQLModel included): diff, push and check schema drift straight from your models"
 readme = "README.md"
 authors = [

{sqlpush-0.3.0 → sqlpush-0.4.0}/src/sqlpush/api.py

@@ -2,6 +2,8 @@
 from __future__ import annotations
 
 import asyncio
+import re
+from pathlib import Path
 from typing import NoReturn
 
 from sqlalchemy import MetaData
@@ -14,11 +16,18 @@ from sqlalchemy.exc import (
 from sqlalchemy.ext.asyncio import AsyncEngine
 
 from sqlpush.apply.executor import apply_plan, with_advisory_lock
+from sqlpush.chain.format import (
+    RISK_RANK,
+    next_revision_id,
+    render_migration_file,
+)
+from sqlpush.chain.migrate import run_migrate, run_stamp
 from sqlpush.core.diff import DiffEngine
 from sqlpush.directives.timescale import hypertable_operations
 from sqlpush.types import (
     CheckResult,
     ConnectFailed,
+    MigrateReport,
     Plan,
     Report,
     SqlpushError,
@@ -116,6 +125,80 @@ def check(metadata, engine, *, schemas=None, exclude=()) -> CheckResult:
     )
 
 
+def revision(
+    metadata, ref_engine, *, out_dir="migrations/versions", message=None, schemas=None, exclude=()
+) -> Path:
+    """Generate the next annotated-SQL migration file from models-vs-ref drift.
+
+    The reference DB must sit at the chain head (caller-provided — sqlpush
+    stays docker-free). Empty drift refuses loudly: no empty files.
+    """
+    p = plan(metadata, ref_engine, schemas=schemas, exclude=exclude)
+    if not p.operations:
+        raise SqlpushError("no drift between models and reference DB — nothing to revise")
+    risk = max((op.risk for op in p.operations), key=lambda r: RISK_RANK[r])
+    ops = [(f"[{op.risk.name}] {op.type} {op.table or '?'}", op.sql) for op in p.operations]
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    rev_id = next_revision_id(out)
+    slug = re.sub(r"[^a-z0-9_]+", "_", (message or "migration").lower())[:40]
+    path = out / f"{rev_id}_{slug}.sql"
+    if path.exists():
+        # NNNN+slug collision (e.g. same message re-run without the file being
+        # consumed) must be loud, never a silent overwrite of chain history
+        raise SqlpushError(f"refusing to overwrite existing migration file {path}")
+    prev_n = int(rev_id) - 1
+    try:
+        path.write_text(
+            render_migration_file(
+                ops=ops,
+                revision_id=rev_id,
+                risk=risk,
+                message=message,
+                parent=f"{prev_n:04d}" if prev_n >= 1 else None,
+            )
+        )
+    except OSError as exc:
+        raise SqlpushError(f"cannot write migration file {path}: {exc}") from exc
+    return path
+
+
+def migrate(target, *, chain_dir="migrations/versions", allow_destructive=False) -> MigrateReport:
+    """Replay annotated-SQL chain files with gates + same-txn bookkeeping.
+
+    ``target`` is a DSN string, sync ``Engine`` or ``AsyncEngine`` (resolved
+    via ``_sync_engine_from``; engines created here are disposed). See
+    ``chain.migrate.run_migrate`` for the execution contract.
+    """
+    engine, dispose = _sync_engine_from(target)
+    try:
+        return run_migrate(engine, chain_dir=chain_dir, allow_destructive=allow_destructive)
+    except SQLAlchemyError as exc:
+        # MigrationFileError/SqlpushError (typed) pass through untouched
+        _raise_typed(exc)
+    finally:
+        if dispose:
+            engine.dispose()
+
+
+def stamp(target, *, chain_dir="migrations/versions") -> MigrateReport:
+    """Bootstrap: register chain files as applied WITHOUT executing SQL.
+
+    For adopting a DB whose schema already reflects the chain. ``target``
+    resolution and error typing match :func:`migrate`. See
+    ``chain.migrate.run_stamp`` for the report convention.
+    """
+    engine, dispose = _sync_engine_from(target)
+    try:
+        return run_stamp(engine, chain_dir=chain_dir)
+    except SQLAlchemyError as exc:
+        # MigrationFileError/SqlpushError (typed) pass through untouched
+        _raise_typed(exc)
+    finally:
+        if dispose:
+            engine.dispose()
+
+
 def _sync_engine_from(target):
     if isinstance(target, AsyncEngine):
         from sqlalchemy import create_engine

sqlpush-0.4.0/src/sqlpush/chain/__init__.py

@@ -0,0 +1,6 @@
+# re-export for downstream consumers (stamp verb): canonical home is
+# chain.format. NOT re-exported from sqlpush.types — types is imported BY
+# chain.format, so a types-level re-export would be circular.
+from sqlpush.chain.format import MigrationFileError
+
+__all__ = ["MigrationFileError"]

sqlpush-0.4.0/src/sqlpush/chain/format.py

@@ -0,0 +1,142 @@
+"""Annotated-SQL migration files: the chain format (spec 2026-09-02 §4).
+
+Files are plain editable SQL with a structured header line that carries
+the risk classification. Labels are legal SQL comments, so generated
+files are directly psql-runnable. Parsing is FAIL-LOUD: a missing or
+malformed header refuses the file — never assume-safe.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from sqlpush.types import RiskClass, SqlpushError
+
+RISK_RANK: dict[RiskClass, int] = {
+    RiskClass.SAFE: 0,
+    RiskClass.RISKY: 1,
+    RiskClass.DESTRUCTIVE: 2,
+}
+"""Explicit ordering — RiskClass is a plain str-Enum with no < (spec P5)."""
+
+# Header line: `-- sqlpush: revision=0007 risk=DESTRUCTIVE ops=3`. Keys other
+# than revision/risk (e.g. ops=, parent=, generated=) are tolerated — the
+# writer may add informative keys, the parser only requires the fail-loud two.
+_HEADER_LINE_RE = re.compile(r"^--\s*sqlpush:\s*(?P<body>.*)$")
+_OP_RE = re.compile(r"^--\s*op\s+\d+\s+\[(?P<label>[^\]]*)\]\s*(?P<desc>.*)$")
+_REV_PREFIX_RE = re.compile(r"^(\d+)_")
+
+
+class MigrationFileError(SqlpushError):
+    """A chain file is missing/malformed — fail-loud, never assume-safe."""
+
+
+@dataclass
+class MigrationFile:
+    name: str
+    revision_id: str
+    risk: RiskClass
+    ops: list[tuple[str, str]] = field(default_factory=list)
+    text: str = ""
+
+
+def checksum(text: str) -> str:
+    normalized = text.replace("\r\n", "\n").rstrip()
+    return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+
+
+def _parse_header(text: str, name: str) -> tuple[str, RiskClass]:
+    for line in text.splitlines():
+        m = _HEADER_LINE_RE.match(line.strip())
+        if m:
+            break
+    else:
+        raise MigrationFileError(f"{name}: missing -- sqlpush: header")
+    body = m.group("body").strip()
+    if not body:
+        raise MigrationFileError(f"{name}: malformed -- sqlpush: header (empty)")
+    pairs: dict[str, str] = {}
+    for token in body.split():
+        key, sep, value = token.partition("=")
+        if not sep or not key or not value:
+            raise MigrationFileError(f"{name}: malformed -- sqlpush: header token {token!r}")
+        pairs[key] = value
+    revision = pairs.get("revision", "")
+    if not revision.isdigit():
+        raise MigrationFileError(f"{name}: malformed -- sqlpush: header: revision=NNNN required")
+    risk_name = pairs.get("risk")
+    if risk_name is None:
+        raise MigrationFileError(
+            f"{name}: malformed -- sqlpush: header: risk= missing (expected SAFE|RISKY|DESTRUCTIVE)"
+        )
+    try:
+        risk = RiskClass[risk_name]
+    except KeyError:
+        raise MigrationFileError(
+            f"{name}: malformed -- sqlpush: header: unknown risk={risk_name!r} "
+            f"(expected SAFE|RISKY|DESTRUCTIVE)"
+        ) from None
+    return revision, risk
+
+
+def render_migration_file(
+    *,
+    ops: list[tuple[str, str]],
+    revision_id: str,
+    risk: RiskClass,
+    message: str | None = None,
+    parent: str | None = None,
+) -> str:
+    lines = [f"-- sqlpush: revision={revision_id} risk={risk.name}"]
+    # parent= is informative-only: the parser never requires it (hand-written
+    # files omit it freely), but the writer always records the slot.
+    lines.append(f"-- parent={parent or ''}")
+    if message:
+        lines.append(f"-- {message}")
+    lines.append("")
+    for i, (label, sql) in enumerate(ops, start=1):
+        lines.append(f"-- op {i} {label}")
+        lines.append(sql.rstrip(";") + ";")
+        lines.append("")
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def parse_migration_file(text: str, *, name: str) -> MigrationFile:
+    revision, risk = _parse_header(text, name)
+
+    ops: list[tuple[str, str]] = []
+    sql_buf: list[str] = []
+    current_label = ""
+    for line in text.splitlines():
+        stripped = line.strip()
+        if stripped.startswith("-- sqlpush:"):
+            continue
+        op_m = _OP_RE.match(stripped)
+        if op_m:
+            if sql_buf:
+                ops.append((current_label, "\n".join(sql_buf).strip()))
+                sql_buf = []
+            current_label = f"[{op_m.group('label')}] {op_m.group('desc')}".strip()
+            continue
+        if stripped.startswith("--"):
+            continue  # comentarios libres (message, parent, generated...) — ignorable
+        if stripped:
+            sql_buf.append(line)
+    if sql_buf:
+        ops.append((current_label, "\n".join(sql_buf).strip()))
+    if not ops:
+        raise MigrationFileError(f"{name}: file has no SQL")
+
+    return MigrationFile(name=name, revision_id=revision, risk=risk, ops=ops, text=text)
+
+
+def next_revision_id(chain_dir: Path | str) -> str:
+    max_n = 0
+    for f in sorted(Path(chain_dir).glob("*.sql")):
+        m = _REV_PREFIX_RE.match(f.name)
+        if m:
+            max_n = max(max_n, int(m.group(1)))
+    return f"{max_n + 1:04d}"

sqlpush-0.4.0/src/sqlpush/chain/migrate.py

@@ -0,0 +1,175 @@
+"""migrate/stamp: chain replay + bootstrap with gates and bookkeeping.
+
+migrate execution contract (spec 2026-09-02 §5): each file's WHOLE TEXT is
+replayed in ONE ``exec_driver_sql()`` call inside a per-file transaction —
+the parsed ``ops`` are for gating/display only, NEVER reconstructed for
+execution (psycopg3 accepts multi-statement strings; nothing is tokenized,
+so dollar-quoted bodies are safe). The checksum row is inserted INSIDE the
+same transaction as the file's SQL — bookkeeping in a separate txn would
+let a crash between apply and registry re-apply the file forever
+(crash-loop over existing objects).
+
+Fail-loud ordering: any blocked file (parse error, checksum mismatch,
+destructive gate, SQL failure) stops the chain — nothing later runs (R4).
+A hung migrate (stuck waiting on the advisory lock) is diagnosed via
+pg_locks / pg_stat_activity.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from collections.abc import Iterator
+from pathlib import Path
+
+from sqlalchemy import text
+from sqlalchemy.engine import Connection, Engine
+
+from sqlpush.apply.executor import advisory_key
+from sqlpush.chain.format import MigrationFileError, checksum, parse_migration_file
+from sqlpush.types import MigrateReport, RiskClass
+
+_VERSIONS_DDL = (
+    "CREATE TABLE IF NOT EXISTS public.sqlpush_versions ("
+    "name TEXT PRIMARY KEY, sha256 TEXT NOT NULL, "
+    "applied_at timestamptz NOT NULL DEFAULT now())"
+)
+
+
+def _chain_files(chain_dir: str | Path) -> list[Path]:
+    chain_path = Path(chain_dir)
+    if not chain_path.is_dir():
+        # a typo'd --dir silently no-op'ing is the footgun; an empty-but-
+        # EXISTING dir is a legitimate idle run (versions table still ensured)
+        raise MigrationFileError(f"chain dir not found: {chain_path}")
+    return sorted(chain_path.glob("*.sql"))
+
+
+@contextlib.contextmanager
+def _chain_session(engine: Engine) -> Iterator[Connection]:
+    """Session-scoped advisory lock + versions table, shared by every verb.
+
+    Same key derivation as push (fnv1a_32(b"sqlpush") ^ db oid): serializes
+    concurrent chain workers and excludes push on the same database. The
+    txn opened by the key query is committed right away — session advisory
+    locks survive COMMIT/ROLLBACK, and an idle-in-transaction session is
+    exposed to idle_in_transaction_session_timeout (executor.py note).
+    """
+    with engine.connect() as conn:
+        key = advisory_key(conn)
+        conn.execute(text("SELECT pg_advisory_lock(:k)"), {"k": key})
+        conn.commit()
+        try:
+            conn.execute(text(_VERSIONS_DDL))
+            conn.commit()
+            yield conn
+        finally:
+            # best-effort unlock (executor pattern): a secondary failure here
+            # must never mask the primary outcome; the session lock dies with
+            # the connection even if this fails
+            if not conn.closed:
+                with contextlib.suppress(Exception):
+                    conn.rollback()
+                    conn.execute(text("SELECT pg_advisory_unlock(:k)"), {"k": key})
+                    conn.commit()
+
+
+def run_migrate(engine: Engine, *, chain_dir: str | Path, allow_destructive: bool) -> MigrateReport:
+    applied: list[str] = []
+    skipped: list[str] = []
+    blocked: list[str] = []
+    notes: list[str] = []
+    partial = False
+    chain = _chain_files(chain_dir)
+    with _chain_session(engine) as conn:
+        recorded = {
+            row[0]: row[1]
+            for row in conn.execute(text("SELECT name, sha256 FROM public.sqlpush_versions"))
+        }
+        conn.commit()
+        for f in chain:
+            raw = f.read_text()
+            try:
+                mf = parse_migration_file(raw, name=f.name)
+            except MigrationFileError as exc:
+                blocked.append(f.name)
+                notes.append(f"{f.name}: {exc}")
+                break  # orden estricto: nada posterior corre
+            if f.name in recorded:
+                if recorded[f.name] != checksum(raw):
+                    blocked.append(f.name)
+                    notes.append(f"{f.name}: checksum mismatch (edited after apply?)")
+                    break
+                skipped.append(f.name)
+                continue
+            if mf.risk is RiskClass.DESTRUCTIVE and not allow_destructive:
+                blocked.append(f.name)
+                notes.append(f"{f.name}: DESTRUCTIVE requires --allow-destructive")
+                break
+            try:
+                with conn.begin():
+                    # whole-file replay: exec_driver_sql bypasses text()'s
+                    # bind-param parsing entirely — ":casts" and ":=" in
+                    # hand-edited SQL must reach the server verbatim
+                    conn.exec_driver_sql(raw)
+                    conn.execute(
+                        text("INSERT INTO public.sqlpush_versions (name, sha256) VALUES (:n, :s)"),
+                        {"n": f.name, "s": checksum(raw)},
+                    )
+                applied.append(f.name)
+            except Exception as exc:  # noqa: BLE001 — report, no mask
+                blocked.append(f.name)
+                notes.append(f"{f.name}: {exc}")
+                partial = True
+                break
+        return MigrateReport(
+            applied=tuple(applied),
+            skipped=tuple(skipped),
+            blocked=tuple(blocked),
+            partial_failure=partial,
+            notes=tuple(notes),
+        )
+
+
+def run_stamp(engine: Engine, *, chain_dir: str | Path) -> MigrateReport:
+    """Register every parseable chain file WITHOUT executing any SQL.
+
+    Bootstrap seam (spec §5 R7): adopt a DB whose schema already reflects
+    the chain. Only the header must parse (fail-loud on THAT) — invalid SQL
+    in a body still registers, because stamp never executes anything. Each
+    registration is an idempotent upsert (``ON CONFLICT (name) DO UPDATE``),
+    so re-stamping refreshes checksums instead of failing.
+
+    Report convention: registered files are listed in ``skipped`` (stamp
+    never populates ``applied`` and never sets ``partial_failure``); a
+    header that fails to parse goes to ``blocked`` + ``notes`` and stops
+    the walk (strict order, same as migrate).
+    """
+    skipped: list[str] = []
+    blocked: list[str] = []
+    notes: list[str] = []
+    chain = _chain_files(chain_dir)
+    with _chain_session(engine) as conn:
+        for f in chain:
+            raw = f.read_text()
+            try:
+                parse_migration_file(raw, name=f.name)
+            except MigrationFileError as exc:
+                blocked.append(f.name)
+                notes.append(f"{f.name}: {exc}")
+                break  # orden estricto: nada posterior se registra
+            with conn.begin():
+                conn.execute(
+                    text(
+                        "INSERT INTO public.sqlpush_versions (name, sha256) VALUES (:n, :s) "
+                        "ON CONFLICT (name) DO UPDATE SET sha256 = EXCLUDED.sha256"
+                    ),
+                    {"n": f.name, "s": checksum(raw)},
+                )
+            skipped.append(f.name)
+        return MigrateReport(
+            applied=(),
+            skipped=tuple(skipped),
+            blocked=tuple(blocked),
+            partial_failure=False,
+            notes=tuple(notes),
+        )

{sqlpush-0.3.0 → sqlpush-0.4.0}/src/sqlpush/cli.py

@@ -2,7 +2,9 @@
 """sqlpush: diff/push/check PostgreSQL schema drift from SQLAlchemy models.
 
 Exit codes: diff always 0; check 0 clean / 2 drift / 3 destructive drift;
-push 0 applied / 1 destructive blocked / 2 error (incl. partial failure).
+push 0 applied / 1 destructive blocked / 2 error (incl. partial failure);
+revision 0 written / 1 error (empty drift refuses); migrate 0 clean /
+1 blocked or partial failure.
 """
 
 from __future__ import annotations
@@ -11,6 +13,7 @@ import importlib
 import json
 import os
 import sys
+from pathlib import Path
 from typing import Annotated
 
 import typer
@@ -33,6 +36,9 @@ app = typer.Typer(add_completion=False, help=__doc__ or "")
 # default, no mutable default) while staying the idiomatic typer style.
 SchemaOpt = Annotated[list[str] | None, typer.Option("--schema")]
 ExcludeOpt = Annotated[list[str] | None, typer.Option("--exclude")]
+# Path-typed options need the Annotated form: B008 (call in default) only
+# exempts typer.Option for non-Path annotations
+DirOpt = Annotated[Path, typer.Option("--dir")]
 
 
 def _load_metadata(spec: str):
@@ -210,6 +216,81 @@ def push(
     raise typer.Exit(code=0)
 
 
+@app.command()
+def revision(
+    metadata_spec: str = typer.Argument(..., help="module:metadata"),
+    ref_dsn: str = typer.Option(..., "--ref-dsn"),
+    message: str | None = typer.Option(None, "--message", "-m"),
+    out_dir: DirOpt = Path("migrations/versions"),
+    schema: SchemaOpt = None,
+    exclude: ExcludeOpt = None,
+):
+    """Generate the next migration file from models vs the reference DB."""
+    md = _load_metadata(metadata_spec)
+    # required --ref-dsn (no DATABASE_URL fallback): the reference DB is a
+    # different database from the push target — conflating them silently
+    # would chain against the wrong head
+    engine = _engine(ref_dsn)
+    try:
+        path = api.revision(
+            md,
+            engine,
+            out_dir=out_dir,
+            message=message,
+            schemas=schema,
+            exclude=exclude or (),
+        )
+    finally:
+        engine.dispose()
+    typer.echo(str(path))
+    raise typer.Exit(code=0)
+
+
+@app.command()
+def migrate(
+    dsn: str | None = typer.Option(None),
+    allow_destructive: bool = typer.Option(False, "--allow-destructive"),
+    out_dir: DirOpt = Path("migrations/versions"),
+):
+    """Replay pending migration files (gates + checksum bookkeeping)."""
+    engine = _engine(dsn)
+    try:
+        report = api.migrate(engine, chain_dir=out_dir, allow_destructive=allow_destructive)
+    finally:
+        engine.dispose()
+    typer.echo(
+        f"applied: {len(report.applied)}, skipped: {len(report.skipped)}, "
+        f"blocked: {len(report.blocked)}, partial_failure: {report.partial_failure}"
+    )
+    for note in report.notes:
+        typer.secho(note, fg="yellow", err=True)
+    if report.blocked or report.partial_failure:
+        raise typer.Exit(code=1)
+    raise typer.Exit(code=0)
+
+
+@app.command()
+def stamp(
+    dsn: str | None = typer.Option(None),
+    out_dir: DirOpt = Path("migrations/versions"),
+):
+    """Adopt an existing DB: register chain files without executing SQL."""
+    engine = _engine(dsn)
+    try:
+        report = api.stamp(engine, chain_dir=out_dir)
+    finally:
+        engine.dispose()
+    typer.echo(
+        f"applied: {len(report.applied)}, skipped (registered): {len(report.skipped)}, "
+        f"blocked: {len(report.blocked)}, partial_failure: {report.partial_failure}"
+    )
+    for note in report.notes:
+        typer.secho(note, fg="yellow", err=True)
+    if report.blocked or report.partial_failure:
+        raise typer.Exit(code=1)
+    raise typer.Exit(code=0)
+
+
 def main() -> None:  # [project.scripts] entry point
     try:
         app()

{sqlpush-0.3.0 → sqlpush-0.4.0}/src/sqlpush/core/diff.py

@@ -17,7 +17,10 @@ from sqlpush.core.classify import classify
 from sqlpush.types import Plan, PlannedOperation
 
 _SYSTEM_SCHEMAS = ("_timescaledb%", "information_schema", "pg_%")
-_SYSTEM_TABLES = ("alembic_version", "spatial_ref_sys")
+# sqlpush_versions: the chain engine's own bookkeeping (migrate/stamp) —
+# same treatment as alembic_version (C1): a public-scoped post-migrate
+# check must be clean, not report its own registry as destructive drift
+_SYSTEM_TABLES = ("alembic_version", "spatial_ref_sys", "sqlpush_versions")
 
 # Leaf op class -> sqlpush op type. Class names follow alembic 1.19.1
 # autogen output as observed in docs/notes/alembic-notes.md and probes:

{sqlpush-0.3.0 → sqlpush-0.4.0}/src/sqlpush/types.py

@@ -83,6 +83,18 @@ class Report:
     duration: float = 0.0
 
 
+@dataclass(frozen=True)
+class MigrateReport:
+    # applied/skipped/blocked carry BARE filenames so callers (tests, CI)
+    # can do exact membership checks; the human-readable why (gate reason,
+    # checksum mismatch, SQL error) lives in `notes` aligned by prefix.
+    applied: tuple[str, ...] = ()
+    skipped: tuple[str, ...] = ()
+    blocked: tuple[str, ...] = ()
+    partial_failure: bool = False
+    notes: tuple[str, ...] = ()
+
+
 class SqlpushError(Exception):
     pass