dbly 0.1.0__tar.gz → 0.2.0__tar.gz

{dbly-0.1.0 → dbly-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dbly
-Version: 0.1.0
+Version: 0.2.0
 Summary: State-based, cross-engine database deployment — git-driven, parser-assisted, SQL-first.
 Project-URL: Homepage, https://angrydata.info/dbly
 Project-URL: Repository, https://github.com/angrydat/dbly
@@ -81,10 +81,17 @@ db/
     v_open_orders.vw
     grants.sql
   init/                 # optional: privileged greenfield groundwork (CREATE DATABASE/ROLE…)
+  migrations/           # optional: ordered, run-once ALTERs for renames / data backfills
 hooks/pre/  hooks/post/  # optional: .sql or .py hooks (e.g. ArcGIS/ArcPy steps)
 .dbignore               # files in the repo that should not be deployed
 ```
 
+Most changes are declarative — edit the object file, dbly figures out the additive diff. For
+changes the diff can't do safely (renaming a column, moving data), drop an ordered script in
+`migrations/` (`0001_…sql`); it runs exactly once, is recorded in the ledger, and the table
+it touches defers to it for that deploy. On a fresh database, migrations are *baselined*
+(recorded, not run) since the object files already describe the end state.
+
 ## Connect
 
 A connection profile (reuses the familiar `connection.properties` format):

{dbly-0.1.0 → dbly-0.2.0}/README.md

@@ -43,10 +43,17 @@ db/
     v_open_orders.vw
     grants.sql
   init/                 # optional: privileged greenfield groundwork (CREATE DATABASE/ROLE…)
+  migrations/           # optional: ordered, run-once ALTERs for renames / data backfills
 hooks/pre/  hooks/post/  # optional: .sql or .py hooks (e.g. ArcGIS/ArcPy steps)
 .dbignore               # files in the repo that should not be deployed
 ```
 
+Most changes are declarative — edit the object file, dbly figures out the additive diff. For
+changes the diff can't do safely (renaming a column, moving data), drop an ordered script in
+`migrations/` (`0001_…sql`); it runs exactly once, is recorded in the ledger, and the table
+it touches defers to it for that deploy. On a fresh database, migrations are *baselined*
+(recorded, not run) since the object files already describe the end state.
+
 ## Connect
 
 A connection profile (reuses the familiar `connection.properties` format):

{dbly-0.1.0 → dbly-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dbly"
-version = "0.1.0"
+version = "0.2.0"
 description = "State-based, cross-engine database deployment — git-driven, parser-assisted, SQL-first."
 readme = "README.md"
 license = { text = "MIT" }

{dbly-0.1.0 → dbly-0.2.0}/src/dbly/__init__.py

@@ -4,4 +4,4 @@ git (what changed) + sqlglot (what it is) + live introspection (what's really th
 See CONCEPT.md for the design rationale.
 """
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"

{dbly-0.1.0 → dbly-0.2.0}/src/dbly/adapters/base.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 
 import abc
 
-from sqlalchemy import Engine
+from sqlalchemy import Engine, text
 
 from dbly.config import ConnectionConfig
 from dbly.engine import make_engine
@@ -95,6 +95,20 @@ class Adapter(abc.ABC):
     @abc.abstractmethod
     def record_deploy(self, ref: str, migration_ids: list[str]) -> None: ...
 
+    # --- explicit migrations (run-once, ledger-tracked) --------------------------------
+    def applied_migrations(self) -> set[str]:
+        """Migration ids already recorded in the ledger (standard SQL, all engines)."""
+        self.ensure_state_table()
+        with self.engine.connect() as conn:
+            rows = conn.execute(
+                text("SELECT DISTINCT migration_id FROM dbly_state WHERE migration_id IS NOT NULL")
+            )
+            return {r[0] for r in rows}
+
+    def record_migration(self, ref: str, migration_id: str) -> None:
+        """Mark a migration applied (reuses the per-engine record_deploy insert)."""
+        self.record_deploy(ref, [migration_id])
+
     # --- pure SQL builders (no connection — used by `plan --sql` export) ----------------
     @abc.abstractmethod
     def state_table_ddl(self) -> str:

{dbly-0.1.0 → dbly-0.2.0}/src/dbly/cli.py

@@ -137,7 +137,7 @@ def apply(
         s.sql for s in plan_obj.steps
         if allow_destructive or s.severity is not Severity.DESTRUCTIVE
     ]
-    if not statements:
+    if not statements and not plan_obj.migrations and not plan_obj.baselined:
         console.print("[green]nothing to apply.[/green]")
         return
 
@@ -147,13 +147,24 @@ def apply(
     try:
         _run_hooks(repo, "pre", py_interpreter)
         adapter.ensure_state_table()
-        adapter.apply(statements)
+        # baseline (bootstrap): record migrations as applied without running them
+        for mid in plan_obj.baselined:
+            adapter.record_migration(plan_obj.to_ref, mid)
+        # explicit migrations run first — they reshape the schema before reconciliation
+        for m in plan_obj.migrations:
+            adapter.run_init_script(m.sql)  # engine-aware multi-statement / PL-SQL runner
+            adapter.record_migration(plan_obj.to_ref, m.id)
+            console.print(f"[magenta]migration[/magenta] applied {m.id}")
+        if statements:
+            adapter.apply(statements)
         adapter.record_deploy(plan_obj.to_ref, migration_ids=[])
         _run_hooks(repo, "post", py_interpreter)
     finally:
         adapter.dispose()
-    console.print(f"[green]applied[/green] {len(statements)} step(s); "
-                  f"deployed ref → {plan_obj.to_ref}")
+    console.print(
+        f"[green]applied[/green] {len(statements)} step(s), "
+        f"{len(plan_obj.migrations)} migration(s); deployed ref → {plan_obj.to_ref}"
+    )
 
 
 @app.command()

{dbly-0.1.0 → dbly-0.2.0}/src/dbly/model.py

@@ -128,6 +128,19 @@ class Step:
     note: str | None = None
 
 
+@dataclass(slots=True)
+class Migration:
+    """An explicit, run-once, hand-written migration script (CONCEPT.md §5).
+
+    For changes the additive diff cannot do safely — renames, type changes with data
+    transformation, backfills. Tracked by ``id`` in the ledger so it runs exactly once.
+    """
+
+    id: str
+    sql: str
+    source_file: Path
+
+
 @dataclass(slots=True)
 class Plan:
     """An ordered, reviewable set of steps — the artifact of ``dbly plan``."""
@@ -135,6 +148,8 @@ class Plan:
     target: str
     from_ref: str | None
     to_ref: str
+    migrations: list[Migration] = field(default_factory=list)   # pending — run before steps
+    baselined: list[str] = field(default_factory=list)          # recorded, not run (bootstrap)
     steps: list[Step] = field(default_factory=list)
     warnings: list[str] = field(default_factory=list)
 

{dbly-0.1.0 → dbly-0.2.0}/src/dbly/parsing.py

@@ -121,6 +121,24 @@ def parse_file(
     return objects
 
 
+def referenced_tables(sql: str, *, dialect: str | None = None) -> set[str]:
+    """Lower-cased table names referenced anywhere in a SQL script (best effort).
+
+    Used to detect which tables a pending migration touches, so the additive diff defers to
+    the migration for those tables. Returns an empty set if sqlglot cannot parse the script.
+    """
+    names: set[str] = set()
+    try:
+        for stmt in sqlglot.parse(sql, read=dialect):
+            if stmt is None:
+                continue
+            for tbl in stmt.find_all(exp.Table):
+                names.add(tbl.name.lower())
+    except Exception:  # noqa: BLE001 — procedural/odd SQL; suppression is best-effort
+        pass
+    return names
+
+
 def canonical_hash(sql: str | None, *, dialect: str | None = None) -> str | None:
     """A formatting-insensitive hash of a definition, for advisory drift detection.
 

{dbly-0.1.0 → dbly-0.2.0}/src/dbly/planner.py

@@ -12,6 +12,7 @@ from dbly import parsing
 from dbly.adapters.base import Adapter
 from dbly.model import (
     ChangeType,
+    Migration,
     ObjectKind,
     ParsedObject,
     Plan,
@@ -31,6 +32,19 @@ def build_plan(
     dialect: str | None,
 ) -> Plan:
     plan = Plan(target=target, from_ref=from_ref, to_ref=to_ref)
+
+    # Explicit migrations (run-once). On bootstrap (no baseline) the canonical objects
+    # already produce the end state, so pending migrations are *baselined* (recorded, not
+    # run); on upgrade they run — before the object steps — to reshape the schema.
+    applied = adapter.applied_migrations()
+    pending = [(mid, p) for mid, p in repo.migration_files(to_ref) if mid not in applied]
+    if from_ref is None:
+        plan.baselined = [mid for mid, _ in pending]
+    else:
+        plan.migrations = [
+            Migration(mid, repo.read_at(to_ref, p), p) for mid, p in pending
+        ]
+
     changes = repo.changed_files(from_ref, to_ref)
 
     # Bucket by kind so the plan is emitted in dependency-safe order regardless of file
@@ -55,9 +69,20 @@ def build_plan(
             else:
                 replaceable.append(obj)
 
+    # Tables touched by a pending migration are migration-managed for this deploy — the
+    # migration reshapes them at apply time, so the (plan-time) additive diff must defer.
+    migration_tables: set[str] = set()
+    for m in plan.migrations:
+        migration_tables |= parsing.referenced_tables(m.sql, dialect=dialect)
+
     for obj in sequences:
         _plan_create_if_missing(adapter, plan, obj)
     for obj in tables:
+        if obj.id.name.lower() in migration_tables:
+            plan.warnings.append(
+                f"{obj.id}: managed by a pending migration — additive diff skipped this deploy"
+            )
+            continue
         _plan_table(adapter, plan, obj, dialect)
     for obj in indexes:
         _plan_create_if_missing(adapter, plan, obj)

{dbly-0.1.0 → dbly-0.2.0}/src/dbly/repo.py

@@ -16,6 +16,7 @@ import pathspec
 from dbly.model import ChangeType
 
 _SQL_SUFFIXES = {".sql", ".tbl", ".vw", ".prc", ".fnc", ".pkg", ".trg", ".typ", ".ddl"}
+MIGRATIONS_DIR = "migrations"  # run-once scripts — not declarative objects
 
 
 @dataclass(slots=True)
@@ -52,6 +53,14 @@ class Repo:
     def _is_sql(rel: Path) -> bool:
         return rel.suffix.lower() in _SQL_SUFFIXES
 
+    @staticmethod
+    def _is_migration(rel: Path) -> bool:
+        return len(rel.parts) > 0 and rel.parts[0] == MIGRATIONS_DIR
+
+    def _is_object(self, rel: Path) -> bool:
+        """A deployable declarative object file (SQL, not a migration, not ignored)."""
+        return self._is_sql(rel) and not self._is_migration(rel) and not self.is_ignored(rel)
+
     def changed_files(self, from_ref: str | None, to_ref: str) -> list[FileChange]:
         """Files changed between two refs (or the full tree at ``to_ref`` for bootstrap)."""
         if from_ref is None:
@@ -60,14 +69,21 @@ class Repo:
         return self._parse_name_status(raw)
 
     def list_files(self, ref: str) -> list[Path]:
-        """All deployable SQL files present at ``ref``."""
+        """All deployable declarative object files present at ``ref`` (excludes migrations)."""
+        raw = self._git("ls-tree", "-r", "--name-only", "-z", ref)
+        return [Path(n) for n in filter(None, raw.split("\0")) if self._is_object(Path(n))]
+
+    def migration_files(self, ref: str) -> list[tuple[str, Path]]:
+        """Ordered (id, path) of migration scripts under ``migrations/`` at ``ref``.
+
+        Id is the filename; order is lexicographic, so prefix files ``0001_…`` / a timestamp.
+        """
         raw = self._git("ls-tree", "-r", "--name-only", "-z", ref)
-        result = []
-        for name in filter(None, raw.split("\0")):
-            rel = Path(name)
-            if self._is_sql(rel) and not self.is_ignored(rel):
-                result.append(rel)
-        return result
+        out = [
+            Path(n) for n in filter(None, raw.split("\0"))
+            if self._is_migration(Path(n)) and Path(n).suffix.lower() == ".sql"
+        ]
+        return [(p.name, p) for p in sorted(out, key=lambda p: p.as_posix())]
 
     def _parse_name_status(self, raw: str) -> list[FileChange]:
         tokens = [t for t in raw.split("\0") if t]
@@ -79,12 +95,12 @@ class Repo:
             if code == "R":  # rename: status, old, new
                 new = Path(tokens[i + 2])
                 i += 3
-                if self._is_sql(new) and not self.is_ignored(new):
+                if self._is_object(new):
                     changes.append(FileChange(new, ChangeType.MODIFIED))
                 continue
             rel = Path(tokens[i + 1])
             i += 2
-            if not self._is_sql(rel) or self.is_ignored(rel):
+            if not self._is_object(rel):
                 continue
             mapping = {"A": ChangeType.ADDED, "M": ChangeType.MODIFIED,
                        "D": ChangeType.DELETED}

{dbly-0.1.0 → dbly-0.2.0}/src/dbly/report.py

@@ -11,7 +11,7 @@ import yaml
 from rich.console import Console
 from rich.table import Table
 
-from dbly.model import ObjectId, ObjectKind, Plan, Severity, Step
+from dbly.model import Migration, ObjectId, ObjectKind, Plan, Severity, Step
 
 
 def render_plan(plan: Plan, console: Console) -> None:
@@ -19,24 +19,32 @@ def render_plan(plan: Plan, console: Console) -> None:
         f"[bold]Plan[/bold] for [cyan]{plan.target}[/cyan]  "
         f"{plan.from_ref or '∅'} → {plan.to_ref}"
     )
-    if not plan.steps and not plan.warnings:
+    if not plan.steps and not plan.warnings and not plan.migrations and not plan.baselined:
         console.print("[green]nothing to do — target is up to date[/green]")
         return
 
-    table = Table(show_header=True, header_style="bold")
-    table.add_column("#", justify="right", style="dim")
-    table.add_column("severity")
-    table.add_column("kind")
-    table.add_column("step")
-    for i, step in enumerate(plan.steps, 1):
-        sev_style = "red" if step.severity is Severity.DESTRUCTIVE else "green"
-        table.add_row(
-            str(i),
-            f"[{sev_style}]{step.severity.value}[/{sev_style}]",
-            step.kind.value,
-            step.title,
+    for m in plan.migrations:
+        console.print(f"[magenta]migration[/magenta] run  {m.id}")
+    if plan.baselined:
+        console.print(
+            f"[dim]migration baseline (recorded, not run): {', '.join(plan.baselined)}[/dim]"
         )
-    console.print(table)
+
+    if plan.steps:
+        table = Table(show_header=True, header_style="bold")
+        table.add_column("#", justify="right", style="dim")
+        table.add_column("severity")
+        table.add_column("kind")
+        table.add_column("step")
+        for i, step in enumerate(plan.steps, 1):
+            sev_style = "red" if step.severity is Severity.DESTRUCTIVE else "green"
+            table.add_row(
+                str(i),
+                f"[{sev_style}]{step.severity.value}[/{sev_style}]",
+                step.kind.value,
+                step.title,
+            )
+        console.print(table)
 
     if plan.warnings:
         console.print("\n[bold yellow]Warnings[/bold yellow]")
@@ -73,6 +81,26 @@ def plan_to_sql(plan: Plan, *, state_ddl: str | None = None, record_sql: str | N
     if state_ddl:
         out += ["-- ledger table (no-op if it already exists)", state_ddl, ""]
 
+    safe_ref = plan.to_ref.replace("'", "''")
+    for m in plan.migrations:
+        mid = m.id.replace("'", "''")
+        out.append(f"-- migration (run-once): {m.id}")
+        body = m.sql.rstrip()
+        out.append(body if body.endswith(";") else body + ";")
+        out.append(
+            "INSERT INTO dbly_state (deployed_sha, migration_id) "
+            f"VALUES ('{safe_ref}', '{mid}');"
+        )
+        out.append("")
+    for mid in plan.baselined:
+        safe_mid = mid.replace("'", "''")
+        out.append(f"-- migration baseline (recorded, not run): {mid}")
+        out.append(
+            "INSERT INTO dbly_state (deployed_sha, migration_id) "
+            f"VALUES ('{safe_ref}', '{safe_mid}');"
+        )
+        out.append("")
+
     for i, step in enumerate(plan.steps, 1):
         mark = " !! DESTRUCTIVE" if step.severity is Severity.DESTRUCTIVE else ""
         out.append(f"-- [{i}] {step.severity.value}{mark}: {step.title}")
@@ -95,6 +123,11 @@ def plan_to_yaml(plan: Plan) -> str:
         "from_ref": plan.from_ref,
         "to_ref": plan.to_ref,
         "warnings": plan.warnings,
+        "migrations": [
+            {"id": m.id, "source_file": str(m.source_file), "sql": m.sql}
+            for m in plan.migrations
+        ],
+        "baselined": plan.baselined,
         "steps": [
             {
                 "title": s.title,
@@ -115,6 +148,11 @@ def plan_from_yaml(text: str) -> Plan:
     doc = yaml.safe_load(text)
     plan = Plan(target=doc["target"], from_ref=doc.get("from_ref"), to_ref=doc["to_ref"])
     plan.warnings = list(doc.get("warnings") or [])
+    plan.baselined = list(doc.get("baselined") or [])
+    plan.migrations = [
+        Migration(m["id"], m["sql"], Path(m["source_file"]))
+        for m in (doc.get("migrations") or [])
+    ]
     for s in doc.get("steps") or []:
         obj = s.get("object")
         oid = None

{dbly-0.1.0 → dbly-0.2.0}/tests/test_integration.py

@@ -152,6 +152,99 @@ def test_check_drift_against_live_db(tmp_path: Path):
     adapter.dispose()
 
 
+def _apply(adapter, plan, to_ref):
+    """Mirror cli.apply's order: baseline records, migrations run, then object steps."""
+    adapter.ensure_state_table()
+    for mid in plan.baselined:
+        adapter.record_migration(to_ref, mid)
+    for m in plan.migrations:
+        adapter.run_init_script(m.sql)
+        adapter.record_migration(to_ref, m.id)
+    if plan.steps:
+        adapter.apply([s.sql for s in plan.steps])
+    adapter.record_deploy(to_ref, [])
+
+
+def test_explicit_migration_runs_on_upgrade_once(tmp_path: Path):
+    repo_root = tmp_path / "db"
+    repo_root.mkdir()
+    _init_repo(repo_root)
+    (repo_root / "kunde.tbl").write_text(
+        "CREATE TABLE IF NOT EXISTS kunde (id INTEGER, name TEXT);", encoding="utf-8"
+    )
+    ref1 = _commit(repo_root, "v1")
+
+    db = tmp_path / "mig.db"
+    adapter = SqliteAdapter(ConnectionConfig(environment="sqlite", service=str(db)))
+    repo = Repo(repo_root)
+    _apply(adapter, build_plan(repo, adapter, from_ref=None, to_ref=ref1,
+                               target="sqlite", dialect="sqlite"), ref1)
+
+    # v2: rename name -> full_name via an explicit migration; canonical table reflects it
+    (repo_root / "kunde.tbl").write_text(
+        "CREATE TABLE IF NOT EXISTS kunde (id INTEGER, full_name TEXT);", encoding="utf-8"
+    )
+    (repo_root / "migrations").mkdir()
+    (repo_root / "migrations" / "0001_rename.sql").write_text(
+        "ALTER TABLE kunde RENAME COLUMN name TO full_name;", encoding="utf-8"
+    )
+    ref2 = _commit(repo_root, "v2")
+
+    plan2 = build_plan(repo, adapter, from_ref=ref1, to_ref=ref2, target="sqlite", dialect="sqlite")
+    assert [m.id for m in plan2.migrations] == ["0001_rename.sql"]   # pending, will run
+    _apply(adapter, plan2, ref2)
+    cols = {c.name.lower() for c in adapter.get_columns(None, "kunde")}
+    assert "full_name" in cols and "name" not in cols                # rename happened
+    assert "0001_rename.sql" in adapter.applied_migrations()
+
+    # re-plan: migration already applied → not pending again
+    plan3 = build_plan(repo, adapter, from_ref=ref1, to_ref=ref2, target="sqlite", dialect="sqlite")
+    assert plan3.migrations == []
+    adapter.dispose()
+
+
+def test_bootstrap_baselines_migrations_without_running(tmp_path: Path):
+    repo_root = tmp_path / "db"
+    repo_root.mkdir()
+    _init_repo(repo_root)
+    (repo_root / "kunde.tbl").write_text(
+        "CREATE TABLE IF NOT EXISTS kunde (id INTEGER, full_name TEXT);", encoding="utf-8"
+    )
+    (repo_root / "migrations").mkdir()
+    # a rename that would FAIL on a fresh DB (column `name` never existed)
+    (repo_root / "migrations" / "0001_rename.sql").write_text(
+        "ALTER TABLE kunde RENAME COLUMN name TO full_name;", encoding="utf-8"
+    )
+    ref = _commit(repo_root, "v1")
+
+    db = tmp_path / "fresh.db"
+    adapter = SqliteAdapter(ConnectionConfig(environment="sqlite", service=str(db)))
+    repo = Repo(repo_root)
+    plan = build_plan(repo, adapter, from_ref=None, to_ref=ref, target="sqlite", dialect="sqlite")
+    assert plan.baselined == ["0001_rename.sql"] and plan.migrations == []
+    _apply(adapter, plan, ref)  # must NOT run the rename (would error) — only record it
+
+    cols = {c.name.lower() for c in adapter.get_columns(None, "kunde")}
+    assert cols == {"id", "full_name"}                               # built from canonical CREATE
+    assert "0001_rename.sql" in adapter.applied_migrations()         # recorded as baseline
+    adapter.dispose()
+
+
+def test_migration_files_excluded_from_objects(tmp_path: Path):
+    repo_root = tmp_path / "db"
+    repo_root.mkdir()
+    _init_repo(repo_root)
+    (repo_root / "kunde.tbl").write_text("CREATE TABLE kunde (id INTEGER);", encoding="utf-8")
+    (repo_root / "migrations").mkdir()
+    (repo_root / "migrations" / "0001_x.sql").write_text("ALTER TABLE kunde ADD c TEXT;",
+                                                         encoding="utf-8")
+    ref = _commit(repo_root, "v1")
+    repo = Repo(repo_root)
+    objs = [p.name for p in repo.list_files(ref)]
+    assert "kunde.tbl" in objs and "0001_x.sql" not in objs          # migrations not objects
+    assert [mid for mid, _ in repo.migration_files(ref)] == ["0001_x.sql"]
+
+
 def test_init_runs_ordered_multistatement_scripts(tmp_path: Path):
     repo_root = tmp_path / "db"
     (repo_root / "init").mkdir(parents=True)

{dbly-0.1.0 → dbly-0.2.0}/uv.lock

@@ -129,7 +129,7 @@ wheels = [
 
 [[package]]
 name = "dbly"
-version = "0.1.0"
+version = "0.2.0"
 source = { editable = "." }
 dependencies = [
     { name = "pathspec" },