dbly 0.0.1__py3-none-any.whl

dbly/__init__.py

@@ -0,0 +1,7 @@
+"""dbly — state-based, cross-engine database deployment.
+
+git (what changed) + sqlglot (what it is) + live introspection (what's really there).
+See CONCEPT.md for the design rationale.
+"""
+
+__version__ = "0.0.1"

dbly/adapters/__init__.py

@@ -0,0 +1,35 @@
+"""Engine-specific adapters. Postgres is the Leitstern (CONCEPT.md §10, §16)."""
+from __future__ import annotations
+
+from dbly.adapters.base import Adapter, Column
+from dbly.config import ConnectionConfig
+from dbly.engine import detect_dialect
+
+_POSTGRES = {"postgres", "postgresql", "pg"}
+_SQLITE = {"sqlite", "sqlite3"}
+_MSSQL = {"sqlserver", "mssql", "ms-sql"}
+_ORACLE = {"oracle"}
+
+
+def get_adapter(cfg: ConnectionConfig) -> Adapter:
+    env = detect_dialect(cfg)
+    if env in _POSTGRES:
+        from dbly.adapters.postgres import PostgresAdapter
+
+        return PostgresAdapter(cfg)
+    if env in _SQLITE:
+        from dbly.adapters.sqlite import SqliteAdapter
+
+        return SqliteAdapter(cfg)
+    if env in _MSSQL:
+        from dbly.adapters.mssql import MssqlAdapter
+
+        return MssqlAdapter(cfg)
+    if env in _ORACLE:
+        from dbly.adapters.oracle import OracleAdapter
+
+        return OracleAdapter(cfg)
+    raise NotImplementedError(f"no adapter for environment {env!r}.")
+
+
+__all__ = ["Adapter", "Column", "get_adapter"]

dbly/adapters/base.py

@@ -0,0 +1,93 @@
+"""Adapter interface — the per-engine execution + introspection contract.
+
+Each adapter encodes its engine's DDL transaction semantics (CONCEPT.md §10):
+
+* Postgres — transactional DDL: wrap the whole apply in one transaction, clean rollback.
+* Oracle   — DDL auto-commits: per-statement, verify-driven, no rollback assumption.
+* SQL Server — mixed: wrap where supported.
+
+The adapter is also the *reality* layer (CONCEPT.md §2): it introspects the live schema so
+the planner can diff desired vs. actual tables.
+"""
+from __future__ import annotations
+
+import abc
+
+from sqlalchemy import Engine
+
+from dbly.config import ConnectionConfig
+from dbly.engine import make_engine
+from dbly.model import Column, ObjectId
+
+__all__ = ["Adapter", "Column"]
+
+
+class Adapter(abc.ABC):
+    """Base adapter. Subclasses implement engine specifics."""
+
+    #: whether DDL participates in transactions (drives apply strategy)
+    transactional_ddl: bool = False
+
+    def __init__(self, cfg: ConnectionConfig):
+        self.cfg = cfg
+        self._engine: Engine | None = None
+
+    @property
+    def engine(self) -> Engine:
+        if self._engine is None:
+            self._engine = make_engine(self.cfg)
+        return self._engine
+
+    # --- introspection (reality layer) ------------------------------------------------
+    @abc.abstractmethod
+    def table_exists(self, schema: str | None, name: str) -> bool: ...
+
+    @abc.abstractmethod
+    def get_columns(self, schema: str | None, name: str) -> list[Column]: ...
+
+    # --- dialect-specific DDL generation -----------------------------------------------
+    @abc.abstractmethod
+    def add_column_sql(self, table: ObjectId, col: Column) -> str:
+        """Generate the additive ``ALTER TABLE … ADD`` for this dialect.
+
+        Postgres/SQLite use ``ADD COLUMN``; T-SQL uses ``ADD``. The planner stays
+        dialect-agnostic and delegates the rendering here.
+        """
+
+    # --- execution ---------------------------------------------------------------------
+    @abc.abstractmethod
+    def apply(self, statements: list[str]) -> None:
+        """Execute statements with the engine's appropriate transaction strategy."""
+
+    @abc.abstractmethod
+    def run_init_script(self, script: str) -> None:
+        """Run a privileged init script (CONCEPT.md §6) verbatim.
+
+        Init scripts are imperative groundwork — possibly multi-statement and containing
+        statements that cannot run inside a transaction (e.g. Postgres ``CREATE DATABASE``).
+        Implementations therefore run in **autocommit** and accept the whole script.
+        """
+
+    # --- state ledger ------------------------------------------------------------------
+    @abc.abstractmethod
+    def ensure_state_table(self) -> None: ...
+
+    @abc.abstractmethod
+    def get_deployed_ref(self) -> str | None: ...
+
+    @abc.abstractmethod
+    def record_deploy(self, ref: str, migration_ids: list[str]) -> None: ...
+
+    # --- pure SQL builders (no connection — used by `plan --sql` export) ----------------
+    @abc.abstractmethod
+    def state_table_ddl(self) -> str:
+        """``CREATE TABLE IF NOT EXISTS dbly_state …`` for this engine."""
+
+    @abc.abstractmethod
+    def record_deploy_sql(self, ref: str) -> str:
+        """A standalone ``INSERT`` recording the deploy — for hand-run scripts."""
+
+    def dispose(self) -> None:
+        if self._engine is not None:
+            self._engine.dispose()
+            self._engine = None

dbly/adapters/mssql.py

@@ -0,0 +1,112 @@
+"""SQL Server adapter (T-SQL via pymssql).
+
+DDL in SQL Server is largely transactional — CREATE/ALTER/DROP of tables, views, procedures
+roll back inside a transaction — so object deploys wrap in one transaction like Postgres.
+T-SQL specifics handled here:
+
+* no ``CREATE TABLE IF NOT EXISTS`` → guarded ``IF NOT EXISTS (…) BEGIN … END``;
+* ``ALTER TABLE … ADD`` (not ``ADD COLUMN``);
+* init scripts are split on ``GO`` batch separators (required so e.g. ``CREATE PROCEDURE``
+  is the first statement in its batch).
+
+Requires the ``mssql`` extra (``pymssql``). End-to-end verification needs a reachable SQL
+Server instance; unit-testable pieces (SQL string builders) work without one.
+"""
+from __future__ import annotations
+
+import re
+
+from sqlalchemy import inspect, text
+
+from dbly.adapters.base import Adapter, Column
+from dbly.model import ObjectId
+
+_GO_RE = re.compile(r"^\s*GO\s*$", re.IGNORECASE | re.MULTILINE)
+
+_STATE_DDL = """
+IF NOT EXISTS (SELECT 1 FROM sys.tables WHERE name = 'dbly_state')
+BEGIN
+    CREATE TABLE dbly_state (
+        id           BIGINT IDENTITY(1,1) PRIMARY KEY,
+        deployed_sha NVARCHAR(64) NOT NULL,
+        migration_id NVARCHAR(200) NULL,
+        applied_at   DATETIME2 NOT NULL DEFAULT SYSUTCDATETIME()
+    )
+END
+"""
+
+
+class MssqlAdapter(Adapter):
+    transactional_ddl = True
+
+    def table_exists(self, schema: str | None, name: str) -> bool:
+        return inspect(self.engine).has_table(name, schema=schema)
+
+    def get_columns(self, schema: str | None, name: str) -> list[Column]:
+        cols = inspect(self.engine).get_columns(name, schema=schema)
+        return [
+            Column(
+                name=c["name"],
+                type=str(c["type"]),
+                nullable=bool(c["nullable"]),
+                default=None if c.get("default") is None else str(c["default"]),
+            )
+            for c in cols
+        ]
+
+    def add_column_sql(self, table: ObjectId, col: Column) -> str:
+        # T-SQL: ADD, not ADD COLUMN
+        parts = [f"ALTER TABLE {table} ADD {col.name} {col.type}"]
+        if not col.nullable:
+            parts.append("NOT NULL")
+        if col.default is not None:
+            parts.append(f"DEFAULT {col.default}")
+        return " ".join(parts) + ";"
+
+    def apply(self, statements: list[str]) -> None:
+        with self.engine.begin() as conn:
+            for stmt in statements:
+                if stmt.strip():
+                    conn.execute(text(stmt))
+
+    def run_init_script(self, script: str) -> None:
+        # Split on GO batch separators and run each batch in autocommit.
+        batches = [b for b in _GO_RE.split(script) if b.strip()]
+        with self.engine.connect() as conn:
+            conn = conn.execution_options(isolation_level="AUTOCOMMIT")
+            for batch in batches:
+                conn.exec_driver_sql(batch)
+
+    def state_table_ddl(self) -> str:
+        return _STATE_DDL.strip()
+
+    def record_deploy_sql(self, ref: str) -> str:
+        safe = ref.replace("'", "''")
+        return f"INSERT INTO dbly_state (deployed_sha) VALUES ('{safe}');"
+
+    def ensure_state_table(self) -> None:
+        with self.engine.begin() as conn:
+            conn.execute(text(_STATE_DDL))
+
+    def get_deployed_ref(self) -> str | None:
+        self.ensure_state_table()
+        with self.engine.connect() as conn:
+            row = conn.execute(
+                text(
+                    "SELECT TOP 1 deployed_sha FROM dbly_state "
+                    "ORDER BY applied_at DESC, id DESC"
+                )
+            ).first()
+        return row[0] if row else None
+
+    def record_deploy(self, ref: str, migration_ids: list[str]) -> None:
+        self.ensure_state_table()
+        with self.engine.begin() as conn:
+            for mid in (migration_ids or [None]):
+                conn.execute(
+                    text(
+                        "INSERT INTO dbly_state (deployed_sha, migration_id) "
+                        "VALUES (:sha, :mid)"
+                    ),
+                    {"sha": ref, "mid": mid},
+                )

dbly/adapters/oracle.py

@@ -0,0 +1,160 @@
+"""Oracle adapter (oracledb) — the hardest semantics (CONCEPT.md §10).
+
+Oracle commits DDL **implicitly** — there is no all-or-nothing rollback for a multi-statement
+deploy. ``apply`` therefore runs statements sequentially (each self-commits); a mid-deploy
+failure leaves earlier statements applied. Mitigate with verify steps and idempotent/guarded
+DDL — never rely on rollback here.
+
+Other Oracle specifics handled in this adapter:
+
+* no ``CREATE TABLE IF NOT EXISTS`` → the ledger is created via a guarded PL/SQL block that
+  swallows ORA-00955 ("name already used");
+* ``ALTER TABLE … ADD col`` with Oracle ordering (``DEFAULT`` before ``NOT NULL``);
+* unquoted identifiers are stored upper-case → introspection normalizes case;
+* python-oracledb rejects a trailing ``;`` on plain SQL but needs the ``;`` *inside* PL/SQL →
+  terminators are stripped for SQL and PL/SQL blocks are kept intact;
+* init scripts are split on SQL*Plus ``/`` block terminators.
+
+Requires the ``oracle`` extra (oracledb). Not yet e2e-verified against a live instance.
+"""
+from __future__ import annotations
+
+import re
+
+from sqlalchemy import inspect, text
+
+from dbly.adapters.base import Adapter, Column
+from dbly.model import ObjectId
+
+_SLASH_RE = re.compile(r"(?m)^\s*/\s*$")
+_PLSQL_RE = re.compile(
+    r"^\s*(CREATE\s+(OR\s+REPLACE\s+)?(EDITIONABLE\s+|NONEDITIONABLE\s+)?"
+    r"(PROCEDURE|FUNCTION|PACKAGE|TRIGGER|TYPE)\b|BEGIN\b|DECLARE\b)",
+    re.IGNORECASE,
+)
+
+_STATE_BLOCK = """
+BEGIN
+    EXECUTE IMMEDIATE '
+        CREATE TABLE dbly_state (
+            id           NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+            deployed_sha VARCHAR2(64) NOT NULL,
+            migration_id VARCHAR2(200),
+            applied_at   TIMESTAMP DEFAULT SYSTIMESTAMP NOT NULL
+        )';
+EXCEPTION
+    WHEN OTHERS THEN
+        IF SQLCODE != -955 THEN RAISE; END IF;  -- ORA-00955: name already used
+END;
+"""
+
+
+def _is_plsql(stmt: str) -> bool:
+    return bool(_PLSQL_RE.match(stmt))
+
+
+def strip_terminator(stmt: str) -> str:
+    """Strip a trailing ``;`` from plain SQL; leave PL/SQL blocks (END;) intact."""
+    s = stmt.strip()
+    if not _is_plsql(s) and s.endswith(";"):
+        return s[:-1].rstrip()
+    return s
+
+
+def split_oracle_script(script: str) -> list[str]:
+    """Split an init script into executable units.
+
+    PL/SQL blocks are terminated by a lone ``/`` (SQL*Plus convention) and run whole; plain
+    SQL between terminators is split on ``;``. Semicolons inside string literals are a known
+    edge case (rare in init scripts).
+    """
+    out: list[str] = []
+    for chunk in _SLASH_RE.split(script):
+        s = chunk.strip()
+        if not s:
+            continue
+        if _is_plsql(s):
+            out.append(s)
+        else:
+            out.extend(st.strip() for st in s.split(";") if st.strip())
+    return out
+
+
+class OracleAdapter(Adapter):
+    transactional_ddl = False  # DDL auto-commits — no rollback
+
+    @staticmethod
+    def _norm(ident: str | None) -> str | None:
+        # unquoted Oracle identifiers are upper-cased on storage
+        return ident.upper() if ident else ident
+
+    def table_exists(self, schema: str | None, name: str) -> bool:
+        return inspect(self.engine).has_table(self._norm(name), schema=self._norm(schema))
+
+    def get_columns(self, schema: str | None, name: str) -> list[Column]:
+        cols = inspect(self.engine).get_columns(self._norm(name), schema=self._norm(schema))
+        return [
+            Column(
+                name=c["name"],
+                type=str(c["type"]),
+                nullable=bool(c["nullable"]),
+                default=None if c.get("default") is None else str(c["default"]),
+            )
+            for c in cols
+        ]
+
+    def add_column_sql(self, table: ObjectId, col: Column) -> str:
+        parts = [f"ALTER TABLE {table} ADD {col.name} {col.type}"]
+        if col.default is not None:
+            parts.append(f"DEFAULT {col.default}")  # Oracle: DEFAULT before NOT NULL
+        if not col.nullable:
+            parts.append("NOT NULL")
+        return " ".join(parts) + ";"
+
+    def apply(self, statements: list[str]) -> None:
+        # No transactional rollback (DDL auto-commits). Sequential; commit covers any DML.
+        with self.engine.connect() as conn:
+            for stmt in statements:
+                s = strip_terminator(stmt)
+                if s:
+                    conn.exec_driver_sql(s)
+            conn.commit()
+
+    def run_init_script(self, script: str) -> None:
+        with self.engine.connect() as conn:
+            for stmt in split_oracle_script(script):
+                conn.exec_driver_sql(stmt)
+            conn.commit()
+
+    def state_table_ddl(self) -> str:
+        return _STATE_BLOCK.strip() + "\n/"  # '/' terminates the block for a SQL*Plus hand-run
+
+    def record_deploy_sql(self, ref: str) -> str:
+        return f"INSERT INTO dbly_state (deployed_sha) VALUES ('{ref.replace(chr(39), chr(39) * 2)}');"
+
+    def ensure_state_table(self) -> None:
+        with self.engine.connect() as conn:
+            conn.exec_driver_sql(_STATE_BLOCK.strip())
+            conn.commit()
+
+    def get_deployed_ref(self) -> str | None:
+        self.ensure_state_table()
+        with self.engine.connect() as conn:
+            row = conn.exec_driver_sql(
+                "SELECT deployed_sha FROM dbly_state "
+                "ORDER BY applied_at DESC, id DESC FETCH FIRST 1 ROW ONLY"
+            ).first()
+        return row[0] if row else None
+
+    def record_deploy(self, ref: str, migration_ids: list[str]) -> None:
+        self.ensure_state_table()
+        with self.engine.connect() as conn:
+            for mid in (migration_ids or [None]):
+                conn.execute(
+                    text(
+                        "INSERT INTO dbly_state (deployed_sha, migration_id) "
+                        "VALUES (:sha, :mid)"
+                    ),
+                    {"sha": ref, "mid": mid},
+                )
+            conn.commit()

dbly/adapters/postgres.py

@@ -0,0 +1,103 @@
+"""PostgreSQL adapter — the Leitstern (CONCEPT.md §16).
+
+Postgres has transactional DDL, so the whole apply runs in a single transaction: on any
+failure everything rolls back and the database is untouched. This is the clean reference
+against which the trickier Oracle/SQL-Server semantics are later measured.
+"""
+from __future__ import annotations
+
+from sqlalchemy import inspect, text
+
+from dbly.adapters.base import Adapter, Column
+from dbly.model import ObjectId
+
+_STATE_DDL = """
+CREATE TABLE IF NOT EXISTS dbly_state (
+    id           bigserial PRIMARY KEY,
+    deployed_sha text        NOT NULL,
+    migration_id text,
+    applied_at   timestamptz NOT NULL DEFAULT now()
+)
+"""
+
+
+class PostgresAdapter(Adapter):
+    transactional_ddl = True
+
+    def table_exists(self, schema: str | None, name: str) -> bool:
+        insp = inspect(self.engine)
+        return insp.has_table(name, schema=schema)
+
+    def get_columns(self, schema: str | None, name: str) -> list[Column]:
+        insp = inspect(self.engine)
+        cols = insp.get_columns(name, schema=schema)
+        return [
+            Column(
+                name=c["name"],
+                type=str(c["type"]),
+                nullable=bool(c["nullable"]),
+                default=None if c.get("default") is None else str(c["default"]),
+            )
+            for c in cols
+        ]
+
+    def add_column_sql(self, table: ObjectId, col: Column) -> str:
+        parts = [f"ALTER TABLE {table} ADD COLUMN {col.name} {col.type}"]
+        if not col.nullable:
+            parts.append("NOT NULL")
+        if col.default is not None:
+            parts.append(f"DEFAULT {col.default}")
+        return " ".join(parts) + ";"
+
+    def apply(self, statements: list[str]) -> None:
+        # transactional DDL → one atomic transaction
+        with self.engine.begin() as conn:
+            for stmt in statements:
+                if stmt.strip():
+                    conn.execute(text(stmt))
+
+    def run_init_script(self, script: str) -> None:
+        # autocommit: CREATE DATABASE & friends cannot run inside a transaction block.
+        # psycopg3 executes a multi-statement string in a single exec_driver_sql call.
+        with self.engine.connect() as conn:
+            conn = conn.execution_options(isolation_level="AUTOCOMMIT")
+            conn.exec_driver_sql(script)
+
+    def state_table_ddl(self) -> str:
+        return _STATE_DDL.strip() + ";"
+
+    def record_deploy_sql(self, ref: str) -> str:
+        return f"INSERT INTO dbly_state (deployed_sha) VALUES ('{ref.replace(chr(39), chr(39) * 2)}');"
+
+    def ensure_state_table(self) -> None:
+        with self.engine.begin() as conn:
+            conn.execute(text(_STATE_DDL))
+
+    def get_deployed_ref(self) -> str | None:
+        self.ensure_state_table()
+        with self.engine.connect() as conn:
+            row = conn.execute(
+                text(
+                    "SELECT deployed_sha FROM dbly_state "
+                    "ORDER BY applied_at DESC, id DESC LIMIT 1"
+                )
+            ).first()
+        return row[0] if row else None
+
+    def record_deploy(self, ref: str, migration_ids: list[str]) -> None:
+        self.ensure_state_table()
+        with self.engine.begin() as conn:
+            if migration_ids:
+                for mid in migration_ids:
+                    conn.execute(
+                        text(
+                            "INSERT INTO dbly_state (deployed_sha, migration_id) "
+                            "VALUES (:sha, :mid)"
+                        ),
+                        {"sha": ref, "mid": mid},
+                    )
+            else:
+                conn.execute(
+                    text("INSERT INTO dbly_state (deployed_sha) VALUES (:sha)"),
+                    {"sha": ref},
+                )

dbly/adapters/sqlite.py

@@ -0,0 +1,94 @@
+"""SQLite adapter — transactional DDL, no native deps. Doubles as the test backend.
+
+SQLite has no schemas; schema-qualified identities are treated as schemaless (the folder
+hint should be omitted for SQLite repos). ALTER TABLE supports ADD COLUMN, which is all the
+additive path needs.
+"""
+from __future__ import annotations
+
+from sqlalchemy import inspect, text
+
+from dbly.adapters.base import Adapter, Column
+from dbly.model import ObjectId
+
+_STATE_DDL = """
+CREATE TABLE IF NOT EXISTS dbly_state (
+    id           INTEGER PRIMARY KEY AUTOINCREMENT,
+    deployed_sha TEXT NOT NULL,
+    migration_id TEXT,
+    applied_at   TEXT NOT NULL DEFAULT (datetime('now'))
+)
+"""
+
+
+class SqliteAdapter(Adapter):
+    transactional_ddl = True
+
+    def table_exists(self, schema: str | None, name: str) -> bool:
+        return inspect(self.engine).has_table(name)
+
+    def get_columns(self, schema: str | None, name: str) -> list[Column]:
+        cols = inspect(self.engine).get_columns(name)
+        return [
+            Column(
+                name=c["name"],
+                type=str(c["type"]),
+                nullable=bool(c["nullable"]),
+                default=None if c.get("default") is None else str(c["default"]),
+            )
+            for c in cols
+        ]
+
+    def add_column_sql(self, table: ObjectId, col: Column) -> str:
+        # SQLite has no schemas; ignore the schema qualifier.
+        parts = [f"ALTER TABLE {table.name} ADD COLUMN {col.name} {col.type}"]
+        if not col.nullable:
+            parts.append("NOT NULL")
+        if col.default is not None:
+            parts.append(f"DEFAULT {col.default}")
+        return " ".join(parts) + ";"
+
+    def apply(self, statements: list[str]) -> None:
+        with self.engine.begin() as conn:
+            for stmt in statements:
+                if stmt.strip():
+                    conn.execute(text(stmt))
+
+    def run_init_script(self, script: str) -> None:
+        # sqlite3's executescript handles multiple statements and auto-commits.
+        raw = self.engine.raw_connection()
+        try:
+            raw.driver_connection.executescript(script)
+            raw.commit()
+        finally:
+            raw.close()
+
+    def state_table_ddl(self) -> str:
+        return _STATE_DDL.strip() + ";"
+
+    def record_deploy_sql(self, ref: str) -> str:
+        return f"INSERT INTO dbly_state (deployed_sha) VALUES ('{ref.replace(chr(39), chr(39) * 2)}');"
+
+    def ensure_state_table(self) -> None:
+        with self.engine.begin() as conn:
+            conn.execute(text(_STATE_DDL))
+
+    def get_deployed_ref(self) -> str | None:
+        self.ensure_state_table()
+        with self.engine.connect() as conn:
+            row = conn.execute(
+                text("SELECT deployed_sha FROM dbly_state ORDER BY id DESC LIMIT 1")
+            ).first()
+        return row[0] if row else None
+
+    def record_deploy(self, ref: str, migration_ids: list[str]) -> None:
+        self.ensure_state_table()
+        with self.engine.begin() as conn:
+            for mid in (migration_ids or [None]):
+                conn.execute(
+                    text(
+                        "INSERT INTO dbly_state (deployed_sha, migration_id) "
+                        "VALUES (:sha, :mid)"
+                    ),
+                    {"sha": ref, "mid": mid},
+                )