crochet-migration 0.1.0__py3-none-any.whl

crochet/ledger/sqlite.py

@@ -0,0 +1,282 @@
+"""SQLite ledger — the authoritative record of applied migrations and data batches."""
+
+from __future__ import annotations
+
+import sqlite3
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterator
+
+from crochet.errors import LedgerError, LedgerIntegrityError
+
+_SCHEMA_VERSION = 1
+
+_INIT_SQL = """\
+CREATE TABLE IF NOT EXISTS ledger_meta (
+    key   TEXT PRIMARY KEY,
+    value TEXT NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS applied_migrations (
+    revision_id   TEXT PRIMARY KEY,
+    parent_id     TEXT,
+    description   TEXT NOT NULL DEFAULT '',
+    schema_hash   TEXT NOT NULL,
+    applied_at    TEXT NOT NULL,
+    rollback_safe INTEGER NOT NULL DEFAULT 1
+);
+
+CREATE TABLE IF NOT EXISTS dataset_batches (
+    batch_id       TEXT PRIMARY KEY,
+    migration_id   TEXT,
+    source_file    TEXT,
+    file_checksum  TEXT,
+    loader_version TEXT,
+    record_count   INTEGER,
+    created_at     TEXT NOT NULL,
+    FOREIGN KEY (migration_id) REFERENCES applied_migrations(revision_id)
+);
+
+CREATE TABLE IF NOT EXISTS schema_snapshots (
+    schema_hash   TEXT PRIMARY KEY,
+    snapshot_json TEXT NOT NULL,
+    created_at    TEXT NOT NULL
+);
+"""
+
+
+@dataclass
+class AppliedMigration:
+    revision_id: str
+    parent_id: str | None
+    description: str
+    schema_hash: str
+    applied_at: str
+    rollback_safe: bool
+
+
+@dataclass
+class DatasetBatch:
+    batch_id: str
+    migration_id: str | None
+    source_file: str | None
+    file_checksum: str | None
+    loader_version: str | None
+    record_count: int | None
+    created_at: str
+
+
+class Ledger:
+    """SQLite-backed ledger for migration and data-batch tracking."""
+
+    def __init__(self, db_path: Path) -> None:
+        self._db_path = db_path
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = sqlite3.connect(str(db_path))
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA foreign_keys=ON")
+        self._init_schema()
+
+    # ------------------------------------------------------------------
+    # Lifecycle
+    # ------------------------------------------------------------------
+
+    def _init_schema(self) -> None:
+        self._conn.executescript(_INIT_SQL)
+        cur = self._conn.execute(
+            "SELECT value FROM ledger_meta WHERE key = 'schema_version'"
+        )
+        row = cur.fetchone()
+        if row is None:
+            self._conn.execute(
+                "INSERT INTO ledger_meta (key, value) VALUES (?, ?)",
+                ("schema_version", str(_SCHEMA_VERSION)),
+            )
+            self._conn.commit()
+
+    def close(self) -> None:
+        self._conn.close()
+
+    def __enter__(self) -> "Ledger":
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    # ------------------------------------------------------------------
+    # Migrations
+    # ------------------------------------------------------------------
+
+    def record_migration(
+        self,
+        revision_id: str,
+        parent_id: str | None,
+        description: str,
+        schema_hash: str,
+        rollback_safe: bool = True,
+    ) -> AppliedMigration:
+        now = datetime.now(timezone.utc).isoformat()
+        try:
+            self._conn.execute(
+                """INSERT INTO applied_migrations
+                   (revision_id, parent_id, description, schema_hash, applied_at, rollback_safe)
+                   VALUES (?, ?, ?, ?, ?, ?)""",
+                (revision_id, parent_id, description, schema_hash, now, int(rollback_safe)),
+            )
+            self._conn.commit()
+        except sqlite3.IntegrityError as exc:
+            raise LedgerError(
+                f"Migration '{revision_id}' is already recorded in the ledger."
+            ) from exc
+        return AppliedMigration(
+            revision_id=revision_id,
+            parent_id=parent_id,
+            description=description,
+            schema_hash=schema_hash,
+            applied_at=now,
+            rollback_safe=rollback_safe,
+        )
+
+    def remove_migration(self, revision_id: str) -> None:
+        self._conn.execute(
+            "DELETE FROM applied_migrations WHERE revision_id = ?", (revision_id,)
+        )
+        self._conn.commit()
+
+    def get_applied_migrations(self) -> list[AppliedMigration]:
+        cur = self._conn.execute(
+            "SELECT revision_id, parent_id, description, schema_hash, applied_at, rollback_safe "
+            "FROM applied_migrations ORDER BY applied_at"
+        )
+        return [
+            AppliedMigration(
+                revision_id=row[0],
+                parent_id=row[1],
+                description=row[2],
+                schema_hash=row[3],
+                applied_at=row[4],
+                rollback_safe=bool(row[5]),
+            )
+            for row in cur.fetchall()
+        ]
+
+    def get_head(self) -> AppliedMigration | None:
+        migrations = self.get_applied_migrations()
+        return migrations[-1] if migrations else None
+
+    def is_applied(self, revision_id: str) -> bool:
+        cur = self._conn.execute(
+            "SELECT 1 FROM applied_migrations WHERE revision_id = ?", (revision_id,)
+        )
+        return cur.fetchone() is not None
+
+    # ------------------------------------------------------------------
+    # Dataset batches
+    # ------------------------------------------------------------------
+
+    def record_batch(
+        self,
+        batch_id: str,
+        migration_id: str | None = None,
+        source_file: str | None = None,
+        file_checksum: str | None = None,
+        loader_version: str | None = None,
+        record_count: int | None = None,
+    ) -> DatasetBatch:
+        now = datetime.now(timezone.utc).isoformat()
+        try:
+            self._conn.execute(
+                """INSERT INTO dataset_batches
+                   (batch_id, migration_id, source_file, file_checksum,
+                    loader_version, record_count, created_at)
+                   VALUES (?, ?, ?, ?, ?, ?, ?)""",
+                (batch_id, migration_id, source_file, file_checksum,
+                 loader_version, record_count, now),
+            )
+            self._conn.commit()
+        except sqlite3.IntegrityError as exc:
+            raise LedgerError(
+                f"Batch '{batch_id}' is already recorded in the ledger."
+            ) from exc
+        return DatasetBatch(
+            batch_id=batch_id,
+            migration_id=migration_id,
+            source_file=source_file,
+            file_checksum=file_checksum,
+            loader_version=loader_version,
+            record_count=record_count,
+            created_at=now,
+        )
+
+    def get_batches(self, migration_id: str | None = None) -> list[DatasetBatch]:
+        if migration_id:
+            cur = self._conn.execute(
+                "SELECT batch_id, migration_id, source_file, file_checksum, "
+                "loader_version, record_count, created_at "
+                "FROM dataset_batches WHERE migration_id = ? ORDER BY created_at",
+                (migration_id,),
+            )
+        else:
+            cur = self._conn.execute(
+                "SELECT batch_id, migration_id, source_file, file_checksum, "
+                "loader_version, record_count, created_at "
+                "FROM dataset_batches ORDER BY created_at"
+            )
+        return [
+            DatasetBatch(
+                batch_id=row[0],
+                migration_id=row[1],
+                source_file=row[2],
+                file_checksum=row[3],
+                loader_version=row[4],
+                record_count=row[5],
+                created_at=row[6],
+            )
+            for row in cur.fetchall()
+        ]
+
+    def remove_batch(self, batch_id: str) -> None:
+        self._conn.execute(
+            "DELETE FROM dataset_batches WHERE batch_id = ?", (batch_id,)
+        )
+        self._conn.commit()
+
+    # ------------------------------------------------------------------
+    # Schema snapshots
+    # ------------------------------------------------------------------
+
+    def store_snapshot(self, schema_hash: str, snapshot_json: str) -> None:
+        now = datetime.now(timezone.utc).isoformat()
+        self._conn.execute(
+            """INSERT OR REPLACE INTO schema_snapshots
+               (schema_hash, snapshot_json, created_at) VALUES (?, ?, ?)""",
+            (schema_hash, snapshot_json, now),
+        )
+        self._conn.commit()
+
+    def get_snapshot(self, schema_hash: str) -> str | None:
+        cur = self._conn.execute(
+            "SELECT snapshot_json FROM schema_snapshots WHERE schema_hash = ?",
+            (schema_hash,),
+        )
+        row = cur.fetchone()
+        return row[0] if row else None
+
+    # ------------------------------------------------------------------
+    # Integrity
+    # ------------------------------------------------------------------
+
+    def verify_chain(self) -> list[str]:
+        """Verify the parent-chain integrity. Returns a list of issues."""
+        issues: list[str] = []
+        migrations = self.get_applied_migrations()
+        ids = {m.revision_id for m in migrations}
+
+        for m in migrations:
+            if m.parent_id is not None and m.parent_id not in ids:
+                issues.append(
+                    f"Migration '{m.revision_id}' references unknown parent "
+                    f"'{m.parent_id}'."
+                )
+        return issues

crochet/migrations/__init__.py

@@ -0,0 +1,6 @@
+"""Migration engine and operations."""
+
+from crochet.migrations.engine import MigrationEngine
+from crochet.migrations.operations import MigrationContext
+
+__all__ = ["MigrationEngine", "MigrationContext"]

crochet/migrations/engine.py

@@ -0,0 +1,279 @@
+"""Migration execution engine — ordering, upgrade, downgrade."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+from pathlib import Path
+from typing import Any
+
+from crochet.config import CrochetConfig
+from crochet.errors import (
+    MigrationChainError,
+    MigrationError,
+    RollbackUnsafeError,
+)
+from crochet.ir.diff import SchemaDiff, diff_snapshots
+from crochet.ir.hash import hash_snapshot
+from crochet.ir.schema import SchemaSnapshot
+from crochet.ledger.sqlite import Ledger
+from crochet.migrations.operations import MigrationContext
+from crochet.migrations.template import (
+    generate_revision_id,
+    render_migration,
+    write_migration_file,
+)
+
+
+class MigrationFile:
+    """Represents a loaded migration module."""
+
+    def __init__(self, module: Any, path: Path) -> None:
+        self.module = module
+        self.path = path
+        self.revision_id: str = module.revision_id
+        self.parent_id: str | None = module.parent_id
+        self.schema_hash: str = module.schema_hash
+        self.rollback_safe: bool = getattr(module, "rollback_safe", True)
+
+    def upgrade(self, ctx: MigrationContext) -> None:
+        self.module.upgrade(ctx)
+
+    def downgrade(self, ctx: MigrationContext) -> None:
+        self.module.downgrade(ctx)
+
+
+class MigrationEngine:
+    """Orchestrates creation, ordering, and execution of migrations."""
+
+    def __init__(self, config: CrochetConfig, ledger: Ledger) -> None:
+        self._config = config
+        self._ledger = ledger
+
+    # ------------------------------------------------------------------
+    # Discovery
+    # ------------------------------------------------------------------
+
+    def discover_migrations(self) -> list[MigrationFile]:
+        """Load all migration files from the migrations directory, ordered."""
+        migrations_dir = self._config.migrations_dir
+        if not migrations_dir.exists():
+            return []
+
+        files: list[MigrationFile] = []
+        for py_file in sorted(migrations_dir.glob("*.py")):
+            if py_file.name.startswith("_"):
+                continue
+            module = self._load_migration_module(py_file)
+            if module is None:
+                continue
+            if not hasattr(module, "revision_id"):
+                continue
+            files.append(MigrationFile(module, py_file))
+
+        return self._sort_by_chain(files)
+
+    def _load_migration_module(self, path: Path) -> Any:
+        mod_name = f"crochet._migrations.{path.stem}"
+        spec = importlib.util.spec_from_file_location(mod_name, path)
+        if spec is None or spec.loader is None:
+            return None
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[mod_name] = module
+        spec.loader.exec_module(module)
+        return module
+
+    def _sort_by_chain(self, migrations: list[MigrationFile]) -> list[MigrationFile]:
+        """Sort migrations by their parent chain (topological order)."""
+        by_id = {m.revision_id: m for m in migrations}
+        ordered: list[MigrationFile] = []
+        visited: set[str] = set()
+
+        # Find root(s)
+        roots = [m for m in migrations if m.parent_id is None]
+        if not roots and migrations:
+            # Fall back to filename sort
+            return sorted(migrations, key=lambda m: m.revision_id)
+
+        def walk(m: MigrationFile) -> None:
+            if m.revision_id in visited:
+                return
+            visited.add(m.revision_id)
+            if m.parent_id and m.parent_id in by_id:
+                walk(by_id[m.parent_id])
+            ordered.append(m)
+
+        for root in roots:
+            walk(root)
+
+        # Add any orphans not reached by the chain walk
+        for m in migrations:
+            if m.revision_id not in visited:
+                ordered.append(m)
+
+        return ordered
+
+    # ------------------------------------------------------------------
+    # Status
+    # ------------------------------------------------------------------
+
+    def pending_migrations(self) -> list[MigrationFile]:
+        """Return migrations that have not yet been applied."""
+        all_migrations = self.discover_migrations()
+        return [m for m in all_migrations if not self._ledger.is_applied(m.revision_id)]
+
+    def applied_migrations(self) -> list[MigrationFile]:
+        """Return migrations that have been applied (in order)."""
+        all_migrations = self.discover_migrations()
+        return [m for m in all_migrations if self._ledger.is_applied(m.revision_id)]
+
+    # ------------------------------------------------------------------
+    # Create
+    # ------------------------------------------------------------------
+
+    def create_migration(
+        self,
+        description: str,
+        current_snapshot: SchemaSnapshot | None = None,
+        rollback_safe: bool = True,
+    ) -> Path:
+        """Scaffold a new migration file.
+
+        If *current_snapshot* is provided, a diff against the previous
+        snapshot is computed and included as comments in the migration.
+        """
+        all_migrations = self.discover_migrations()
+        seq = len(all_migrations) + 1
+        parent_id: str | None = None
+        if all_migrations:
+            parent_id = all_migrations[-1].revision_id
+
+        revision_id = generate_revision_id(seq, description)
+
+        # Compute schema hash and diff
+        schema_hash = ""
+        diff_summary = ""
+        if current_snapshot is not None:
+            current_snapshot = hash_snapshot(current_snapshot)
+            schema_hash = current_snapshot.schema_hash
+
+            # Store the snapshot
+            self._ledger.store_snapshot(schema_hash, current_snapshot.to_json())
+
+            # Try to diff against the previous snapshot
+            if parent_id and all_migrations:
+                prev_hash = all_migrations[-1].schema_hash
+                prev_json = self._ledger.get_snapshot(prev_hash)
+                if prev_json:
+                    prev_snapshot = SchemaSnapshot.from_json(prev_json)
+                    diff = diff_snapshots(prev_snapshot, current_snapshot)
+                    if diff.has_changes:
+                        diff_summary = diff.summary()
+
+        content = render_migration(
+            revision_id=revision_id,
+            parent_id=parent_id,
+            description=description,
+            schema_hash=schema_hash,
+            rollback_safe=rollback_safe,
+            diff_summary=diff_summary,
+        )
+
+        return write_migration_file(
+            self._config.migrations_dir, revision_id, content
+        )
+
+    # ------------------------------------------------------------------
+    # Upgrade
+    # ------------------------------------------------------------------
+
+    def upgrade(
+        self,
+        target: str | None = None,
+        driver: Any | None = None,
+        dry_run: bool = False,
+    ) -> list[str]:
+        """Apply pending migrations up to *target* (or all).
+
+        Returns the list of applied revision IDs.
+        """
+        pending = self.pending_migrations()
+        if not pending:
+            return []
+
+        applied_ids: list[str] = []
+        for mf in pending:
+            if target and mf.revision_id == target:
+                self._apply_one(mf, driver, dry_run)
+                applied_ids.append(mf.revision_id)
+                break
+            self._apply_one(mf, driver, dry_run)
+            applied_ids.append(mf.revision_id)
+            if target and mf.revision_id == target:
+                break
+
+        return applied_ids
+
+    def _apply_one(
+        self, mf: MigrationFile, driver: Any | None, dry_run: bool
+    ) -> None:
+        ctx = MigrationContext(driver=driver, dry_run=dry_run)
+        try:
+            mf.upgrade(ctx)
+        except Exception as exc:
+            raise MigrationError(
+                f"Migration '{mf.revision_id}' failed during upgrade: {exc}"
+            ) from exc
+
+        if not dry_run:
+            self._ledger.record_migration(
+                revision_id=mf.revision_id,
+                parent_id=mf.parent_id,
+                description="",
+                schema_hash=mf.schema_hash,
+                rollback_safe=mf.rollback_safe,
+            )
+
+    # ------------------------------------------------------------------
+    # Downgrade
+    # ------------------------------------------------------------------
+
+    def downgrade(
+        self,
+        target: str | None = None,
+        driver: Any | None = None,
+        dry_run: bool = False,
+    ) -> list[str]:
+        """Revert applied migrations back to *target* (or one step).
+
+        Returns the list of reverted revision IDs.
+        """
+        applied = list(reversed(self.applied_migrations()))
+        if not applied:
+            return []
+
+        reverted_ids: list[str] = []
+        for mf in applied:
+            if target and mf.revision_id == target:
+                break
+            if not mf.rollback_safe:
+                raise RollbackUnsafeError(mf.revision_id)
+
+            ctx = MigrationContext(driver=driver, dry_run=dry_run)
+            try:
+                mf.downgrade(ctx)
+            except Exception as exc:
+                raise MigrationError(
+                    f"Migration '{mf.revision_id}' failed during downgrade: {exc}"
+                ) from exc
+
+            if not dry_run:
+                self._ledger.remove_migration(mf.revision_id)
+
+            reverted_ids.append(mf.revision_id)
+
+            # If no target, only revert one step
+            if target is None:
+                break
+
+        return reverted_ids