sqlspec 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

sqlspec/migrations/tracker.py

@@ -4,10 +4,14 @@ This module provides functionality to track applied migrations in the database.
 """
 
 import os
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any
 
+from rich.console import Console
+
+from sqlspec.builder import sql
 from sqlspec.migrations.base import BaseMigrationTracker
 from sqlspec.utils.logging import get_logger
+from sqlspec.utils.version import parse_version
 
 if TYPE_CHECKING:
     from sqlspec.driver import AsyncDriverAdapterBase, SyncDriverAdapterBase
@@ -20,16 +24,75 @@ logger = get_logger("migrations.tracker")
 class SyncMigrationTracker(BaseMigrationTracker["SyncDriverAdapterBase"]):
     """Synchronous migration version tracker."""
 
+    def _migrate_schema_if_needed(self, driver: "SyncDriverAdapterBase") -> None:
+        """Check for and add any missing columns to the tracking table.
+
+        Uses the adapter's data_dictionary to query existing columns,
+        then compares to the target schema and adds missing columns one by one.
+
+        Args:
+            driver: The database driver to use.
+        """
+        try:
+            columns_data = driver.data_dictionary.get_columns(driver, self.version_table)
+            if not columns_data:
+                logger.debug("Migration tracking table does not exist yet")
+                return
+
+            existing_columns = {col["column_name"] for col in columns_data}
+            missing_columns = self._detect_missing_columns(existing_columns)
+
+            if not missing_columns:
+                logger.debug("Migration tracking table schema is up-to-date")
+                return
+
+            console = Console()
+            console.print(
+                f"[cyan]Migrating tracking table schema, adding columns: {', '.join(sorted(missing_columns))}[/]"
+            )
+
+            for col_name in sorted(missing_columns):
+                self._add_column(driver, col_name)
+
+            driver.commit()
+            console.print("[green]Migration tracking table schema updated successfully[/]")
+
+        except Exception as e:
+            logger.warning("Could not check or migrate tracking table schema: %s", e)
+
+    def _add_column(self, driver: "SyncDriverAdapterBase", column_name: str) -> None:
+        """Add a single column to the tracking table.
+
+        Args:
+            driver: The database driver to use.
+            column_name: Name of the column to add (lowercase).
+        """
+        target_create = self._get_create_table_sql()
+        column_def = next((col for col in target_create.columns if col.name.lower() == column_name), None)
+
+        if not column_def:
+            return
+
+        alter_sql = sql.alter_table(self.version_table).add_column(
+            name=column_def.name, dtype=column_def.dtype, default=column_def.default, not_null=column_def.not_null
+        )
+        driver.execute(alter_sql)
+        logger.debug("Added column %s to tracking table", column_name)
+
     def ensure_tracking_table(self, driver: "SyncDriverAdapterBase") -> None:
         """Create the migration tracking table if it doesn't exist.
 
+        Also checks for and adds any missing columns to support schema migrations.
+
         Args:
             driver: The database driver to use.
         """
         driver.execute(self._get_create_table_sql())
         self._safe_commit(driver)
 
-    def get_current_version(self, driver: "SyncDriverAdapterBase") -> Optional[str]:
+        self._migrate_schema_if_needed(driver)
+
+    def get_current_version(self, driver: "SyncDriverAdapterBase") -> str | None:
         """Get the latest applied migration version.
 
         Args:
@@ -58,6 +121,9 @@ class SyncMigrationTracker(BaseMigrationTracker["SyncDriverAdapterBase"]):
     ) -> None:
         """Record a successfully applied migration.
 
+        Parses version to determine type (sequential or timestamp) and
+        auto-increments execution_sequence for application order tracking.
+
         Args:
             driver: The database driver to use.
             version: Version number of the migration.
@@ -65,9 +131,21 @@ class SyncMigrationTracker(BaseMigrationTracker["SyncDriverAdapterBase"]):
             execution_time_ms: Execution time in milliseconds.
             checksum: MD5 checksum of the migration content.
         """
+        parsed_version = parse_version(version)
+        version_type = parsed_version.type.value
+
+        result = driver.execute(self._get_next_execution_sequence_sql())
+        next_sequence = result.data[0]["next_seq"] if result.data else 1
+
         driver.execute(
             self._get_record_migration_sql(
-                version, description, execution_time_ms, checksum, os.environ.get("USER", "unknown")
+                version,
+                version_type,
+                next_sequence,
+                description,
+                execution_time_ms,
+                checksum,
+                os.environ.get("USER", "unknown"),
             )
         )
         self._safe_commit(driver)
@@ -82,21 +160,52 @@ class SyncMigrationTracker(BaseMigrationTracker["SyncDriverAdapterBase"]):
         driver.execute(self._get_remove_migration_sql(version))
         self._safe_commit(driver)
 
-    def _safe_commit(self, driver: "SyncDriverAdapterBase") -> None:
-        """Safely commit a transaction only if autocommit is disabled.
+    def update_version_record(self, driver: "SyncDriverAdapterBase", old_version: str, new_version: str) -> None:
+        """Update migration version record from timestamp to sequential.
+
+        Updates version_num and version_type while preserving execution_sequence,
+        applied_at, and other tracking metadata. Used during fix command.
+
+        Idempotent: If the version is already updated, logs and continues without error.
+        This allows fix command to be safely re-run after pulling changes.
 
         Args:
             driver: The database driver to use.
+            old_version: Current timestamp version string.
+            new_version: New sequential version string.
+
+        Raises:
+            ValueError: If neither old_version nor new_version found in database.
         """
-        try:
-            connection = getattr(driver, "connection", None)
-            if connection and hasattr(connection, "autocommit") and getattr(connection, "autocommit", False):
-                return
+        parsed_new_version = parse_version(new_version)
+        new_version_type = parsed_new_version.type.value
+
+        result = driver.execute(self._get_update_version_sql(old_version, new_version, new_version_type))
 
-            driver_features = getattr(driver, "driver_features", {})
-            if driver_features and driver_features.get("autocommit", False):
+        if result.rows_affected == 0:
+            check_result = driver.execute(self._get_applied_migrations_sql())
+            applied_versions = {row["version_num"] for row in check_result.data} if check_result.data else set()
+
+            if new_version in applied_versions:
+                logger.debug("Version already updated: %s -> %s", old_version, new_version)
                 return
 
+            msg = f"Migration version {old_version} not found in database"
+            raise ValueError(msg)
+
+        self._safe_commit(driver)
+        logger.debug("Updated version record: %s -> %s", old_version, new_version)
+
+    def _safe_commit(self, driver: "SyncDriverAdapterBase") -> None:
+        """Safely commit a transaction only if autocommit is disabled.
+
+        Args:
+            driver: The database driver to use.
+        """
+        if driver.driver_features.get("autocommit", False):
+            return
+
+        try:
             driver.commit()
         except Exception:
             logger.debug("Failed to commit transaction, likely due to autocommit being enabled")
@@ -105,16 +214,77 @@ class SyncMigrationTracker(BaseMigrationTracker["SyncDriverAdapterBase"]):
 class AsyncMigrationTracker(BaseMigrationTracker["AsyncDriverAdapterBase"]):
     """Asynchronous migration version tracker."""
 
+    async def _migrate_schema_if_needed(self, driver: "AsyncDriverAdapterBase") -> None:
+        """Check for and add any missing columns to the tracking table.
+
+        Uses the driver's data_dictionary to query existing columns,
+        then compares to the target schema and adds missing columns one by one.
+
+        Args:
+            driver: The database driver to use.
+        """
+        try:
+            columns_data = await driver.data_dictionary.get_columns(driver, self.version_table)
+            if not columns_data:
+                logger.debug("Migration tracking table does not exist yet")
+                return
+
+            existing_columns = {col["column_name"] for col in columns_data}
+            missing_columns = self._detect_missing_columns(existing_columns)
+
+            if not missing_columns:
+                logger.debug("Migration tracking table schema is up-to-date")
+                return
+
+            from rich.console import Console
+
+            console = Console()
+            console.print(
+                f"[cyan]Migrating tracking table schema, adding columns: {', '.join(sorted(missing_columns))}[/]"
+            )
+
+            for col_name in sorted(missing_columns):
+                await self._add_column(driver, col_name)
+
+            await driver.commit()
+            console.print("[green]Migration tracking table schema updated successfully[/]")
+
+        except Exception as e:
+            logger.warning("Could not check or migrate tracking table schema: %s", e)
+
+    async def _add_column(self, driver: "AsyncDriverAdapterBase", column_name: str) -> None:
+        """Add a single column to the tracking table.
+
+        Args:
+            driver: The database driver to use.
+            column_name: Name of the column to add (lowercase).
+        """
+        target_create = self._get_create_table_sql()
+        column_def = next((col for col in target_create.columns if col.name.lower() == column_name), None)
+
+        if not column_def:
+            return
+
+        alter_sql = sql.alter_table(self.version_table).add_column(
+            name=column_def.name, dtype=column_def.dtype, default=column_def.default, not_null=column_def.not_null
+        )
+        await driver.execute(alter_sql)
+        logger.debug("Added column %s to tracking table", column_name)
+
     async def ensure_tracking_table(self, driver: "AsyncDriverAdapterBase") -> None:
         """Create the migration tracking table if it doesn't exist.
 
+        Also checks for and adds any missing columns to support schema migrations.
+
         Args:
             driver: The database driver to use.
         """
         await driver.execute(self._get_create_table_sql())
         await self._safe_commit_async(driver)
 
-    async def get_current_version(self, driver: "AsyncDriverAdapterBase") -> Optional[str]:
+        await self._migrate_schema_if_needed(driver)
+
+    async def get_current_version(self, driver: "AsyncDriverAdapterBase") -> str | None:
         """Get the latest applied migration version.
 
         Args:
@@ -143,6 +313,9 @@ class AsyncMigrationTracker(BaseMigrationTracker["AsyncDriverAdapterBase"]):
     ) -> None:
         """Record a successfully applied migration.
 
+        Parses version to determine type (sequential or timestamp) and
+        auto-increments execution_sequence for application order tracking.
+
         Args:
             driver: The database driver to use.
             version: Version number of the migration.
@@ -150,9 +323,21 @@ class AsyncMigrationTracker(BaseMigrationTracker["AsyncDriverAdapterBase"]):
             execution_time_ms: Execution time in milliseconds.
             checksum: MD5 checksum of the migration content.
         """
+        parsed_version = parse_version(version)
+        version_type = parsed_version.type.value
+
+        result = await driver.execute(self._get_next_execution_sequence_sql())
+        next_sequence = result.data[0]["next_seq"] if result.data else 1
+
         await driver.execute(
             self._get_record_migration_sql(
-                version, description, execution_time_ms, checksum, os.environ.get("USER", "unknown")
+                version,
+                version_type,
+                next_sequence,
+                description,
+                execution_time_ms,
+                checksum,
+                os.environ.get("USER", "unknown"),
             )
         )
         await self._safe_commit_async(driver)
@@ -167,21 +352,52 @@ class AsyncMigrationTracker(BaseMigrationTracker["AsyncDriverAdapterBase"]):
         await driver.execute(self._get_remove_migration_sql(version))
         await self._safe_commit_async(driver)
 
-    async def _safe_commit_async(self, driver: "AsyncDriverAdapterBase") -> None:
-        """Safely commit a transaction only if autocommit is disabled.
+    async def update_version_record(self, driver: "AsyncDriverAdapterBase", old_version: str, new_version: str) -> None:
+        """Update migration version record from timestamp to sequential.
+
+        Updates version_num and version_type while preserving execution_sequence,
+        applied_at, and other tracking metadata. Used during fix command.
+
+        Idempotent: If the version is already updated, logs and continues without error.
+        This allows fix command to be safely re-run after pulling changes.
 
         Args:
             driver: The database driver to use.
+            old_version: Current timestamp version string.
+            new_version: New sequential version string.
+
+        Raises:
+            ValueError: If neither old_version nor new_version found in database.
         """
-        try:
-            connection = getattr(driver, "connection", None)
-            if connection and hasattr(connection, "autocommit") and getattr(connection, "autocommit", False):
-                return
+        parsed_new_version = parse_version(new_version)
+        new_version_type = parsed_new_version.type.value
+
+        result = await driver.execute(self._get_update_version_sql(old_version, new_version, new_version_type))
 
-            driver_features = getattr(driver, "driver_features", {})
-            if driver_features and driver_features.get("autocommit", False):
+        if result.rows_affected == 0:
+            check_result = await driver.execute(self._get_applied_migrations_sql())
+            applied_versions = {row["version_num"] for row in check_result.data} if check_result.data else set()
+
+            if new_version in applied_versions:
+                logger.debug("Version already updated: %s -> %s", old_version, new_version)
                 return
 
+            msg = f"Migration version {old_version} not found in database"
+            raise ValueError(msg)
+
+        await self._safe_commit_async(driver)
+        logger.debug("Updated version record: %s -> %s", old_version, new_version)
+
+    async def _safe_commit_async(self, driver: "AsyncDriverAdapterBase") -> None:
+        """Safely commit a transaction only if autocommit is disabled.
+
+        Args:
+            driver: The database driver to use.
+        """
+        if driver.driver_features.get("autocommit", False):
+            return
+
+        try:
             await driver.commit()
         except Exception:
             logger.debug("Failed to commit transaction, likely due to autocommit being enabled")

sqlspec/migrations/utils.py

@@ -3,16 +3,20 @@
 This module provides helper functions for migration operations.
 """
 
+import logging
 import os
+import subprocess
 from datetime import datetime, timezone
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from sqlspec.driver import AsyncDriverAdapterBase
 
 __all__ = ("create_migration_file", "drop_all", "get_author")
 
+logger = logging.getLogger(__name__)
+
 
 def create_migration_file(migrations_dir: Path, version: str, message: str, file_type: str = "sql") -> Path:
     """Create a new migration file from template.
@@ -108,13 +112,57 @@ DROP TABLE placeholder;
 def get_author() -> str:
     """Get current user for migration metadata.
 
+    Attempts to retrieve git user configuration (name and email).
+    Falls back to system username if git is not configured or unavailable.
+
+    Returns:
+        Author string in format 'Name <email>' if git configured,
+        otherwise system username from environment.
+    """
+    git_name = _get_git_config("user.name")
+    git_email = _get_git_config("user.email")
+
+    if git_name and git_email:
+        return f"{git_name} <{git_email}>"
+
+    return _get_system_username()
+
+
+def _get_git_config(config_key: str) -> str | None:
+    """Retrieve git configuration value.
+
+    Args:
+        config_key: Git config key (e.g., 'user.name', 'user.email').
+
+    Returns:
+        Configuration value if found, None otherwise.
+    """
+    try:
+        result = subprocess.run(  # noqa: S603
+            ["git", "config", config_key],  # noqa: S607
+            capture_output=True,
+            text=True,
+            timeout=2,
+            check=False,
+        )
+        if result.returncode == 0 and result.stdout.strip():
+            return result.stdout.strip()
+    except (subprocess.SubprocessError, FileNotFoundError, OSError) as e:
+        logger.debug("Failed to get git config %s: %s", config_key, e)
+
+    return None
+
+
+def _get_system_username() -> str:
+    """Get system username from environment.
+
     Returns:
-        Username from environment or 'unknown'.
+        Username from USER environment variable, or 'unknown' if not set.
     """
     return os.environ.get("USER", "unknown")
 
 
-async def drop_all(engine: "AsyncDriverAdapterBase", version_table_name: str, metadata: Optional[Any] = None) -> None:
+async def drop_all(engine: "AsyncDriverAdapterBase", version_table_name: str, metadata: Any | None = None) -> None:
     """Drop all tables from the database.
 
     Args:

sqlspec/migrations/validation.py

@@ -0,0 +1,177 @@
+"""Migration validation and out-of-order detection for SQLSpec.
+
+This module provides functionality to detect and handle out-of-order migrations,
+which can occur when branches with migrations merge in different orders across
+staging and production environments.
+"""
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+
+from sqlspec.exceptions import OutOfOrderMigrationError
+from sqlspec.utils.version import parse_version
+
+if TYPE_CHECKING:
+    from sqlspec.utils.version import MigrationVersion
+
+__all__ = ("MigrationGap", "detect_out_of_order_migrations", "format_out_of_order_warning")
+
+console = Console()
+
+
+@dataclass(frozen=True)
+class MigrationGap:
+    """Represents a migration that is out of order.
+
+    An out-of-order migration occurs when a pending migration has a timestamp
+    earlier than already-applied migrations, indicating it was created in a branch
+    that merged after other migrations were already applied.
+
+    Attributes:
+        missing_version: The out-of-order migration version.
+        applied_after: List of already-applied migrations with later timestamps.
+    """
+
+    missing_version: "MigrationVersion"
+    applied_after: "list[MigrationVersion]"
+
+
+def detect_out_of_order_migrations(
+    pending_versions: "list[str]", applied_versions: "list[str]"
+) -> "list[MigrationGap]":
+    """Detect migrations created before already-applied migrations.
+
+    Identifies pending migrations with timestamps earlier than the latest applied
+    migration, which indicates they were created in branches that merged late or
+    were cherry-picked across environments.
+
+    Extension migrations are excluded from out-of-order detection as they maintain
+    independent sequences within their own namespaces.
+
+    Args:
+        pending_versions: List of migration versions not yet applied.
+        applied_versions: List of migration versions already applied.
+
+    Returns:
+        List of migration gaps representing out-of-order migrations.
+        Empty list if no out-of-order migrations detected.
+
+    Example:
+        Applied: [20251011120000, 20251012140000]
+        Pending: [20251011130000, 20251013090000]
+        Result: Gap for 20251011130000 (created between applied migrations)
+
+        Applied: [ext_litestar_0001, 0001, 0002]
+        Pending: [ext_adk_0001]
+        Result: [] (extensions excluded from out-of-order detection)
+    """
+    if not applied_versions or not pending_versions:
+        return []
+
+    gaps: list[MigrationGap] = []
+
+    parsed_applied = [parse_version(v) for v in applied_versions]
+    parsed_pending = [parse_version(v) for v in pending_versions]
+
+    core_applied = [v for v in parsed_applied if v.extension is None]
+    core_pending = [v for v in parsed_pending if v.extension is None]
+
+    if not core_applied or not core_pending:
+        return []
+
+    latest_applied = max(core_applied)
+
+    for pending in core_pending:
+        if pending < latest_applied:
+            applied_after = [a for a in core_applied if a > pending]
+            if applied_after:
+                gaps.append(MigrationGap(missing_version=pending, applied_after=applied_after))
+
+    return gaps
+
+
+def format_out_of_order_warning(gaps: "list[MigrationGap]") -> str:
+    """Create user-friendly warning message for out-of-order migrations.
+
+    Formats migration gaps into a clear warning message explaining which migrations
+    are out of order and what migrations were already applied after them.
+
+    Args:
+        gaps: List of migration gaps to format.
+
+    Returns:
+        Formatted warning message string.
+
+    Example:
+        >>> gaps = [MigrationGap(version1, [version2, version3])]
+        >>> print(format_out_of_order_warning(gaps))
+        Out-of-order migrations detected:
+
+        - 20251011130000 created before:
+          - 20251012140000
+          - 20251013090000
+    """
+    if not gaps:
+        return ""
+
+    lines = ["Out-of-order migrations detected:", ""]
+
+    for gap in gaps:
+        lines.append(f"- {gap.missing_version.raw} created before:")
+        lines.extend(f"  - {applied.raw}" for applied in gap.applied_after)
+        lines.append("")
+
+    lines.extend(
+        (
+            "These migrations will be applied but may cause issues if they",
+            "depend on schema changes from later migrations.",
+            "",
+            "To prevent this in the future, ensure migrations are merged in",
+            "chronological order or use strict_ordering mode in migration_config.",
+        )
+    )
+
+    return "\n".join(lines)
+
+
+def validate_migration_order(
+    pending_versions: "list[str]", applied_versions: "list[str]", strict_ordering: bool = False
+) -> None:
+    """Validate migration order and raise error if out-of-order in strict mode.
+
+    Checks for out-of-order migrations and either warns or raises an error
+    depending on the strict_ordering configuration.
+
+    Args:
+        pending_versions: List of migration versions not yet applied.
+        applied_versions: List of migration versions already applied.
+        strict_ordering: If True, raise error for out-of-order migrations.
+            If False (default), log warning but allow.
+
+    Raises:
+        OutOfOrderMigrationError: If out-of-order migrations detected and
+            strict_ordering is True.
+
+    Example:
+        >>> validate_migration_order(
+        ...     ["20251011130000"],
+        ...     ["20251012140000"],
+        ...     strict_ordering=True,
+        ... )
+        OutOfOrderMigrationError: Out-of-order migrations detected...
+    """
+    gaps = detect_out_of_order_migrations(pending_versions, applied_versions)
+
+    if not gaps:
+        return
+
+    warning_message = format_out_of_order_warning(gaps)
+
+    if strict_ordering:
+        msg = f"{warning_message}\n\nStrict ordering is enabled. Use --allow-missing to override."
+        raise OutOfOrderMigrationError(msg)
+
+    console.print("[yellow]Out-of-order migrations detected[/]")
+    console.print(f"[yellow]{warning_message}[/]")