jetbase 0.7.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

jetbase/engine/formatters.py

@@ -0,0 +1,61 @@
+import datetime as dt
+
+
+def get_display_version(
+    migration_type: str,
+    version: str | None = None,
+) -> str:
+    """
+    Get the display string for a migration version.
+
+    Returns the version string for versioned migrations, or a label
+    for repeatable migrations.
+
+    Args:
+        migration_type (str): The type of migration ('runs_always',
+            'runs_on_change', or 'versioned').
+        version (str | None): The version string for versioned migrations.
+            Defaults to None.
+
+    Returns:
+        str: The version string if provided, "RUNS_ALWAYS" for runs_always
+            migrations, or "RUNS_ON_CHANGE" for runs_on_change migrations.
+
+    Raises:
+        ValueError: If migration_type is invalid and version is None.
+
+    Example:
+        >>> get_display_version("runs_always")
+        'RUNS_ALWAYS'
+    """
+
+    if version:
+        return version
+    if migration_type.lower() == "runs_always":
+        return "RUNS_ALWAYS"
+    elif migration_type.lower() == "runs_on_change":
+        return "RUNS_ON_CHANGE"
+    raise ValueError("Invalid migration type for display version.")
+
+
+def format_applied_at(applied_at: dt.datetime | str | None) -> str:
+    """
+    Format a timestamp for display in migration history.
+
+    Handles both datetime objects (PostgreSQL) and string timestamps
+    (SQLite) by truncating to a consistent 22-character format.
+
+    Args:
+        applied_at (dt.datetime | str | None): The timestamp to format.
+            Can be a datetime object, string, or None.
+
+    Returns:
+        str: Formatted timestamp string truncated to 22 characters,
+            or empty string if input is None.
+    """
+    if applied_at is None:
+        return ""
+    if isinstance(applied_at, str):
+        # SQLite returns strings - just truncate to match format
+        return applied_at[:22]
+    return applied_at.strftime("%Y-%m-%d %H:%M:%S.%f")[:22]

jetbase/engine/lock.py

@@ -0,0 +1,65 @@
+import uuid
+from contextlib import contextmanager
+from typing import Generator
+
+from sqlalchemy.engine import CursorResult
+
+from jetbase.repositories.lock_repo import lock_database, release_lock
+
+
+def acquire_lock() -> str:
+    """
+    Acquire the migration lock immediately.
+
+    Attempts to acquire the database migration lock using a unique process ID.
+    The lock prevents concurrent migrations from running.
+
+    Returns:
+        str: Unique UUID process identifier for this lock acquisition.
+
+    Raises:
+        RuntimeError: If the lock is already held by another process.
+    """
+    process_id = str(uuid.uuid4())
+
+    result: CursorResult = lock_database(process_id=process_id)
+
+    if result.rowcount == 0:  # already locked
+        raise RuntimeError(
+            "Migration lock is already held by another process.\n\n"
+            "If you are completely sure that no other migrations are running, "
+            "you can unlock using:\n"
+            "  jetbase unlock\n\n"
+            "WARNING: Unlocking then running a migration while another migration process is running may "
+            "lead to database corruption."
+        )
+
+    return process_id
+
+
+@contextmanager
+def migration_lock() -> Generator[None, None, None]:
+    """
+    Context manager for acquiring and releasing the migration lock.
+
+    Acquires the lock on entry and ensures it is released on exit,
+    even if an exception occurs. Fails immediately if the lock is
+    already held by another process.
+
+    Yields:
+        None: Yields control to the context block.
+
+    Raises:
+        RuntimeError: If the lock is already held by another process.
+
+    Example:
+        >>> with migration_lock():
+        ...     run_migration()
+    """
+    process_id: str | None = None
+    try:
+        process_id = acquire_lock()
+        yield
+    finally:
+        if process_id:
+            release_lock(process_id=process_id)

jetbase/engine/repeatable.py

@@ -0,0 +1,125 @@
+import os
+
+from jetbase.constants import RUNS_ALWAYS_FILE_PREFIX, RUNS_ON_CHANGE_FILE_PREFIX
+from jetbase.engine.checksum import calculate_checksum
+from jetbase.engine.file_parser import (
+    parse_upgrade_statements,
+    validate_filename_format,
+)
+from jetbase.repositories.migrations_repo import (
+    get_existing_on_change_filenames_to_checksums,
+)
+
+
+def get_repeatable_always_filepaths(directory: str) -> list[str]:
+    """
+    Get file paths for all runs-always (RA__) migrations in a directory.
+
+    Scans the directory for migration files starting with the RA__ prefix
+    and validates their filename format.
+
+    Args:
+        directory (str): Path to the migrations directory to scan.
+
+    Returns:
+        list[str]: Sorted list of absolute file paths for RA__ migrations.
+
+    Raises:
+        InvalidMigrationFilenameError: If any file has an invalid format.
+        MigrationFilenameTooLongError: If any filename exceeds 512 characters.
+    """
+    repeatable_always_filepaths: list[str] = []
+    for root, _, files in os.walk(directory):
+        for filename in files:
+            validate_filename_format(filename=filename)
+            if filename.startswith(RUNS_ALWAYS_FILE_PREFIX):
+                filepath: str = os.path.join(root, filename)
+                repeatable_always_filepaths.append(filepath)
+
+    repeatable_always_filepaths.sort()
+    return repeatable_always_filepaths
+
+
+def get_runs_on_change_filepaths(
+    directory: str, changed_only: bool = False
+) -> list[str]:
+    """
+    Get file paths for runs-on-change (ROC__) migrations in a directory.
+
+    Scans the directory for migration files starting with the ROC__ prefix.
+    Optionally filters to only include files whose checksums have changed
+    since they were last applied.
+
+    Args:
+        directory (str): Path to the migrations directory to scan.
+        changed_only (bool): If True, only returns files that have been
+            modified since last migration. Defaults to False.
+
+    Returns:
+        list[str]: Sorted list of absolute file paths for ROC__ migrations.
+
+    Raises:
+        InvalidMigrationFilenameError: If any file has an invalid format.
+        MigrationFilenameTooLongError: If any filename exceeds 512 characters.
+    """
+    runs_on_change_filepaths: list[str] = []
+    for root, _, files in os.walk(directory):
+        for filename in files:
+            validate_filename_format(filename=filename)
+            if filename.startswith(RUNS_ON_CHANGE_FILE_PREFIX):
+                filepath: str = os.path.join(root, filename)
+                runs_on_change_filepaths.append(filepath)
+
+    if runs_on_change_filepaths and changed_only:
+        existing_on_change_migrations: dict[str, str] = (
+            get_existing_on_change_filenames_to_checksums()
+        )
+
+        for filepath in runs_on_change_filepaths.copy():
+            filename: str = os.path.basename(filepath)
+            sql_statements: list[str] = parse_upgrade_statements(file_path=filepath)
+            checksum: str = calculate_checksum(sql_statements=sql_statements)
+
+            if existing_on_change_migrations.get(filename) == checksum:
+                runs_on_change_filepaths.remove(filepath)
+
+    runs_on_change_filepaths.sort()
+    return runs_on_change_filepaths
+
+
+def get_ra_filenames() -> list[str]:
+    """
+    Get all runs-always (RA__) migration filenames from the migrations directory.
+
+    Scans the 'migrations' subdirectory in the current working directory
+    for files starting with the RA__ prefix.
+
+    Returns:
+        list[str]: List of RA__ migration filenames (not full paths).
+    """
+    ra_filenames: list[str] = []
+    for root, _, files in os.walk(os.path.join(os.getcwd(), "migrations")):
+        for filename in files:
+            if filename.startswith(RUNS_ALWAYS_FILE_PREFIX):
+                ra_filenames.append(filename)
+    return ra_filenames
+
+
+def get_repeatable_filenames() -> list[str]:
+    """
+    Get all repeatable migration filenames from the migrations directory.
+
+    Scans the 'migrations' subdirectory in the current working directory
+    for files starting with either RA__ or ROC__ prefix.
+
+    Returns:
+        list[str]: List of all repeatable migration filenames (not full paths).
+    """
+    repeatable_filenames: list[str] = []
+    for root, _, files in os.walk(os.path.join(os.getcwd(), "migrations")):
+        for filename in files:
+            if filename.startswith(RUNS_ALWAYS_FILE_PREFIX) or filename.startswith(
+                RUNS_ON_CHANGE_FILE_PREFIX
+            ):
+                repeatable_filenames.append(filename)
+    return repeatable_filenames

jetbase/engine/validation.py

@@ -0,0 +1,238 @@
+import os
+
+from packaging.version import parse as parse_version
+
+from jetbase.config import get_config
+from jetbase.constants import MIGRATIONS_DIR
+from jetbase.engine.checksum import calculate_checksum
+from jetbase.engine.file_parser import parse_upgrade_statements
+from jetbase.engine.repeatable import get_repeatable_filenames
+from jetbase.engine.version import (
+    get_migration_filepaths_by_version,
+)
+from jetbase.exceptions import (
+    ChecksumMismatchError,
+    OutOfOrderMigrationError,
+)
+from jetbase.repositories.migrations_repo import (
+    fetch_repeatable_migrations,
+    get_checksums_by_version,
+    get_migrated_versions,
+)
+
+
+def validate_current_migration_files_match_checksums(
+    migrated_filepaths_by_version: dict[str, str],
+    migrated_versions_and_checksums: list[tuple[str, str]],
+) -> None:
+    """
+    Validate that migration files have not been modified since being applied.
+
+    Compares the current checksum of each migration file against the
+    checksum stored in the database when the migration was applied.
+
+    Args:
+        migrated_filepaths_by_version (dict[str, str]): Mapping of version
+            strings to file paths for migrations to check.
+        migrated_versions_and_checksums (list[tuple[str, str]]): List of
+            (version, checksum) tuples from the database.
+
+    Returns:
+        None: Returns silently if all checksums match.
+
+    Raises:
+        ChecksumMismatchError: If any migration file's current checksum
+            differs from its stored checksum.
+    """
+    versions_changed: list[str] = []
+    for index, (file_version, filepath) in enumerate(
+        migrated_filepaths_by_version.items()
+    ):
+        sql_statements: list[str] = parse_upgrade_statements(file_path=filepath)
+        checksum: str = calculate_checksum(sql_statements=sql_statements)
+
+        for migrated_version, migrated_checksum in migrated_versions_and_checksums:
+            if file_version == migrated_version:
+                if checksum != migrated_checksum:
+                    versions_changed.append(file_version)
+
+        if versions_changed:
+            raise ChecksumMismatchError(
+                f"Checksum mismatch for versions: {', '.join(versions_changed)}. Files have been changed since migration."
+            )
+
+
+def validate_migrated_versions_in_current_migration_files(
+    migrated_versions: list[str],
+    current_migration_filepaths_by_version: dict[str, str],
+) -> None:
+    """
+    Ensure all migrated versions have corresponding migration files.
+
+    Verifies that every version recorded in the database still has
+    its migration file present in the migrations directory.
+
+    Args:
+        migrated_versions (list[str]): List of version strings that have
+            been applied to the database.
+        current_migration_filepaths_by_version (dict[str, str]): Mapping
+            of version strings to file paths for existing files.
+
+    Returns:
+        None: Returns silently if all files are present.
+
+    Raises:
+        FileNotFoundError: If any migrated version is missing its file.
+    """
+    for migrated_version in migrated_versions:
+        if migrated_version not in current_migration_filepaths_by_version:
+            raise FileNotFoundError(
+                f"Version {migrated_version} has been migrated but is missing from the current migration files."
+            )
+
+
+def validate_no_new_migration_files_with_lower_version_than_latest_migration(
+    current_migration_filepaths_by_version: dict[str, str],
+    migrated_versions: list[str],
+    latest_migrated_version: str,
+) -> None:
+    """
+    Ensure no new migration files have versions lower than the latest applied.
+
+    Prevents out-of-order migrations by checking that all new migration
+    files have version numbers higher than the most recently applied version.
+
+    Args:
+        current_migration_filepaths_by_version (dict[str, str]): Mapping
+            of version strings to file paths for all migration files.
+        migrated_versions (list[str]): List of versions already applied.
+        latest_migrated_version (str): The most recently applied version.
+
+    Returns:
+        None: Returns silently if validation passes.
+
+    Raises:
+        OutOfOrderMigrationError: If a new migration file has a version
+            lower than the latest migrated version.
+    """
+    for file_version, filepath in current_migration_filepaths_by_version.items():
+        if (
+            parse_version(file_version) < parse_version(latest_migrated_version)
+            and file_version not in migrated_versions
+        ):
+            filename: str = os.path.basename(filepath)
+            raise OutOfOrderMigrationError(
+                f"{filename} has version {file_version} which is lower than the latest migrated version {latest_migrated_version}.\n"
+                "New migration files cannot have versions lower than the latest migrated version.\n"
+                f"Please rename the file to have a version higher than {latest_migrated_version}.\n"
+            )
+
+
+def validate_migrated_repeatable_versions_in_migration_files(
+    migrated_repeatable_filenames: list[str],
+    all_repeatable_filenames: list[str],
+) -> None:
+    """
+    Ensure all migrated repeatable migrations have corresponding files.
+
+    Verifies that every repeatable migration recorded in the database
+    still has its file present in the migrations directory.
+
+    Args:
+        migrated_repeatable_filenames (list[str]): List of repeatable
+            migration filenames from the database.
+        all_repeatable_filenames (list[str]): List of all repeatable
+            migration filenames currently in the migrations directory.
+
+    Returns:
+        None: Returns silently if all files are present.
+
+    Raises:
+        FileNotFoundError: If any migrated repeatable file is missing.
+    """
+    missing_filenames: list[str] = []
+    for r_file in migrated_repeatable_filenames:
+        if r_file not in all_repeatable_filenames:
+            missing_filenames.append(r_file)
+    if missing_filenames:
+        raise FileNotFoundError(
+            f"The following migrated repeatable files are missing: {', '.join(missing_filenames)}"
+        )
+
+
+def run_migration_validations(
+    latest_migrated_version: str,
+    skip_validation: bool = False,
+    skip_checksum_validation: bool = False,
+    skip_file_validation: bool = False,
+) -> None:
+    """
+    Run all migration validations before performing an upgrade.
+
+    Executes validation checks including duplicate version detection,
+    file presence verification, out-of-order detection, and checksum
+    validation. Validations can be skipped via parameters or config.
+
+    Args:
+        latest_migrated_version (str): The most recently applied version,
+            used to check for out-of-order migrations.
+        skip_validation (bool): If True, skips all validations except
+            duplicate version check. Defaults to False.
+        skip_checksum_validation (bool): If True, skips checksum validation.
+            Defaults to False.
+        skip_file_validation (bool): If True, skips file presence and
+            out-of-order validations. Defaults to False.
+
+    Returns:
+        None: Returns silently if all validations pass.
+
+    Raises:
+        DuplicateMigrationVersionError: If duplicate versions are found.
+        FileNotFoundError: If migration files are missing.
+        OutOfOrderMigrationError: If out-of-order migrations are detected.
+        ChecksumMismatchError: If file checksums don't match stored values.
+    """
+
+    skip_validation_config: bool = get_config().skip_validation
+    skip_checksum_validation_config: bool = get_config().skip_checksum_validation
+    skip_file_validation_config: bool = get_config().skip_file_validation
+
+    migrations_directory_path: str = os.path.join(os.getcwd(), MIGRATIONS_DIR)
+
+    migration_filepaths_by_version: dict[str, str] = get_migration_filepaths_by_version(
+        directory=migrations_directory_path
+    )
+
+    if not skip_validation and not skip_validation_config:
+        if not skip_file_validation and not skip_file_validation_config:
+            migrated_versions: list[str] = get_migrated_versions()
+
+            validate_no_new_migration_files_with_lower_version_than_latest_migration(
+                current_migration_filepaths_by_version=migration_filepaths_by_version,
+                migrated_versions=migrated_versions,
+                latest_migrated_version=latest_migrated_version,
+            )
+
+            validate_migrated_versions_in_current_migration_files(
+                migrated_versions=migrated_versions,
+                current_migration_filepaths_by_version=migration_filepaths_by_version,
+            )
+
+            validate_migrated_repeatable_versions_in_migration_files(
+                migrated_repeatable_filenames=[
+                    r.filename for r in fetch_repeatable_migrations()
+                ],
+                all_repeatable_filenames=get_repeatable_filenames(),
+            )
+
+        migrated_filepaths_by_version: dict[str, str] = (
+            get_migration_filepaths_by_version(
+                directory=migrations_directory_path, end_version=latest_migrated_version
+            )
+        )
+
+        if not skip_checksum_validation and not skip_checksum_validation_config:
+            validate_current_migration_files_match_checksums(
+                migrated_filepaths_by_version=migrated_filepaths_by_version,
+                migrated_versions_and_checksums=get_checksums_by_version(),
+            )

jetbase/engine/version.py

@@ -0,0 +1,144 @@
+import os
+
+from packaging.version import parse as parse_version
+
+from jetbase.constants import (
+    VERSION_FILE_PREFIX,
+)
+from jetbase.engine.file_parser import (
+    is_filename_format_valid,
+    is_filename_length_valid,
+)
+from jetbase.exceptions import (
+    DuplicateMigrationVersionError,
+    InvalidMigrationFilenameError,
+    MigrationFilenameTooLongError,
+)
+
+
+def _get_version_key_from_filename(filename: str) -> str:
+    """
+    Extract and normalize the version key from a migration filename.
+
+    Parses the version portion between 'V' and '__', then normalizes
+    underscores to periods for consistent version comparison.
+
+    Args:
+        filename (str): The migration filename to parse.
+
+    Returns:
+        str: Normalized version string with periods as separators.
+
+    Raises:
+        ValueError: If the filename doesn't follow the expected format.
+
+    Example:
+        >>> _get_version_key_from_filename("V1_2__desc.sql")
+        '1.2'
+    """
+    try:
+        version = filename.split("__")[0][1:]
+    except Exception:
+        raise (
+            ValueError(
+                "Filename must be in the following format: V1__my_description.sql, V1_1__my_description.sql, V1.1__my_description.sql"
+            )
+        )
+    return version.replace("_", ".")
+
+
+def get_migration_filepaths_by_version(
+    directory: str,
+    version_to_start_from: str | None = None,
+    end_version: str | None = None,
+) -> dict[str, str]:
+    """
+    Get versioned migration file paths sorted by version number.
+
+    Scans the directory for SQL migration files, validates their format,
+    and returns a dictionary mapping version strings to file paths.
+    Results can be filtered by version range.
+
+    Args:
+        directory (str): Path to the migrations directory to scan.
+        version_to_start_from (str | None): Minimum version to include
+            (inclusive). If provided, only files with versions >= this
+            are returned. Defaults to None.
+        end_version (str | None): Maximum version to include (inclusive).
+            If provided, only files with versions <= this are returned.
+            Defaults to None.
+
+    Returns:
+        dict[str, str]: Dictionary mapping normalized version strings
+            to their absolute file paths, sorted by version number.
+
+    Raises:
+        InvalidMigrationFilenameError: If a file has an invalid format.
+        MigrationFilenameTooLongError: If a filename exceeds 512 characters.
+        DuplicateMigrationVersionError: If duplicate versions are detected.
+
+    Example:
+        >>> get_migration_filepaths_by_version('/migrations')
+        {'1.0': '/migrations/V1__init.sql', '1.1': '/migrations/V1_1__add.sql'}
+    """
+    version_to_filepath_dict: dict[str, str] = {}
+    seen_versions: set[str] = set()
+
+    for root, _, files in os.walk(directory):
+        for filename in files:
+            if filename.endswith(".sql") and not is_filename_format_valid(
+                filename=filename
+            ):
+                raise InvalidMigrationFilenameError(
+                    f"Invalid migration filename format: {filename}.\n"
+                    "Filenames must start with 'V', followed by the version number, "
+                    "two underscores '__', a description, and end with '.sql'.\n"
+                    "V<version_number>__<my_description>.sql. "
+                    "Examples: 'V1_2_0__add_new_table.sql' or 'V1.2.0__add_new_table.sql'\n"
+                )
+
+            if filename.endswith(".sql") and not is_filename_length_valid(
+                filename=filename
+            ):
+                raise MigrationFilenameTooLongError(
+                    f"Migration filename too long: {filename}.\n"
+                    f"Filename is currently {len(filename)} characters.\n"
+                    "Filenames must not exceed 512 characters."
+                )
+
+            if is_filename_format_valid(filename=filename):
+                if filename.startswith(VERSION_FILE_PREFIX):
+                    file_path: str = os.path.join(root, filename)
+                    file_version: str = _get_version_key_from_filename(
+                        filename=filename
+                    )
+
+                    if file_version in seen_versions:
+                        raise DuplicateMigrationVersionError(
+                            f"Duplicate migration version detected: {file_version}.\n"
+                            "Each file must have a unique version.\n"
+                            "Please rename the file to have a unique version."
+                        )
+                    seen_versions.add(file_version)
+
+                    if end_version:
+                        if parse_version(file_version) > parse_version(end_version):
+                            continue
+
+                    if version_to_start_from:
+                        if parse_version(file_version) >= parse_version(
+                            version_to_start_from
+                        ):
+                            version_to_filepath_dict[file_version] = file_path
+
+                    else:
+                        version_to_filepath_dict[file_version] = file_path
+
+    ordered_version_to_filepath_dict: dict[str, str] = dict(
+        sorted(
+            version_to_filepath_dict.items(),
+            key=lambda item: parse_version(item[0]),
+        )
+    )
+
+    return ordered_version_to_filepath_dict

jetbase/enums.py

@@ -1,6 +1,42 @@
 from enum import Enum
 
 
-class MigrationOperationType(Enum):
+class MigrationDirectionType(Enum):
+    """
+    Enum representing the direction of a migration operation.
+
+    Attributes:
+        UPGRADE: Apply migrations forward (run upgrade SQL).
+        ROLLBACK: Revert migrations backward (run rollback SQL).
+    """
+
     UPGRADE = "upgrade"
     ROLLBACK = "rollback"
+
+
+class MigrationType(Enum):
+    """
+    Enum representing the type of migration.
+
+    Attributes:
+        VERSIONED: Standard migration with a version number, runs once.
+        RUNS_ON_CHANGE: Repeatable migration that runs when its checksum changes.
+        RUNS_ALWAYS: Repeatable migration that runs on every upgrade.
+    """
+
+    VERSIONED = "VERSIONED"
+    RUNS_ON_CHANGE = "RUNS_ON_CHANGE"
+    RUNS_ALWAYS = "RUNS_ALWAYS"
+
+
+class DatabaseType(Enum):
+    """
+    Enum representing supported database types.
+
+    Attributes:
+        POSTGRESQL: PostgreSQL database.
+        SQLITE: SQLite database.
+    """
+
+    POSTGRESQL = "postgresql"
+    SQLITE = "sqlite"