plain.postgres 0.84.0__py3-none-any.whl

plain/postgres/migrations/loader.py

@@ -0,0 +1,377 @@
+from __future__ import annotations
+
+import pkgutil
+import sys
+from importlib import import_module, reload
+from typing import TYPE_CHECKING, Any
+
+from plain.packages import packages_registry
+from plain.postgres.migrations.graph import MigrationGraph
+from plain.postgres.migrations.recorder import MigrationRecorder
+
+from .exceptions import (
+    AmbiguityError,
+    BadMigrationError,
+    InconsistentMigrationHistory,
+    NodeNotFoundError,
+)
+
+if TYPE_CHECKING:
+    from plain.postgres.connection import DatabaseConnection
+    from plain.postgres.migrations.migration import Migration
+
+MIGRATIONS_MODULE_NAME = "migrations"
+
+
+class MigrationLoader:
+    """
+    Load migration files from disk and their status from the database.
+
+    Migration files are expected to live in the "migrations" directory of
+    an app. Their names are entirely unimportant from a code perspective,
+    but will probably follow the 1234_name.py convention.
+
+    On initialization, this class will scan those directories, and open and
+    read the Python files, looking for a class called Migration, which should
+    inherit from plain.postgres.migrations.Migration. See
+    plain.postgres.migrations.migration for what that looks like.
+
+    Some migrations will be marked as "replacing" another set of migrations.
+    These are loaded into a separate set of migrations away from the main ones.
+    If all the migrations they replace are either unapplied or missing from
+    disk, then they are injected into the main set, replacing the named migrations.
+    Any dependency pointers to the replaced migrations are re-pointed to the
+    new migration.
+
+    This does mean that this class MUST also talk to the database as well as
+    to disk, but this is probably fine. We're already not just operating
+    in memory.
+    """
+
+    def __init__(
+        self,
+        connection: DatabaseConnection | None,
+        load: bool = True,
+        ignore_no_migrations: bool = False,
+        replace_migrations: bool = True,
+    ):
+        self.connection = connection
+        self.disk_migrations: dict[tuple[str, str], Migration] | None = None
+        self.applied_migrations: dict[tuple[str, str], Any] | None = None
+        self.ignore_no_migrations = ignore_no_migrations
+        self.replace_migrations = replace_migrations
+        self.unmigrated_packages: set[str]
+        self.migrated_packages: set[str]
+        self.graph: MigrationGraph
+        self.replacements: dict[tuple[str, str], Migration]
+        if load:
+            self.build_graph()
+
+    @classmethod
+    def migrations_module(cls, package_label: str) -> tuple[str | None, bool]:
+        """
+        Return the path to the migrations module for the specified package_label
+        and a boolean indicating if the module is specified in
+        settings.MIGRATION_MODULE.
+        """
+
+        # This package (plain-postgres) has different code under migrations/
+        if package_label == "plainpostgres":
+            return None, True
+
+        app = packages_registry.get_package_config(package_label)
+        return f"{app.name}.{MIGRATIONS_MODULE_NAME}", False
+
+    def load_disk(self) -> None:
+        """Load the migrations from all INSTALLED_PACKAGES from disk."""
+        self.disk_migrations = {}
+        self.unmigrated_packages = set()
+        self.migrated_packages = set()
+        for package_config in packages_registry.get_package_configs():
+            # Get the migrations module directory
+            module_name, explicit = self.migrations_module(package_config.package_label)
+            if module_name is None:
+                self.unmigrated_packages.add(package_config.package_label)
+                continue
+            was_loaded = module_name in sys.modules
+            try:
+                module = import_module(module_name)
+            except ModuleNotFoundError as e:
+                if (explicit and self.ignore_no_migrations) or (
+                    not explicit
+                    and e.name is not None
+                    and MIGRATIONS_MODULE_NAME in e.name.split(".")
+                ):
+                    self.unmigrated_packages.add(package_config.package_label)
+                    continue
+                raise
+            else:
+                # Module is not a package (e.g. migrations.py).
+                if not hasattr(module, "__path__"):
+                    self.unmigrated_packages.add(package_config.package_label)
+                    continue
+                # Empty directories are namespaces. Namespace packages have no
+                # __file__ and don't use a list for __path__. See
+                # https://docs.python.org/3/reference/import.html#namespace-packages
+                if getattr(module, "__file__", None) is None and not isinstance(
+                    module.__path__, list
+                ):
+                    self.unmigrated_packages.add(package_config.package_label)
+                    continue
+                # Force a reload if it's already loaded (tests need this)
+                if was_loaded:
+                    reload(module)
+            self.migrated_packages.add(package_config.package_label)
+            migration_names = {
+                name
+                for _, name, is_pkg in pkgutil.iter_modules(module.__path__)
+                if not is_pkg and name[0] not in "_~"
+            }
+            # Load migrations
+            for migration_name in migration_names:
+                migration_path = f"{module_name}.{migration_name}"
+                try:
+                    migration_module = import_module(migration_path)
+                except ImportError as e:
+                    if "bad magic number" in str(e):
+                        raise ImportError(
+                            f"Couldn't import {migration_path!r} as it appears to be a stale "
+                            ".pyc file."
+                        ) from e
+                    else:
+                        raise
+                if not hasattr(migration_module, "Migration"):
+                    raise BadMigrationError(
+                        f"Migration {migration_name} in app {package_config.package_label} has no Migration class"
+                    )
+                self.disk_migrations[package_config.package_label, migration_name] = (
+                    migration_module.Migration(
+                        migration_name,
+                        package_config.package_label,
+                    )
+                )
+
+    def get_migration(self, package_label: str, name_prefix: str) -> Migration | None:
+        """Return the named migration or raise NodeNotFoundError."""
+        return self.graph.nodes[package_label, name_prefix]
+
+    def get_migration_by_prefix(
+        self, package_label: str, name_prefix: str
+    ) -> Migration:
+        """
+        Return the migration(s) which match the given app label and name_prefix.
+        """
+        # Do the search
+        assert self.disk_migrations is not None, "load_disk() must be called first"
+        results = []
+        for migration_package_label, migration_name in self.disk_migrations:
+            if migration_package_label == package_label and migration_name.startswith(
+                name_prefix
+            ):
+                results.append((migration_package_label, migration_name))
+        if len(results) > 1:
+            raise AmbiguityError(
+                f"There is more than one migration for '{package_label}' with the prefix '{name_prefix}'"
+            )
+        elif not results:
+            raise KeyError(
+                f"There is no migration for '{package_label}' with the prefix "
+                f"'{name_prefix}'"
+            )
+        else:
+            return self.disk_migrations[results[0]]
+
+    def check_key(
+        self, key: tuple[str, str], current_package: str
+    ) -> tuple[str, str] | None:
+        if (key[1] != "__first__" and key[1] != "__latest__") or key in self.graph:
+            return key
+        # Special-case __first__, which means "the first migration" for
+        # migrated packages, and is ignored for unmigrated packages. It allows
+        # makemigrations to declare dependencies on packages before they even have
+        # migrations.
+        if key[0] == current_package:
+            # Ignore __first__ references to the same app (#22325)
+            return None
+        if key[0] in self.unmigrated_packages:
+            # This app isn't migrated, but something depends on it.
+            # The models will get auto-added into the state, though
+            # so we're fine.
+            return None
+        if key[0] in self.migrated_packages:
+            try:
+                if key[1] == "__first__":
+                    return self.graph.root_nodes(key[0])[0]
+                else:  # "__latest__"
+                    return self.graph.leaf_nodes(key[0])[0]
+            except IndexError:
+                if self.ignore_no_migrations:
+                    return None
+                else:
+                    raise ValueError(f"Dependency on app with no migrations: {key[0]}")
+        raise ValueError(f"Dependency on unknown app: {key[0]}")
+
+    def add_internal_dependencies(
+        self, key: tuple[str, str], migration: Migration
+    ) -> None:
+        """
+        Internal dependencies need to be added first to ensure `__first__`
+        dependencies find the correct root node.
+        """
+        for parent in migration.dependencies:
+            # Ignore __first__ references to the same app.
+            if parent[0] == key[0] and parent[1] != "__first__":
+                # Migration object is used only for error messages in add_dependency
+                self.graph.add_dependency(migration, key, parent, skip_validation=True)
+
+    def add_external_dependencies(
+        self, key: tuple[str, str], migration: Migration
+    ) -> None:
+        for parent in migration.dependencies:
+            # Skip internal dependencies
+            if key[0] == parent[0]:
+                continue
+            parent = self.check_key(parent, key[0])
+            if parent is not None:
+                # Migration object is used only for error messages in add_dependency
+                self.graph.add_dependency(migration, key, parent, skip_validation=True)
+
+    def build_graph(self) -> None:
+        """
+        Build a migration dependency graph using both the disk and database.
+        You'll need to rebuild the graph if you apply migrations. This isn't
+        usually a problem as generally migration stuff runs in a one-shot process.
+        """
+        # Load disk data
+        self.load_disk()
+        assert self.disk_migrations is not None  # load_disk() ensures this
+        # Load database data
+        if self.connection is None:
+            self.applied_migrations = {}
+        else:
+            recorder = MigrationRecorder(self.connection)
+            self.applied_migrations = recorder.applied_migrations()
+        # To start, populate the migration graph with nodes for ALL migrations
+        # and their dependencies. Also make note of replacing migrations at this step.
+        self.graph = MigrationGraph()
+        self.replacements = {}
+        for key, migration in self.disk_migrations.items():
+            self.graph.add_node(key, migration)
+            # Replacing migrations.
+            if migration.replaces:
+                self.replacements[key] = migration
+        for key, migration in self.disk_migrations.items():
+            # Internal (same app) dependencies.
+            self.add_internal_dependencies(key, migration)
+        # Add external dependencies now that the internal ones have been resolved.
+        for key, migration in self.disk_migrations.items():
+            self.add_external_dependencies(key, migration)
+        # Carry out replacements where possible and if enabled.
+        if self.replace_migrations:
+            for key, migration in self.replacements.items():
+                # Get applied status of each of this migration's replacement
+                # targets.
+                applied_statuses = [
+                    (target in self.applied_migrations) for target in migration.replaces
+                ]
+                # The replacing migration is only marked as applied if all of
+                # its replacement targets are.
+                if all(applied_statuses):
+                    self.applied_migrations[key] = migration
+                else:
+                    self.applied_migrations.pop(key, None)
+                # A replacing migration can be used if either all or none of
+                # its replacement targets have been applied.
+                if all(applied_statuses) or (not any(applied_statuses)):
+                    self.graph.remove_replaced_nodes(key, migration.replaces)
+                else:
+                    # This replacing migration cannot be used because it is
+                    # partially applied. Remove it from the graph and remap
+                    # dependencies to it (#25945).
+                    self.graph.remove_replacement_node(key, migration.replaces)
+        # Ensure the graph is consistent.
+        try:
+            self.graph.validate_consistency()
+        except NodeNotFoundError as exc:
+            # Check if the missing node could have been replaced by any squash
+            # migration but wasn't because the squash migration was partially
+            # applied before. In that case raise a more understandable exception
+            # (#23556).
+            # Get reverse replacements.
+            reverse_replacements = {}
+            for key, migration in self.replacements.items():
+                for replaced in migration.replaces:
+                    reverse_replacements.setdefault(replaced, set()).add(key)
+            # Try to reraise exception with more detail.
+            if exc.node in reverse_replacements:
+                candidates = reverse_replacements.get(exc.node, set())
+                is_replaced = any(
+                    candidate in self.graph.nodes for candidate in candidates
+                )
+                if not is_replaced:
+                    tries = ", ".join("{}.{}".format(*c) for c in candidates)
+                    raise NodeNotFoundError(
+                        f"Migration {exc.origin} depends on nonexistent node ('{exc.node[0]}', '{exc.node[1]}'). "
+                        f"Plain tried to replace migration {exc.node[0]}.{exc.node[1]} with any of [{tries}] "
+                        "but wasn't able to because some of the replaced migrations "
+                        "are already applied.",
+                        exc.node,
+                    ) from exc
+            raise
+        self.graph.ensure_not_cyclic()
+
+    def check_consistent_history(self, connection: DatabaseConnection) -> None:
+        """
+        Raise InconsistentMigrationHistory if any applied migrations have
+        unapplied dependencies.
+        """
+        recorder = MigrationRecorder(connection)
+        applied = recorder.applied_migrations()
+        for migration in applied:
+            # If the migration is unknown, skip it.
+            if migration not in self.graph.nodes:
+                continue
+            for parent in self.graph.node_map[migration].parents:
+                if parent not in applied:
+                    # Skip unapplied squashed migrations that have all of their
+                    # `replaces` applied.
+                    # Use parent.key for dict lookup (Node.__eq__ allows `in` check)
+                    if parent.key in self.replacements:
+                        if all(
+                            m in applied for m in self.replacements[parent.key].replaces
+                        ):
+                            continue
+                    raise InconsistentMigrationHistory(
+                        f"Migration {migration[0]}.{migration[1]} is applied before its dependency "
+                        f"{parent[0]}.{parent[1]} on the database."
+                    )
+
+    def detect_conflicts(self) -> dict[str, list[str]]:
+        """
+        Look through the loaded graph and detect any conflicts - packages
+        with more than one leaf migration. Return a dict of the app labels
+        that conflict with the migration names that conflict.
+        """
+        seen_packages = {}
+        conflicting_packages = set()
+        for package_label, migration_name in self.graph.leaf_nodes():
+            if package_label in seen_packages:
+                conflicting_packages.add(package_label)
+            seen_packages.setdefault(package_label, set()).add(migration_name)
+        return {
+            package_label: sorted(seen_packages[package_label])
+            for package_label in conflicting_packages
+        }
+
+    def project_state(
+        self, nodes: tuple[str, str] | None = None, at_end: bool = True
+    ) -> Any:
+        """
+        Return a ProjectState object representing the most recent state
+        that the loaded migrations represent.
+
+        See graph.make_state() for the meaning of "nodes" and "at_end".
+        """
+        return self.graph.make_state(
+            nodes=nodes, at_end=at_end, real_packages=self.unmigrated_packages
+        )

plain/postgres/migrations/migration.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+from plain.postgres.migrations.utils import get_migration_name_timestamp
+from plain.postgres.transaction import atomic
+
+if TYPE_CHECKING:
+    from plain.postgres.migrations.state import ProjectState
+    from plain.postgres.schema import DatabaseSchemaEditor
+
+
+class Migration:
+    """
+    The base class for all migrations.
+
+    Migration files will import this from plain.postgres.migrations.Migration
+    and subclass it as a class called Migration. It will have one or more
+    of the following attributes:
+
+     - operations: A list of Operation instances, probably from
+       plain.postgres.migrations.operations
+     - dependencies: A list of tuples of (app_path, migration_name)
+     - replaces: A list of migration_names
+
+    Note that all migrations come out of migrations and into the Loader or
+    Graph as instances, having been initialized with their app label and name.
+    """
+
+    # Operations to apply during this migration, in order.
+    operations: list[Any] = []
+
+    # Other migrations that should be run before this migration.
+    # Should be a list of (app, migration_name).
+    dependencies: list[tuple[str, str]] = []
+
+    # Migration names in this app that this migration replaces. If this is
+    # non-empty, this migration will only be applied if all these migrations
+    # are not applied.
+    # Note: Despite the comment saying "migration names", this is actually a list of tuples
+    # (app_label, migration_name) as used throughout the codebase.
+    replaces: list[tuple[str, str]] = []
+
+    # Is this an initial migration? Initial migrations are skipped on
+    # --fake-initial if the table or fields already exist. If None, check if
+    # the migration has any dependencies to determine if there are dependencies
+    # to tell if db introspection needs to be done. If True, always perform
+    # introspection. If False, never perform introspection.
+    initial: bool | None = None
+
+    # Whether to wrap the whole migration in a transaction.
+    atomic: bool = True
+
+    def __init__(self, name: str, package_label: str) -> None:
+        self.name = name
+        self.package_label = package_label
+        # Copy dependencies & other attrs as we might mutate them at runtime
+        self.operations = list(self.__class__.operations)
+        self.dependencies = list(self.__class__.dependencies)
+        self.replaces = list(self.__class__.replaces)
+
+    def __eq__(self, other: object) -> bool:
+        return (
+            isinstance(other, Migration)
+            and self.name == other.name
+            and self.package_label == other.package_label
+        )
+
+    def __repr__(self) -> str:
+        return f"<Migration {self.package_label}.{self.name}>"
+
+    def __str__(self) -> str:
+        return f"{self.package_label}.{self.name}"
+
+    def __hash__(self) -> int:
+        return hash(f"{self.package_label}.{self.name}")
+
+    def mutate_state(self, project_state: Any, preserve: bool = True) -> Any:
+        """
+        Take a ProjectState and return a new one with the migration's
+        operations applied to it. Preserve the original object state by
+        default and return a mutated state from a copy.
+        """
+        new_state = project_state
+        if preserve:
+            new_state = project_state.clone()
+
+        for operation in self.operations:
+            operation.state_forwards(self.package_label, new_state)
+        return new_state
+
+    def apply(
+        self,
+        project_state: ProjectState,
+        schema_editor: DatabaseSchemaEditor,
+        operation_callback: Callable[..., Any] | None = None,
+    ) -> ProjectState:
+        """
+        Take a project_state representing all migrations prior to this one
+        and a schema_editor for a live database and apply the migration
+        in a forwards order.
+
+        Return the resulting project state for efficient reuse by following
+        Migrations.
+        """
+        for operation in self.operations:
+            # Clear any previous SQL statements before starting this operation
+            schema_editor.executed_sql = []
+
+            if operation_callback:
+                operation_callback("operation_start", operation=operation)
+            # Save the state before the operation has run
+            old_state = project_state.clone()
+            operation.state_forwards(self.package_label, project_state)
+            # Run the operation
+            atomic_operation = operation.atomic or (
+                self.atomic and operation.atomic is not False
+            )
+            if not schema_editor.atomic_migration and atomic_operation:
+                # Force a transaction for an atomic operation inside a non-atomic migration.
+                with atomic():
+                    operation.database_forwards(
+                        self.package_label, schema_editor, old_state, project_state
+                    )
+            else:
+                # Normal behaviour
+                operation.database_forwards(
+                    self.package_label, schema_editor, old_state, project_state
+                )
+            if operation_callback:
+                # Pass the accumulated SQL statements for this operation
+                operation_callback(
+                    "operation_success",
+                    operation=operation,
+                    sql_statements=schema_editor.executed_sql,
+                )
+        return project_state
+
+    def suggest_name(self) -> str:
+        """
+        Suggest a name for the operations this migration might represent. Names
+        are not guaranteed to be unique, but put some effort into the fallback
+        name to avoid VCS conflicts if possible.
+        """
+        if self.initial:
+            return "initial"
+
+        raw_fragments = [op.migration_name_fragment for op in self.operations]
+        fragments = [re.sub(r"\W+", "_", name) for name in raw_fragments if name]
+
+        if not fragments or len(fragments) != len(self.operations):
+            return f"auto_{get_migration_name_timestamp()}"
+
+        name = fragments[0]
+        for fragment in fragments[1:]:
+            new_name = f"{name}_{fragment}"
+            if len(new_name) > 52:
+                name = f"{name}_and_more"
+                break
+            name = new_name
+        return name
+
+
+class SettingsTuple(tuple):
+    """
+    Subclass of tuple so Plain can tell this was originally a settings
+    dependency when it reads the migration file.
+    """
+
+    def __new__(cls, value: tuple[str, str], setting: str) -> SettingsTuple:
+        self = tuple.__new__(cls, value)
+        self.setting = setting
+        return self
+
+
+def settings_dependency(value: str) -> SettingsTuple:
+    """Turn a setting value into a dependency."""
+    return SettingsTuple((value.split(".", 1)[0], "__first__"), value)

plain/postgres/migrations/operations/__init__.py

@@ -0,0 +1,34 @@
+from .fields import AddField, AlterField, RemoveField, RenameField
+from .models import (
+    AddConstraint,
+    AddIndex,
+    AlterModelOptions,
+    AlterModelTable,
+    CreateModel,
+    DeleteModel,
+    RemoveConstraint,
+    RemoveIndex,
+    RenameIndex,
+    RenameModel,
+)
+from .special import RunPython, RunSQL, SeparateDatabaseAndState
+
+__all__ = [
+    "CreateModel",
+    "DeleteModel",
+    "AlterModelTable",
+    "RenameModel",
+    "AlterModelOptions",
+    "AddIndex",
+    "RemoveIndex",
+    "RenameIndex",
+    "AddField",
+    "RemoveField",
+    "AlterField",
+    "RenameField",
+    "AddConstraint",
+    "RemoveConstraint",
+    "SeparateDatabaseAndState",
+    "RunSQL",
+    "RunPython",
+]