python-ddd-framework 0.3.1__py3-none-any.whl

python_ddd_framework/sqlalchemy/migration.py

@@ -0,0 +1,378 @@
+"""Host-owned Alembic orchestration；只消费 live Module metadata，不维护模块清单。"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+from collections.abc import Callable
+from dataclasses import dataclass
+from functools import partial
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from alembic import command
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.script import ScriptDirectory
+from sqlalchemy import Connection
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from ..background_workers import BackgroundExecutionOptions, BackgroundExecutionProfile
+from ..lifecycle import ApplicationState
+from ..modularity import AppModule, ModuleKey
+from ..options import OptionsSnapshot
+from .errors import (
+    MigrationHeadsMismatchError,
+    SoftDeleteMigrationBlockedError,
+    SqlAlchemyMigrationError,
+    SqlAlchemyModelRegistrationError,
+)
+from .metadata import SqlAlchemyModelCatalog, _build_model_catalog
+from .migration_options import SqlAlchemyMigrationOptions
+from .module_migration import _ModuleMigration
+from .options import ConnectionStringsOptions
+
+if TYPE_CHECKING:
+    from ..application import Application
+
+
+class SqlAlchemyMigrator:
+    __slots__ = ("_application",)
+
+    def __init__(self, application: Application) -> None:
+        self._application = application
+
+    @property
+    def model_catalog(self) -> SqlAlchemyModelCatalog:
+        """供原生 Alembic authoring 消费 live ownership，不维护第二份迁移路径清单。"""
+        if self._application.state is not ApplicationState.COMPOSED:
+            raise RuntimeError(
+                "SqlAlchemyMigrator requires composed Application, "
+                f"got {self._application.state.value}"
+            )
+        models = _build_model_catalog(self._application.modules, self._application.options)
+        models._assert_unchanged()
+        return models
+
+    async def upgrade_heads(self, *, module: type[AppModule] | None = None) -> tuple[str, ...]:
+        return await self._run(
+            "upgrade heads",
+            _upgrade_heads,
+            module=module,
+            module_operation=_ModuleMigration.upgrade,
+        )
+
+    async def check_current_heads(
+        self, *, module: type[AppModule] | None = None
+    ) -> tuple[str, ...]:
+        return await self._run("check current heads", _check_current_heads, module=module)
+
+    async def revision(self, module: type[AppModule]) -> tuple[str, ...]:
+        return await self._run(
+            "revision",
+            _check_current_heads,
+            module=module,
+            module_operation=_ModuleMigration.revision,
+        )
+
+    async def _run(
+        self,
+        operation: str,
+        callback: Callable[[Connection, Config, str], None],
+        *,
+        module: type[AppModule] | None = None,
+        module_operation: Callable[
+            [_ModuleMigration, Connection, Config, str], None
+        ] = _ModuleMigration.status,
+    ) -> tuple[str, ...]:
+        models = self.model_catalog
+        owner = ModuleKey(module.__module__, module.__qualname__) if module is not None else None
+        if module_operation is _ModuleMigration.revision and not any(
+            item.owner == owner for item in models.contributions
+        ):
+            raise SqlAlchemyModelRegistrationError(
+                owner=str(owner), reason="revision requires a Module with registered ORM models"
+            )
+        options = self._application.options.get(ConnectionStringsOptions)
+        configured_names = set(options.connection_names)
+        missing = set(models.connection_names) - configured_names
+        if missing:
+            from .errors import UnknownSqlAlchemyConnectionError
+
+            raise UnknownSqlAlchemyConnectionError(sorted(missing)[0])
+
+        completed: list[str] = []
+        for connection_name in options.connection_names:
+            migration_sources = models._migration_sources_for_connection(connection_name)
+            if owner is not None and not any(item.owner == owner for item in migration_sources):
+                continue
+            if not migration_sources:
+                completed.append(connection_name)
+                continue
+            config = _alembic_config(models, connection_name, self._application.options)
+            execution_options = self._application.options.get(BackgroundExecutionOptions)
+            config.attributes["background_execution_disabled"] = (
+                execution_options.profile is BackgroundExecutionProfile.DISABLED
+            )
+            _validate_revision_graph(
+                models,
+                connection_name,
+                config,
+                owner=owner,
+                allow_empty=module_operation is _ModuleMigration.revision,
+            )
+            selected_callback = callback
+            if module is not None:
+                selected = _ModuleMigration(models, module, connection_name, config)
+                selected_callback = partial(module_operation, selected)
+            outcome = await _run_connection_migration(
+                options._url_for(connection_name),
+                config=config,
+                connection_name=connection_name,
+                callback=selected_callback,
+            )
+            if outcome.operation_completed:
+                completed.append(connection_name)
+            _raise_migration_error(
+                outcome,
+                operation=operation,
+                connection_name=connection_name,
+                completed_connections=tuple(completed),
+            )
+        return tuple(completed)
+
+
+@dataclass(frozen=True, slots=True)
+class _MigrationOutcome:
+    operation_completed: bool
+    error_type: str | None
+    cleanup_error_types: tuple[str, ...]
+    control_flow_error: BaseException | None
+    affected_tables: tuple[tuple[str, int], ...] = ()
+    registration_error: SqlAlchemyModelRegistrationError | None = None
+
+
+async def _run_connection_migration(
+    url: str,
+    *,
+    config: Config,
+    connection_name: str,
+    callback: Callable[[Connection, Config, str], None],
+) -> _MigrationOutcome:
+    engine: AsyncEngine | None = None
+    operation_completed = False
+    error_type: str | None = None
+    control_flow_error: BaseException | None = None
+    affected_tables: tuple[tuple[str, int], ...] = ()
+    registration_error: SqlAlchemyModelRegistrationError | None = None
+    try:
+        engine = create_async_engine(url, poolclass=NullPool)
+        async with engine.connect() as connection:
+            config.attributes["connection"] = connection
+            await connection.run_sync(
+                partial(
+                    _invoke_callback,
+                    callback=callback,
+                    config=config,
+                    connection_name=connection_name,
+                )
+            )
+        operation_completed = True
+    except SqlAlchemyModelRegistrationError as invalid:
+        registration_error = invalid
+    except SoftDeleteMigrationBlockedError as blocked:
+        error_type = type(blocked).__name__
+        affected_tables = blocked.affected_tables
+    except Exception as original_error:
+        error_type = type(original_error).__name__
+    except BaseException as original_control_flow:
+        control_flow_error = original_control_flow
+
+    cleanup_error_types: tuple[str, ...] = ()
+    if engine is not None:
+        cleanup_error_types, cleanup_control_flow = await _dispose_engine(engine)
+        if control_flow_error is None:
+            control_flow_error = cleanup_control_flow
+    return _MigrationOutcome(
+        operation_completed=operation_completed,
+        error_type=error_type,
+        cleanup_error_types=cleanup_error_types,
+        control_flow_error=control_flow_error,
+        affected_tables=affected_tables,
+        registration_error=registration_error,
+    )
+
+
+async def _dispose_engine(
+    engine: AsyncEngine,
+) -> tuple[tuple[str, ...], BaseException | None]:
+    # dispose 自己持有独立 task；外层 cancellation 只能延迟传播，不能截断连接池释放。
+    disposal = asyncio.create_task(engine.dispose())
+    control_flow_error: BaseException | None = None
+    cleanup_error_types: tuple[str, ...] = ()
+    while not disposal.done():
+        try:
+            await asyncio.shield(disposal)
+        except asyncio.CancelledError as original_cancellation:
+            if control_flow_error is None:
+                control_flow_error = original_cancellation
+        except Exception as cleanup_error:
+            cleanup_error_types = (type(cleanup_error).__name__,)
+            break
+    if disposal.done() and not cleanup_error_types:
+        try:
+            disposal.result()
+        except asyncio.CancelledError as disposal_cancellation:
+            if control_flow_error is None:
+                control_flow_error = disposal_cancellation
+        except Exception as cleanup_error:
+            cleanup_error_types = (type(cleanup_error).__name__,)
+    return cleanup_error_types, control_flow_error
+
+
+def _raise_migration_error(
+    outcome: _MigrationOutcome,
+    *,
+    operation: str,
+    connection_name: str,
+    completed_connections: tuple[str, ...],
+) -> None:
+    failure: BaseException | None = None
+    if outcome.error_type is not None or outcome.cleanup_error_types:
+        failure = SqlAlchemyMigrationError(
+            operation=operation,
+            failed_connection=connection_name,
+            completed_connections=completed_connections,
+            error_type=outcome.error_type or outcome.cleanup_error_types[0],
+            cleanup_error_types=outcome.cleanup_error_types,
+            affected_tables=outcome.affected_tables,
+        )
+    if outcome.registration_error is not None:
+        outcome.registration_error.__cause__ = failure
+        failure = outcome.registration_error
+    # 控制流先传播，已确认的前置条件错误与清理故障保留在 cause 链中。
+    if outcome.control_flow_error is not None:
+        if failure is not None:
+            raise outcome.control_flow_error from failure
+        raise outcome.control_flow_error
+    if failure is not None:
+        raise failure
+
+
+def _invoke_callback(
+    connection: Connection,
+    *,
+    callback: Callable[[Connection, Config, str], None],
+    config: Config,
+    connection_name: str,
+) -> None:
+    callback(connection, config, connection_name)
+
+
+def _alembic_config(
+    models: SqlAlchemyModelCatalog, connection_name: str, options: OptionsSnapshot
+) -> Config:
+    migration_sources = models._migration_sources_for_connection(connection_name)
+    model_contributions = models.for_connection(connection_name)
+    migration_options = options.get(SqlAlchemyMigrationOptions)
+    unknown = set(migration_options.named) - set(
+        options.get(ConnectionStringsOptions).connection_names
+    )
+    if unknown:
+        from .errors import UnknownSqlAlchemyConnectionError
+
+        raise UnknownSqlAlchemyConnectionError(sorted(unknown)[0])
+    version_table = migration_options._for_connection(connection_name)
+    for contribution in models.migrations_for_connection(connection_name):
+        if version_table.version_table_schema in contribution.registration.owned_schemas:
+            # migration-only owner 可整体删除自己的 Schema，不能同时拥有迁移记录。
+            raise SqlAlchemyModelRegistrationError(
+                owner=str(contribution.owner),
+                reason="version table schema must be separate from migration-owned schemas",
+            )
+    config = Config()
+    config.set_main_option(
+        "script_location",
+        str(Path(__file__).with_name("alembic_runtime")),
+    )
+    config.set_main_option(
+        "version_locations",
+        os.pathsep.join(str(item.migration_path) for item in migration_sources),
+    )
+    config.set_main_option("path_separator", "os")
+    config.attributes["target_metadata"] = tuple(
+        item.registration.metadata for item in model_contributions
+    )
+    # 原生 Alembic context 消费构建 catalog 的同一快照，不再读取进程环境。
+    config.attributes["options_snapshot"] = options
+    config.attributes["version_table_options"] = {
+        "version_table": version_table.version_table,
+        "version_table_schema": version_table.version_table_schema,
+    }
+    return config
+
+
+def _validate_revision_graph(
+    models: SqlAlchemyModelCatalog,
+    connection_name: str,
+    config: Config,
+    *,
+    owner: ModuleKey | None = None,
+    allow_empty: bool = False,
+) -> None:
+    migration_sources = models._migration_sources_for_connection(connection_name)
+    # walk_revisions 强制 Alembic 在任何 engine 创建前加载并校验组合 graph/depends_on。
+    revisions = tuple(ScriptDirectory.from_config(config).walk_revisions())
+    for source in migration_sources:
+        owned_revisions = tuple(
+            revision
+            for revision in revisions
+            if Path(revision.path).resolve().parent == source.migration_path
+        )
+        if not owned_revisions:
+            if (owner is not None and source.owner != owner) or allow_empty:
+                continue
+            raise SqlAlchemyModelRegistrationError(
+                owner=str(source.owner),
+                reason=(
+                    f"migration package {source.migrations_package!r} "
+                    "does not contain an Alembic revision; "
+                    "run pddd db revision --module <alias> first"
+                ),
+            )
+        expected_labels = {source.branch_label}
+        for revision in owned_revisions:
+            if revision.branch_labels != expected_labels:
+                actual_labels = tuple(sorted(revision.branch_labels))
+                raise SqlAlchemyModelRegistrationError(
+                    owner=str(source.owner),
+                    reason=(
+                        f"revision {revision.revision!r} exposes branch_labels "
+                        f"{actual_labels!r}, expected {(source.branch_label,)!r}"
+                    ),
+                )
+
+
+def _upgrade_heads(connection: Connection, config: Config, connection_name: str) -> None:
+    config.attributes["connection"] = connection
+    command.upgrade(config, "heads")
+
+
+def _check_current_heads(connection: Connection, config: Config, connection_name: str) -> None:
+    config.attributes["connection"] = connection
+    script = ScriptDirectory.from_config(config)
+    current = tuple(
+        sorted(
+            MigrationContext.configure(
+                connection, opts=config.attributes["version_table_options"]
+            ).get_current_heads()
+        )
+    )
+    expected = tuple(sorted(script.get_heads()))
+    if current != expected:
+        raise MigrationHeadsMismatchError(
+            connection_name=connection_name,
+            current_heads=current,
+            expected_heads=expected,
+        )

python_ddd_framework/sqlalchemy/migration_options.py

@@ -0,0 +1,56 @@
+"""Host-owned Alembic version-table locations；revision 身份仍由脚本拥有。"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Mapping
+
+from pydantic import ConfigDict, Field, field_validator
+
+from ..options import BaseOptions
+from .options import _validate_connection_name
+
+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,62}")
+
+
+class SqlAlchemyVersionTable(BaseOptions):
+    model_config = ConfigDict(extra="forbid")
+
+    # Alembic 1.19.1 为主键追加 _pkc；完整约束名也必须不超过 PostgreSQL 的 63 字节。
+    version_table: str = Field(default="alembic_version", max_length=59)
+    version_table_schema: str = "public"
+
+    @field_validator("version_table", "version_table_schema")
+    @classmethod
+    def _identifier_is_valid(cls, value: str) -> str:
+        if not _IDENTIFIER.fullmatch(value):
+            raise ValueError("version-table identifiers must contain 1-63 ASCII identifier bytes")
+        return value.lower()
+
+    @field_validator("version_table_schema")
+    @classmethod
+    def _schema_is_writable(cls, value: str) -> str:
+        if value == "information_schema" or value.startswith("pg_"):
+            raise ValueError("version table must not use a PostgreSQL system schema")
+        return value
+
+
+class SqlAlchemyMigrationOptions(BaseOptions):
+    model_config = ConfigDict(extra="forbid")
+
+    default: SqlAlchemyVersionTable = Field(default_factory=SqlAlchemyVersionTable)
+    named: Mapping[str, SqlAlchemyVersionTable] = Field(default_factory=dict)
+
+    @field_validator("named")
+    @classmethod
+    def _named_connections_are_valid(
+        cls, value: Mapping[str, SqlAlchemyVersionTable]
+    ) -> Mapping[str, SqlAlchemyVersionTable]:
+        for name in value:
+            _validate_connection_name(name)
+            if name == "default":
+                raise ValueError("named migration options must not redefine 'default'")
+        return value
+
+    def _for_connection(self, connection_name: str) -> SqlAlchemyVersionTable:
+        return self.named.get(connection_name, self.default)

python_ddd_framework/sqlalchemy/module.py

@@ -0,0 +1,51 @@
+"""Host 显式依赖的 provider-neutral SQLAlchemy persistence Module。"""
+
+from __future__ import annotations
+
+from dishka import Provider, Scope
+
+from ..lifecycle import ConfigureContext, InitializeContext
+from ..modularity import AppModule, ModuleRegistry
+from ..options import OptionsSnapshot
+from ..unit_of_work.contracts import _UnitOfWorkFactory
+from .metadata import SqlAlchemyModelCatalog, _build_model_catalog
+from .migration_options import SqlAlchemyMigrationOptions
+from .options import ConnectionStringsOptions
+from .runtime import (
+    _engine_registry,
+    _SqlAlchemyEngineRegistry,
+)
+from .session_provider import SqlAlchemySessionProvider
+from .unit_of_work import _SqlAlchemyUnitOfWorkFactory
+
+
+class SqlAlchemyPersistenceModule(AppModule):
+    def configure(self, context: ConfigureContext) -> None:
+        context.configure(
+            ConnectionStringsOptions,
+            section="connection_strings",
+            secret_paths=frozenset({"default", "named"}),
+        )
+        context.configure(SqlAlchemyMigrationOptions, section="sqlalchemy_migrations")
+        provider = Provider()
+        provider.provide(
+            _model_catalog,
+            provides=SqlAlchemyModelCatalog,
+            scope=Scope.APP,
+        )
+        provider.provide(
+            _engine_registry,
+            provides=_SqlAlchemyEngineRegistry,
+            scope=Scope.APP,
+        )
+        provider.provide(_SqlAlchemyUnitOfWorkFactory, provides=_UnitOfWorkFactory, scope=Scope.APP)
+        provider.provide(SqlAlchemySessionProvider, scope=Scope.APP)
+        context.services.contribute(provider, reason="SQLAlchemy PostgreSQL persistence boundary")
+
+    async def initialize(self, context: InitializeContext) -> None:
+        engines = await context.container.get(_SqlAlchemyEngineRegistry)
+        await engines.ping_all()
+
+
+def _model_catalog(registry: ModuleRegistry, options: OptionsSnapshot) -> SqlAlchemyModelCatalog:
+    return _build_model_catalog(registry, options)

python_ddd_framework/sqlalchemy/module_migration.py

@@ -0,0 +1,151 @@
+"""单模块 Alembic 操作：原生图决定前置关系，模型 catalog 决定比较边界。"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from sysconfig import get_path
+from typing import Any
+
+from alembic import command
+from alembic.autogenerate import produce_migrations
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.operations.ops import MigrationScript
+from alembic.script import ScriptDirectory
+from sqlalchemy import Connection
+
+from ..modularity import AppModule, ModuleKey
+from .errors import SqlAlchemyModelRegistrationError
+from .metadata import SqlAlchemyModelCatalog
+
+
+class _ModuleMigration:
+    def __init__(
+        self,
+        models: SqlAlchemyModelCatalog,
+        module: type[AppModule],
+        connection_name: str,
+        config: Config,
+    ) -> None:
+        self.owner = ModuleKey(module.__module__, module.__qualname__)
+        sources = models._migration_sources_for_connection(connection_name)
+        self.sources = tuple(item for item in sources if item.owner == self.owner)
+        self.metadata = tuple(
+            item.registration.metadata
+            for item in models.for_connection(connection_name)
+            if item.owner == self.owner
+        )
+        self.script = ScriptDirectory.from_config(config)
+        self._version_table_options = config.attributes["version_table_options"]
+        self.revisions = tuple(self.script.walk_revisions())
+
+    def _owned(self) -> frozenset[str]:
+        paths = {item.migration_path for item in self.sources}
+        return frozenset(
+            item.revision for item in self.revisions if Path(item.path).resolve().parent in paths
+        )
+
+    def _heads(self) -> tuple[str, ...]:
+        return tuple(
+            revision.revision
+            for source in self.sources
+            for revision in self.script.get_revisions(f"{source.branch_label}@heads")
+        )
+
+    def _applied(self, connection: Connection) -> frozenset[str]:
+        current = MigrationContext.configure(
+            connection, opts=self._version_table_options
+        ).get_current_heads()
+        # 由 Alembic 展开已应用的 ancestry／depends_on；不维护第二套 revision graph。
+        return frozenset(
+            item.revision
+            for item in self.script.iterate_revisions(current, "base", implicit_base=True)
+        )
+
+    def _require(self, required: frozenset[str], applied: frozenset[str]) -> None:
+        missing = sorted(required - applied)
+        if missing:
+            raise SqlAlchemyModelRegistrationError(
+                owner=str(self.owner),
+                reason=(
+                    f"Required migrations {missing} are not applied; "
+                    "run pddd db upgrade --module <owning-alias> first"
+                ),
+            )
+
+    def upgrade(self, connection: Connection, config: Config, connection_name: str) -> None:
+        heads = self._heads()
+        ancestry = frozenset(
+            item.revision
+            for item in self.script.iterate_revisions(heads, "base", implicit_base=True)
+        )
+        # 预检 SELECT 会触发 SQLAlchemy autobegin；同一个显式事务拥有预检和全部目标分支，
+        # 避免 Alembic 将它视为外部事务后，连接退出时回滚已经执行的升级。
+        with connection.begin():
+            self._require(ancestry - self._owned(), self._applied(connection))
+            config.attributes["connection"] = connection
+            for head in heads:
+                command.upgrade(config, head)
+
+    def status(self, connection: Connection, config: Config, connection_name: str) -> None:
+        self._require(frozenset(self._heads()), self._applied(connection))
+
+    def revision(self, connection: Connection, config: Config, connection_name: str) -> None:
+        if len(self.sources) != 1 or not self.metadata:
+            raise SqlAlchemyModelRegistrationError(
+                owner=str(self.owner),
+                reason="revision requires one model-owned migration branch",
+            )
+        source = self.sources[0]
+        if not source.migration_path.is_relative_to(Path.cwd().resolve()) or any(
+            source.migration_path.is_relative_to(Path(get_path(kind)).resolve())
+            for kind in ("purelib", "platlib")
+        ):
+            raise SqlAlchemyModelRegistrationError(
+                owner=str(self.owner),
+                reason="Revision authoring requires project-owned source, not an installed package",
+            )
+        owned = self._owned()
+        heads = self._heads() if owned else ()
+        required = frozenset(
+            item.revision
+            for item in self.script.iterate_revisions(heads, "base", implicit_base=True)
+        )
+        # Schema 依赖只来自原生 revision；Module 的运行时依赖不推断为数据库前置关系。
+        self._require(required, self._applied(connection))
+        config.attributes.update(
+            connection=connection,
+            target_metadata=self.metadata,
+            include_name=self._include_name,
+        )
+        config.set_main_option("revision_environment", "true")
+        # command.revision 的整库 autogenerate 检查会误阻塞无关未升级分支；改用其公开 hook
+        # 和 produce_migrations，仍由原生 renderer／ScriptDirectory 写 revision 与依赖声明。
+        command.revision(
+            config,
+            message=f"Update {source.branch_label}",
+            head=f"{source.branch_label}@head" if heads else "base",
+            branch_label=None if heads else source.branch_label,
+            version_path=str(source.migration_path),
+            process_revision_directives=self._generate,
+        )
+
+    def _include_name(self, name: str | None, type_: str, parents: dict[str, Any]) -> bool:
+        if type_ == "schema":
+            return name in {
+                table.schema for meta in self.metadata for table in meta.tables.values()
+            }
+        if type_ == "table":
+            return any(
+                parents["schema_qualified_table_name"] in meta.tables for meta in self.metadata
+            )
+        return True
+
+    def _generate(
+        self, context: MigrationContext, revision: Any, directives: list[MigrationScript]
+    ) -> None:
+        # 一个 migration source 对应一份已登记 MetaData；不复制表形成影子 metadata。
+        generated = produce_migrations(context, self.metadata[0])
+        assert generated.upgrade_ops is not None and generated.downgrade_ops is not None
+        directives[0].upgrade_ops = generated.upgrade_ops
+        directives[0].downgrade_ops = generated.downgrade_ops