weightsdb 0.2.0__py3-none-any.whl

weightsdb/migrations.py

@@ -0,0 +1,328 @@
+"""weightsdb.migrations — a programmatic Alembic wrapper.
+
+One linear history per application, from the first release (database standards §5). This module
+never shells out to the ``alembic`` CLI or reads an ``alembic.ini`` — it builds an in-memory
+:class:`alembic.config.Config` and drives :mod:`alembic.command` directly, always against the
+already-configured :class:`~sqlalchemy.Engine` the caller built with
+:func:`~weightsdb.engine.create_engine_for`, so every migration runs with the same pragmas and
+transaction semantics as the rest of the application — never a second, differently configured
+connection built from the bare URL.
+
+Moved from FreeWeight's ``infrastructure.db.migration`` (ADR-0011), renamed to the plural
+``migrations`` to match this package's own module-naming choice; behaviour is unchanged except that
+backup and restore now go through :mod:`weightsdb.backup` instead of an application-local module.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from alembic import command
+from alembic.autogenerate import compare_metadata
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.script import ScriptDirectory
+
+from weightsdb.backup import backup as take_backup
+from weightsdb.backup import restore as restore_backup
+from weightsdb.backup import sqlite_path
+from weightsdb.errors import MigrationFailed
+
+if TYPE_CHECKING:
+    from sqlalchemy import Connection, Engine, MetaData
+
+__all__ = ["MigrationOutcome", "MigrationRunner", "ParityResult"]
+
+# The filename family `prune_backups` rotates. An operator-named backup never starts with
+# this, and is therefore never a rotation candidate.
+_PRE_MIGRATION_PREFIX = "pre-migration-"
+
+
+@dataclass(frozen=True, slots=True)
+class MigrationOutcome:
+    """The result of :meth:`MigrationRunner.upgrade` or :meth:`MigrationRunner.downgrade`.
+
+    Attributes:
+        from_revision: The revision before this operation ran, or ``None`` for a fresh database.
+        to_revision: The revision after this operation ran, or ``None`` when a downgrade reached
+            ``base`` (no revision at all — an unmigrated database).
+        backed_up: Whether a backup was taken first.
+        backup_path: Where the backup was written, or ``None`` when ``backed_up`` is ``False``.
+        pruned_backups: Older automatic backups rotated out by this call, oldest first.
+        restore_on_failure_available: Whether a failure of *this* migration would have been rolled
+            back automatically. ``True`` only on SQLite with a backup in hand; ``False`` on
+            PostgreSQL, where the guarantee does not exist (spec §11.4), and ``False`` for a fresh
+            database, where there is nothing to roll back to. This is the field that states the
+            dialect difference rather than papering over it.
+        dialect: ``"sqlite"`` or ``"postgresql"``.
+    """
+
+    from_revision: str | None
+    to_revision: str | None
+    backed_up: bool
+    backup_path: Path | None
+    pruned_backups: tuple[Path, ...]
+    restore_on_failure_available: bool
+    dialect: str
+
+
+@dataclass(frozen=True, slots=True)
+class ParityResult:
+    """The result of :meth:`MigrationRunner.check_parity`.
+
+    Attributes:
+        matches: ``True`` when autogenerate finds no difference between the live schema and the
+            given metadata.
+        diff: A human-readable rendering of the drift found, empty when ``matches`` is ``True``.
+            Alembic's autogenerate comparator does not reliably detect a changed ``CheckConstraint``
+            or a partial (``WHERE``-qualified) index — a known upstream limitation, not something
+            this wrapper works around; a migration that changes only one of those needs its own
+            explicit test, not a mechanical parity check.
+    """
+
+    matches: bool
+    diff: str
+
+
+class MigrationRunner:
+    """Runs one application's own migration history against one engine.
+
+    Stateless beyond its constructor arguments: every method opens its own connection (or is
+    handed one internally) and closes it, so one instance is safely reused across calls but is not
+    itself a resource that needs closing.
+    """
+
+    def __init__(
+        self,
+        engine: Engine,
+        *,
+        script_location: str,
+        version_table: str = "alembic_version",
+        backup_retention: int = 5,
+    ) -> None:
+        """Build a runner for ``engine``'s migration history under ``script_location``.
+
+        Args:
+            engine: The engine migrations run against. Reused as-is — this class never builds its
+                own engine from a bare URL, so the caller's pragmas and transaction semantics
+                (``BEGIN IMMEDIATE`` on SQLite) apply to every migration exactly as they apply to
+                the rest of the application.
+            script_location: Filesystem path to the ``migrations/`` directory containing ``env.py``
+                and ``versions/``.
+            version_table: The table Alembic records the current revision in. Defaults to Alembic's
+                own default; overridable so two applications sharing infrastructure-but-not-schema
+                could coexist without colliding on table name (spec §11.6 — nothing here refers to
+                another application's tables, and a distinct version table per consumer is part of
+                keeping that true even when, as in this package's own test suite, two schemas share
+                one physical database).
+            backup_retention: How many automatic pre-migration backups to keep (database standards
+                §7, default 5). Rotation applies only to this runner's own ``pre-migration-*``
+                files; a backup an operator asked for by name is never rotated.
+        """
+        self._engine = engine
+        self._script_location = script_location
+        self._version_table = version_table
+        self._backup_retention = backup_retention
+        self._script_config = Config()
+        self._script_config.set_main_option("script_location", script_location)
+
+    def current(self) -> str | None:
+        """Return the database's current revision, or ``None`` for an unmigrated database."""
+        with self._engine.connect() as connection:
+            context = MigrationContext.configure(
+                connection, opts={"version_table": self._version_table}
+            )
+            return context.get_current_revision()
+
+    def heads(self) -> tuple[str, ...]:
+        """Return this script directory's head revision(s) — normally exactly one."""
+        script = ScriptDirectory.from_config(self._script_config)
+        return tuple(script.get_heads())
+
+    def known_revisions(self) -> frozenset[str]:
+        """Return every revision ID this script directory's history contains.
+
+        Used to detect :class:`~weightsdb.errors.SchemaAhead`: a database whose current revision is
+        not in this set was written by a build whose migrations this one does not have.
+        """
+        script = ScriptDirectory.from_config(self._script_config)
+        return frozenset(revision.revision for revision in script.walk_revisions())
+
+    def is_at_head(self) -> bool:
+        """Return whether the database's current revision is a head revision."""
+        current = self.current()
+        return current is not None and current in self.heads()
+
+    def upgrade(self, revision: str = "head", *, backup: bool = True) -> MigrationOutcome:
+        """Migrate to ``revision``, taking a backup first (database standards §5.1, §7).
+
+        A no-op when already at ``revision`` — ``command.upgrade`` is itself idempotent, and this
+        method takes no backup and performs no write in that case (CLI standards §11).
+
+        Args:
+            revision: The target revision, or ``"head"``.
+            backup: Take a backup before migrating. Ignored when the database is unmigrated (there
+                is nothing to back up) or when the dialect is PostgreSQL, where the automatic
+                restore-on-failure guarantee does not apply (spec §11.4) and a caller that wants a
+                PostgreSQL backup takes one explicitly.
+
+        Returns:
+            The :class:`MigrationOutcome`.
+
+        Raises:
+            MigrationFailed: The migration raised. On SQLite, the pre-migration backup has already
+                been restored by the time this is raised (``details["restored"] is True``) and the
+                original database is byte-identical. On PostgreSQL, no restore is attempted;
+                ``details`` names the revision actually reached and, if one was taken, the backup to
+                restore from manually.
+        """
+        return self._run(command.upgrade, revision, backup=backup)
+
+    def downgrade(self, revision: str) -> MigrationOutcome:
+        """Migrate down to ``revision``. Same backup and failure semantics as :meth:`upgrade`."""
+        return self._run(command.downgrade, revision, backup=True)
+
+    def _run(self, alembic_command: object, revision: str, *, backup: bool) -> MigrationOutcome:
+        from_revision = self.current()
+        dialect = self._engine.dialect.name
+
+        script = ScriptDirectory.from_config(self._script_config)
+        target_revision = script.as_revision_number(revision)
+        if target_revision == from_revision:
+            # Already there: a genuine no-op (CLI standards §11), not merely "alembic ran and
+            # changed nothing" — no backup is taken and alembic is never invoked, so this is also
+            # the case that keeps `upgrade()` cheap when called opportunistically at every startup.
+            return MigrationOutcome(
+                from_revision=from_revision,
+                to_revision=from_revision,
+                backed_up=False,
+                backup_path=None,
+                pruned_backups=(),
+                restore_on_failure_available=False,
+                dialect=dialect,
+            )
+
+        backed_up = False
+        backup_path: Path | None = None
+        pruned: tuple[Path, ...] = ()
+        if backup and from_revision is not None and dialect == "sqlite":
+            result = take_backup(
+                self._engine,
+                self._pre_migration_backup_path(from_revision),
+                keep=self._backup_retention,
+                prefix=_PRE_MIGRATION_PREFIX,
+            )
+            backed_up = True
+            backup_path = result.path
+            pruned = result.pruned
+
+        try:
+            with self._engine.connect() as connection:
+                config = self._connection_config(connection)
+                alembic_command(config, revision)  # type: ignore[operator]
+        except Exception as exc:
+            restored = False
+            if backed_up and backup_path is not None and dialect == "sqlite":
+                restore_backup(
+                    self._engine,
+                    backup_path,
+                    confirm=True,
+                    known_revisions=self.known_revisions(),
+                )
+                restored = True
+            raise MigrationFailed(
+                f"Migration to {revision!r} failed: {exc}",
+                details={
+                    "restored": restored,
+                    "backup_path": str(backup_path) if backup_path else None,
+                    "reached_revision": self.current(),
+                    "restore_command": None
+                    if restored
+                    else f"restore(engine, {backup_path!r}, confirm=True)"
+                    if backup_path
+                    else None,
+                },
+            ) from exc
+
+        # `to_revision` legitimately ends up `None` here for a downgrade all the way to `base` —
+        # an unmigrated database is not a broken outcome, so nothing above raises for it; a
+        # migration that fails to reach any real revision would have raised inside the `try`
+        # block above instead of returning normally.
+        to_revision = self.current()
+        return MigrationOutcome(
+            from_revision=from_revision,
+            to_revision=to_revision,
+            backed_up=backed_up,
+            backup_path=backup_path,
+            pruned_backups=pruned,
+            restore_on_failure_available=backed_up and dialect == "sqlite",
+            dialect=dialect,
+        )
+
+    def stamp(self, revision: str = "head") -> None:
+        """Mark the database as being at ``revision`` without running any migration.
+
+        For recovering a database whose schema is known to already match a revision — never for
+        routine use, and never called automatically by anything in this module.
+        """
+        with self._engine.connect() as connection:
+            config = self._connection_config(connection)
+            command.stamp(config, revision)
+
+    def check_parity(self, metadata: MetaData) -> ParityResult:
+        """Diff ``metadata`` against the database's live schema (database standards §5.2).
+
+        Independent of Alembic's own revision bookkeeping — this reflects the actual tables,
+        columns and indexes in the database and compares them to ``metadata`` directly, so it
+        catches a model changed without a matching migration even if the ``alembic_version`` table
+        is technically at head.
+
+        Args:
+            metadata: The consumer's own declarative ``MetaData`` — never this package's, which is
+                always empty (spec §7, §20).
+
+        Returns:
+            The :class:`ParityResult`.
+        """
+        with self._engine.connect() as connection:
+            context = MigrationContext.configure(
+                connection, opts={"version_table": self._version_table}
+            )
+            diff = compare_metadata(context, metadata)
+        if not diff:
+            return ParityResult(matches=True, diff="")
+        return ParityResult(matches=False, diff="\n".join(repr(item) for item in diff))
+
+    def _connection_config(self, connection: Connection) -> Config:
+        """Build a per-call :class:`Config` bound to an already-open connection.
+
+        A fresh object every call rather than a cached one: ``config.attributes["connection"]``
+        must point at *this* call's connection, and reusing one :class:`Config` across calls would
+        leak the previous, already-closed connection into the next migration's ``env.py``.
+        """
+        config = Config()
+        config.set_main_option("script_location", self._script_location)
+        config.set_main_option(
+            "sqlalchemy.url", self._engine.url.render_as_string(hide_password=True)
+        )
+        config.attributes["connection"] = connection
+        config.attributes["version_table"] = self._version_table
+        return config
+
+    def _pre_migration_backup_path(self, from_revision: str) -> Path:
+        """Choose ``<db directory>/backups/pre-migration-<revision>-<UTC timestamp>.sqlite3``.
+
+        The revision is part of the name, not only the timestamp (database standards §7): an
+        operator picking a file out of a backups directory needs to know which schema it holds
+        without opening it, and rotation keeps several generations side by side.
+        """
+        source = sqlite_path(self._engine)
+        stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S%fZ")
+        return (
+            source.parent
+            / "backups"
+            / f"{_PRE_MIGRATION_PREFIX}{from_revision}-{stamp}{source.suffix}"
+        )

weightsdb/redaction.py

@@ -0,0 +1,26 @@
+"""weightsdb.redaction — strip credentials from a database URL before it is logged or raised.
+
+Used by every error and log path in this package (spec §14): a connection URL reaches
+:class:`~weightsdb.errors.DatabaseUnavailable`, a health payload or a DEBUG log line only after
+going through :func:`redact_url`, never in its raw form.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy.engine import make_url
+
+__all__ = ["redact_url"]
+
+
+def redact_url(url: str) -> str:
+    """Return ``url`` with any password replaced by ``***``.
+
+    Args:
+        url: A SQLAlchemy database URL, e.g. ``postgresql://user:secret@host/db``.
+
+    Returns:
+        The same URL with its password component masked (``postgresql://user:***@host/db``), or
+        unchanged if it carried no password. Every other component (user, host, database name) is
+        preserved, since those are needed to identify *which* database a message is about.
+    """
+    return make_url(url).render_as_string(hide_password=True)

weightsdb/session.py

@@ -0,0 +1,115 @@
+"""weightsdb.session — session factory, unit-of-work scope, and explicit transaction control.
+
+SQLite's transaction-start behaviour (``BEGIN IMMEDIATE``, so lock contention fails fast rather than
+at commit) is dialect-specific plumbing configured once on the engine by :mod:`weightsdb.engine`;
+nothing here branches on dialect.
+
+Generalized from FreeWeight's ``infrastructure.db.session`` (ADR-0011): the application version
+folded read-only declaration into ``session_scope`` itself via a ``read_only`` flag.  Two consumers
+made that the wrong shape to keep — a caller wants to declare read-only-ness for one sub-block of a
+unit of work, not the whole session — so it is split here into :func:`session_scope` (commit,
+rollback, close: the session's lifecycle) and :func:`transaction` (declares how the transaction
+that lifecycle wraps begins).
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from sqlalchemy import Engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from weightsdb.engine import READ_ONLY_EXECUTION_OPTION
+from weightsdb.errors import DatabaseError
+
+__all__ = ["session_factory", "session_scope", "transaction"]
+
+
+def session_factory(engine: Engine) -> sessionmaker[Session]:
+    """Build a session factory bound to ``engine``.
+
+    ``expire_on_commit=False``: a repository's return value is a detached, plain-data snapshot
+    (coding standards §4 — ORM objects never leave the repository layer), and a caller reading an
+    attribute off it after commit must not trigger a lazy load on a session that may already be
+    closed.
+
+    Args:
+        engine: The engine to bind every session this factory produces to.
+
+    Returns:
+        A ``sessionmaker`` producing sessions against ``engine``.
+    """
+    return sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)
+
+
+@contextmanager
+def session_scope(factory: sessionmaker[Session]) -> Iterator[Session]:
+    """Run one unit of work: commit on success, roll back on any exception, always close.
+
+    "Any exception" includes ``KeyboardInterrupt`` and ``SystemExit`` — a ``Ctrl-C`` mid-write must
+    leave the database in its pre-write state, not a half-committed one, so the ``except`` below is
+    deliberately ``BaseException`` rather than ``Exception``.
+
+    Args:
+        factory: A session factory from :func:`session_factory`.
+
+    Yields:
+        A session open for exactly this unit of work. Wrap it in :func:`transaction` to declare
+        read-only intent for SQLite's benefit; a plain write is the default and needs no wrapping.
+    """
+    session = factory()
+    try:
+        yield session
+        session.commit()
+    except BaseException:
+        session.rollback()
+        raise
+    finally:
+        session.close()
+
+
+@contextmanager
+def transaction(session: Session, *, immediate: bool = True) -> Iterator[Session]:
+    """Declare how ``session``'s transaction begins.
+
+    On SQLite, ``immediate=True`` (the default) uses ``BEGIN IMMEDIATE`` so lock contention fails
+    fast, at the start of the transaction, rather than at commit time (:mod:`weightsdb.engine`'s own
+    docstring explains why that matters). ``immediate=False`` uses a deferred ``BEGIN`` and enforces
+    it: an attempted write inside raises rather than silently taking the write lock. Inert on
+    PostgreSQL, where ordinary MVCC already lets readers and writers proceed without blocking each
+    other.
+
+    This only *declares* the transaction's start semantics — commit, rollback and close remain
+    whichever enclosing :func:`session_scope` is responsible for them. It does not itself commit or
+    roll back, so it composes as a sub-block of one:
+
+    .. code-block:: python
+
+        with session_scope(factory) as session, transaction(session, immediate=False):
+            ...  # read-only work
+
+    Args:
+        session: An open session whose connection has not yet begun a transaction — normally one
+            freshly obtained from :func:`session_scope`, before any query has run on it.
+        immediate: ``True`` for ``BEGIN IMMEDIATE`` (a writer); ``False`` for a deferred ``BEGIN``
+            (a reader).
+
+    Yields:
+        The same session, with its transaction-start semantics declared.
+
+    Raises:
+        DatabaseError: ``session`` already has an active transaction — database standards §6
+            forbids nested transactions, and entering this twice on one session is exactly that.
+    """
+    if session.in_transaction():
+        raise DatabaseError(
+            "transaction() cannot nest: this session already has an active transaction "
+            "(database standards §6 — use a savepoint deliberately if partial rollback is needed)."
+        )
+    if immediate:
+        session.connection()
+    else:
+        read_only_options: dict[str, bool] = {READ_ONLY_EXECUTION_OPTION: True}
+        session.connection(execution_options=read_only_options)
+    yield session

weightsdb/testing.py

@@ -0,0 +1,131 @@
+"""weightsdb.testing — fixtures for a consumer's own test suite, shipped as supported API.
+
+Not a test module itself: importable by any application's tests, and part of this package's public
+contract (spec §7) rather than an internal helper that happens to be reachable.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from sqlalchemy import text
+
+from weightsdb.engine import create_engine_for
+from weightsdb.migrations import MigrationRunner
+
+if TYPE_CHECKING:
+    from sqlalchemy import Engine, MetaData
+
+__all__ = ["MigrationHarness", "migration_harness", "temporary_postgres", "temporary_sqlite"]
+
+_DEFAULT_POSTGRES_URL = "postgresql+psycopg://weightsdb:weightsdb@localhost:5432/weightsdb_test"
+
+
+@contextmanager
+def temporary_sqlite() -> Iterator[Engine]:
+    """Yield an engine on a fresh, empty SQLite database file, disposed and deleted on exit.
+
+    A real file rather than ``:memory:``: backup, restore and the WAL checkpoint all operate on a
+    file on disk, and an in-memory database would silently exempt every one of those from a test
+    that believes it is exercising them.
+    """
+    with tempfile.TemporaryDirectory(prefix="weightsdb-") as directory:
+        url = f"sqlite:///{Path(directory) / 'test.sqlite3'}"
+        engine = create_engine_for(url)
+        try:
+            yield engine
+        finally:
+            engine.dispose()
+
+
+@contextmanager
+def temporary_postgres() -> Iterator[Engine]:
+    """Yield an engine on a freshly reset PostgreSQL schema, disposed on exit.
+
+    Skips (``pytest.skip``) when no server is reachable at ``WEIGHTSDB_POSTGRES_URL`` (default
+    ``postgresql+psycopg://weightsdb:weightsdb@localhost:5432/weightsdb_test``), **except** when
+    ``WEIGHTSDB_REQUIRE_POSTGRES=1``, which turns the skip into a failure: a silently skipped
+    dialect is an untested dialect, and the both-dialects promise (spec §7) is only as good as its
+    enforcement.
+
+    The server is reused across calls, so this resets to empty by dropping and recreating the
+    ``public`` schema — including any ``alembic_version`` table a previous test left behind —
+    rather than assuming a pristine database.
+    """
+    url = os.environ.get("WEIGHTSDB_POSTGRES_URL", _DEFAULT_POSTGRES_URL)
+    require = os.environ.get("WEIGHTSDB_REQUIRE_POSTGRES") == "1"
+    try:
+        probe = create_engine_for(url)
+        try:
+            with probe.connect() as connection:
+                connection.execute(text("SELECT 1"))
+        finally:
+            probe.dispose()
+    except Exception as exc:  # noqa: BLE001 — any failure means "no usable server", by design
+        if require:
+            pytest.fail(f"WEIGHTSDB_REQUIRE_POSTGRES=1 but {url} is unreachable: {exc}")
+        pytest.skip(f"no PostgreSQL server available at {url}: {exc}")
+
+    reset_engine = create_engine_for(url)
+    try:
+        with reset_engine.begin() as connection:
+            connection.execute(text("DROP SCHEMA public CASCADE"))
+            connection.execute(text("CREATE SCHEMA public"))
+    finally:
+        reset_engine.dispose()
+
+    engine = create_engine_for(url)
+    try:
+        yield engine
+    finally:
+        engine.dispose()
+
+
+@dataclass(slots=True)
+class MigrationHarness:
+    """One consumer's migration script location and metadata, ready for its own tests.
+
+    Bundles the two pieces of information every migration test needs so an application's test
+    module does not repeat ``MigrationRunner(engine, script_location=...)`` boilerplate in every
+    test function; :meth:`sqlite` and :meth:`postgres` each combine a fresh :func:`temporary_sqlite`
+    or :func:`temporary_postgres` engine with a :class:`~weightsdb.migrations.MigrationRunner` bound
+    to it.
+
+    Attributes:
+        script_location: Filesystem path to the consumer's ``migrations/`` directory.
+        metadata: The consumer's own declarative ``MetaData``, for ``check_parity``.
+    """
+
+    script_location: str
+    metadata: MetaData
+
+    @contextmanager
+    def sqlite(self) -> Iterator[MigrationRunner]:
+        """A :class:`~weightsdb.migrations.MigrationRunner` on a fresh temporary SQLite database."""
+        with temporary_sqlite() as engine:
+            yield MigrationRunner(engine, script_location=self.script_location)
+
+    @contextmanager
+    def postgres(self) -> Iterator[MigrationRunner]:
+        """A :class:`~weightsdb.migrations.MigrationRunner` on a freshly reset PostgreSQL DB."""
+        with temporary_postgres() as engine:
+            yield MigrationRunner(engine, script_location=self.script_location)
+
+
+def migration_harness(script_location: str, metadata: MetaData) -> MigrationHarness:
+    """Build a :class:`MigrationHarness` for one consumer's migration history.
+
+    Args:
+        script_location: Filesystem path to the consumer's ``migrations/`` directory (containing
+            ``env.py`` and ``versions/``).
+        metadata: The consumer's own declarative ``MetaData`` — never this package's, which is
+            always empty (spec §7, §20).
+    """
+    return MigrationHarness(script_location=script_location, metadata=metadata)