dirigent-core 0.9.0__py3-none-any.whl

dirigent_core/configdocs.py

@@ -0,0 +1,134 @@
+"""Rendering the settings model as the two things a person reads: a page, and an example file.
+
+Both are generated from :class:`~dirigent_core.config.Settings` itself, so a setting added
+without a description fails the build rather than shipping an empty line.
+"""
+
+import json
+import textwrap
+from datetime import timedelta
+from typing import Any, Final
+
+from pydantic_core import PydanticUndefined
+
+from dirigent_common import as_markdown, format_duration
+from dirigent_core.config import Settings
+
+#: Relative to the repository root.
+PAGE_PATH: Final = "docs/settings.md"
+
+BANNER: Final = "<!-- Generated from dirigent_core.config.Settings by dirigent_core.configdocs. Do not edit. -->"
+
+PREAMBLE: Final = """# Settings reference
+
+Every setting an instance has, its default, and what it does. This page is generated from the
+settings model, so it cannot drift from the code: a setting added either shows up here or
+fails the build.
+
+Each one is read from three places, most specific first: a `DIRIGENT_`-prefixed environment
+variable, the YAML configuration file, then the default below. `dg init` writes a
+`dirigent.example.yaml` holding this same list, commented out, beside the working
+`dirigent.yaml` -- so the file a person edits stays short and the whole surface is still
+one file away.
+"""
+
+#: The settings a scaffolded project sets rather than leaves to a default. Everything else
+#: reaches the example file commented out.
+SCAFFOLDED: Final[tuple[str, ...]] = ("environment", "log_level", "worker_concurrency")
+
+EXAMPLE_HEADER: Final = """\
+# Every setting dirigent has, with its default and what it does.
+#
+# This file is read, never loaded: dirigent does not look for it. Copy a line into
+# dirigent.yaml to change it, or set the environment variable named beside it.
+#
+# Generated from the settings model, so it lists what this version actually has.
+"""
+
+PROJECT_HEADER: Final = """\
+# This instance's settings. Everything not named here keeps its default; the whole list,
+# with defaults and descriptions, is in dirigent.example.yaml beside this file.
+"""
+
+
+def environment_name(field: str) -> str:
+    """Name the environment variable one setting is read from."""
+    return f"DIRIGENT_{field.upper()}"
+
+
+def rendered_default(name: str) -> Any:
+    """Read one field's default as the YAML and the table should show it."""
+    field = Settings.model_fields[name]
+    if field.default is not PydanticUndefined:
+        return field.default
+    return field.default_factory() if field.default_factory is not None else None  # type: ignore[call-arg]
+
+
+def described(name: str) -> str:
+    """Read what one field is, as a single line, in the spelling markdown reads.
+
+    The first paragraph only. A docstring says what the setting is and then, often, how to
+    choose a value and what it interacts with; a reference table wants the first of those,
+    and a page of tuning advice inside one cell is unreadable. The rest stays where somebody
+    reading the code will find it.
+    """
+    description = Settings.model_fields[name].description or ""
+    return one_line(first_paragraph(description))
+
+
+def first_paragraph(text: str) -> str:
+    """Take what a description says before its first blank line."""
+    return text.strip().split("\n\n", 1)[0]
+
+
+def one_line(text: str) -> str:
+    """Flatten a description onto one line, in the spelling markdown reads."""
+    return as_markdown(" ".join(text.split()))
+
+
+def _commented(text: str, width: int = 92) -> list[str]:
+    """Wrap a description into comment lines, in the spelling a YAML file reads."""
+    return [f"# {line}" for line in textwrap.wrap(text.replace("`", ""), width=width)] or ["#"]
+
+
+def _yaml_value(value: Any) -> str:
+    """Render one default the way the YAML file writes it."""
+    if value is None:
+        return "null"
+    if isinstance(value, timedelta):
+        return json.dumps(format_duration(value))
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return json.dumps(value)
+    if isinstance(value, (list, dict)):
+        return json.dumps(value)
+    return json.dumps(str(value))
+
+
+def example_document() -> str:
+    """Render the commented reference file, every setting with its default and description."""
+    lines = [EXAMPLE_HEADER]
+    for name in Settings.model_fields:
+        lines.extend(_commented(described(name)))
+        lines.append(f"# {environment_name(name)}")
+        lines.append(f"# {name}: {_yaml_value(rendered_default(name))}")
+        lines.append("")
+    return "\n".join(lines)
+
+
+def project_document(**chosen: Any) -> str:
+    """Render the working configuration file, holding only what a project is likely to change."""
+    values = {name: chosen.get(name, rendered_default(name)) for name in SCAFFOLDED}
+    body = "\n".join(f"{name}: {_yaml_value(value)}" for name, value in values.items())
+    return f"{PROJECT_HEADER}\n{body}\n"
+
+
+def render() -> str:
+    """Render the settings reference page."""
+    rows = "\n".join(
+        f"| `{name}` | `{environment_name(name)}` | `{_yaml_value(rendered_default(name))}` | {described(name)} |"
+        for name in Settings.model_fields
+    )
+    header = "| Setting | Environment | Default | What it does |\n| --- | --- | --- | --- |"
+    return f"{BANNER}\n{PREAMBLE}\n{header}\n{rows}\n"

dirigent_core/database.py

@@ -0,0 +1,177 @@
+"""Async engine and session plumbing."""
+
+import asyncio
+import random
+from collections.abc import AsyncGenerator, Awaitable, Callable
+from contextlib import asynccontextmanager, suppress
+from typing import Any, Final
+
+import sqlalchemy as sa
+import structlog
+from sqlalchemy import event
+from sqlalchemy.exc import DBAPIError
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from dirigent_core.config import Settings
+
+_logger = structlog.get_logger("dirigent.database")
+
+
+#: How long a SQLite connection waits for the write lock before it gives up.
+#:
+#: Every transaction takes the lock at ``BEGIN IMMEDIATE`` and holds it to commit, so this is
+#: how long one caller waits behind the callers ahead of it. A fan-out settling forty items
+#: puts forty short transactions in that queue, and an apply arriving mid-fan-out waits for
+#: all of them; fifteen seconds outlasts that queue on a laptop whose fsync stalls, and is
+#: still short enough that a transaction nothing will ever release fails rather than hangs.
+SQLITE_BUSY_TIMEOUT_MS: Final = 15_000
+
+#: What every SQLite connection is configured with, in order.
+#:
+#: SQLite enforces no foreign key without being asked to, per connection, and the engine
+#: relies on them: an attempt's run and a log entry's attempt are what make an orphan
+#: impossible. WAL is what lets the API, the scheduler and a worker share one file, since a
+#: reader no longer blocks a writer; ``NORMAL`` is the durability that goes with it, losing
+#: at most the last commits to a power cut and nothing to a crashed process. WAL still
+#: allows one writer at a time, and without a busy timeout the second writer fails with
+#: "database is locked" instead of waiting its turn.
+SQLITE_PRAGMAS: Final[tuple[str, ...]] = (
+    "foreign_keys = ON",
+    "journal_mode = WAL",
+    "synchronous = NORMAL",
+    f"busy_timeout = {SQLITE_BUSY_TIMEOUT_MS}",
+)
+
+
+def _configure_sqlite(engine: AsyncEngine) -> None:
+    """Apply the pragmas to every SQLite connection, and open every transaction immediate.
+
+    The busy timeout only covers a connection that waits for the write lock. It does not
+    cover the one case SQLite refuses outright: a transaction that began as a reader and
+    then asks to write while another connection holds the write lock. SQLite fails that
+    upgrade with "database is locked" the instant it is asked, because the reader's snapshot
+    is what the writer ahead of it is invalidating and waiting could only deadlock. An apply
+    reads the pipeline, its versions and the plan before it inserts, which is exactly that
+    shape whenever a worker is claiming or settling an attempt at the same moment.
+
+    ``BEGIN IMMEDIATE`` takes the write lock at the start of the transaction instead, where
+    waiting is safe, so the busy timeout covers it. Every transaction opens that way rather
+    than only the ones that write: which statements a session will run is not known when it
+    begins, and one process is what SQLite is sanctioned for here.
+    """
+
+    @event.listens_for(engine.sync_engine, "connect")
+    def _pragmas(connection: Any, _record: Any) -> None:  # pyright: ignore[reportUnusedFunction] - the event registers it
+        # None hands transaction control to SQLAlchemy: the driver stops emitting its own
+        # deferred BEGIN before a write, and the begin below is what opens every transaction.
+        connection.isolation_level = None
+        cursor = connection.cursor()
+        try:
+            for pragma in SQLITE_PRAGMAS:
+                cursor.execute(f"PRAGMA {pragma}")
+        finally:
+            cursor.close()
+
+    @event.listens_for(engine.sync_engine, "begin")
+    def _immediate(connection: Any) -> None:  # pyright: ignore[reportUnusedFunction] - the event registers it
+        connection.exec_driver_sql("BEGIN IMMEDIATE")
+
+
+def create_engine(settings: Settings) -> AsyncEngine:
+    """Build the async engine for the configured database."""
+    if settings.is_sqlite:
+        ensure_sqlite_directory(settings)
+        engine = create_async_engine(settings.database_url, echo=settings.database_echo, future=True)
+        _configure_sqlite(engine)
+        return engine
+    return create_async_engine(
+        settings.database_url,
+        echo=settings.database_echo,
+        pool_size=settings.database_pool_size,
+        max_overflow=settings.database_max_overflow,
+        pool_timeout=settings.database_pool_timeout.total_seconds(),
+        pool_recycle=int(settings.database_pool_recycle.total_seconds()) if settings.database_pool_recycle else -1,
+        pool_pre_ping=True,
+        future=True,
+    )
+
+
+def ensure_sqlite_directory(settings: Settings) -> None:
+    """Create the directory a SQLite file lives in; the driver opens a file, never a path.
+
+    A directory that cannot be created is left to surface as the connection error it causes.
+    """
+    path = settings.sqlite_path
+    if path is None:
+        return
+    with suppress(OSError):
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+
+def create_lock_engine(settings: Settings) -> AsyncEngine:
+    """Build an engine for a connection that is held, not pooled.
+
+    The scheduler's advisory lock is session-scoped, so its connection is held for the
+    process's lifetime; taking it from the API's pool would permanently consume a slot.
+    """
+    return create_async_engine(settings.database_url, poolclass=NullPool, pool_pre_ping=True, future=True)
+
+
+def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
+    """Build the session factory the engine and the API share."""
+    return async_sessionmaker(engine, expire_on_commit=False, autoflush=False)
+
+
+@asynccontextmanager
+async def session_scope(factory: async_sessionmaker[AsyncSession]) -> AsyncGenerator[AsyncSession]:
+    """Run a unit of work in one transaction: one commit per state transition."""
+    async with factory() as session:
+        try:
+            yield session
+            await session.commit()
+        except BaseException:
+            await session.rollback()
+            raise
+
+
+#: PostgreSQL's SQLSTATE for a deadlock it broke by aborting one of the transactions.
+DEADLOCK: Final = "40P01"
+
+#: How many times a transaction aborted by a deadlock is run again before it is raised.
+DEADLOCK_ATTEMPTS: Final = 3
+
+
+def is_deadlock(error: BaseException) -> bool:
+    """Report whether the database aborted this transaction to break a deadlock."""
+    original = getattr(error, "orig", None)
+    return getattr(original, "sqlstate", None) == DEADLOCK
+
+
+async def with_deadlock_retry[T](work: Callable[[], Awaitable[T]], *, attempts: int = DEADLOCK_ATTEMPTS) -> T:
+    """Run one transaction again when the database aborted it to break a deadlock.
+
+    The claim orders its locks attempt-then-run and an outcome orders them run-then-attempt,
+    which two workers can close into a cycle. Both transactions re-read everything they
+    decide on, so the one PostgreSQL picked off is safe to run again; nothing of it survived
+    the rollback.
+    """
+    for remaining in range(attempts - 1, -1, -1):
+        try:
+            return await work()
+        except DBAPIError as error:
+            if remaining == 0 or not is_deadlock(error):
+                raise
+            _logger.debug("transaction retried after a deadlock", attempts_left=remaining)
+            await asyncio.sleep(random.uniform(0.01, 0.05))
+    raise AssertionError("unreachable: the loop returns or raises")  # pragma: no cover
+
+
+async def ping(engine: AsyncEngine) -> bool:
+    """Report whether the database answers."""
+    try:
+        async with engine.connect() as connection:
+            await connection.execute(sa.text("SELECT 1"))
+    except Exception:
+        return False
+    return True

dirigent_core/directory.py

@@ -0,0 +1,339 @@
+"""Applying a directory of documents: the seed at boot, and the reconcile behind prune.
+
+A DIRECTORY CARRIES SCHEMAS AS WELL AS DOCUMENTS. A plain JSON Schema file is stored the way
+``dg schema create`` stores it, and stored before the pipelines, so a document whose
+``requires.schemas`` names one converges in the same pass.
+
+THE WALK NEVER STOPS FOR ONE DOCUMENT. A directory deployed from git holds many pipelines,
+and a broken one must not keep a server from starting the others: a document that cannot be
+read, carries its own connections or schemas, or fails validation lands in the summary as
+refused, with a warning in the log, and the walk carries on.
+
+PRUNE ONLY EVER TOUCHES WHAT A DIRECTORY APPLIED. A pipeline is deactivated -- never
+deleted, so history survives -- and only when its current version's provenance says a
+directory apply wrote it. A pipeline authored in the UI or applied from a file by hand is
+never pruned, however absent it is. A walk that discovered nothing refuses to prune at all:
+an empty mount is an accident, not an instruction to turn everything off.
+"""
+
+from pathlib import Path
+from typing import Final, cast
+
+import sqlalchemy as sa
+import yaml
+from pydantic import BaseModel
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from dirigent_client.enums import ProvenanceSource
+from dirigent_client.schemas import PlanAction, PruneResult
+from dirigent_common import JsonMap
+from dirigent_core.database import session_scope
+from dirigent_core.documents import DocumentError, carried_refusal, load_text, safe_load
+from dirigent_core.engine.definition import KIND_TRIGGERS
+from dirigent_core.engine.runs import Provenance
+from dirigent_core.engine.services import EngineServices
+from dirigent_core.logging import get_logger
+from dirigent_core.models import Pipeline, PipelineVersion
+from dirigent_core.pipelines import apply_document, set_active
+from dirigent_core.scheduler import release_lead, try_lead
+from dirigent_core.schemas import SchemaRefused, check_valid_schema, find_schema, resolve_identity, store_schema
+from dirigent_core.trigger_documents import delete_absent_trigger_documents
+
+_logger = get_logger(__name__)
+
+#: The suffixes a document may carry, the same set a project's pipelines directory takes.
+DOCUMENT_SUFFIXES: Final = (".yaml", ".yml", ".json")
+
+#: Where a file sorts in the walk, and which applier it goes to.
+_RANK_SCHEMA: Final = 0
+_RANK_PIPELINE: Final = 1
+_RANK_TRIGGERS: Final = 2
+
+#: The keywords a plain JSON Schema is recognised by when it names no ``$schema``.
+_SCHEMA_KEYWORDS: Final = ("type", "properties", "$id")
+
+
+def discover_documents(root: Path) -> list[Path]:
+    """List every document under the directory, however deep, in a stable order.
+
+    A corpus deployed from git nests -- a directory per team, per system, per cadence -- and
+    a walk that stopped at the first level would silently apply half of it. The order is the
+    relative path, so two runs of one tree agree.
+    """
+    if not root.is_dir():
+        return []
+    return sorted(
+        (path for path in root.rglob("*") if path.suffix in DOCUMENT_SUFFIXES and path.is_file()),
+        key=lambda path: path.relative_to(root).as_posix(),
+    )
+
+
+class RefusedDocument(BaseModel):
+    """One document the walk could not apply, and the first reason why."""
+
+    path: str
+    code: str | None = None
+    message: str
+
+
+class DirectorySummary(BaseModel):
+    """What one walk of the directory did, list by list."""
+
+    applied: list[str] = []
+    updated: list[str] = []
+    unchanged: list[str] = []
+    refused: list[RefusedDocument] = []
+    pruned: list[str] = []
+    trigger_documents_removed: list[str] = []
+    """The triggers documents the directory no longer holds, deleted with the rows they owned."""
+
+    schemas: list[str] = []
+    """Which of the codes above are schemas rather than pipelines, so a reader can tell them apart."""
+
+    prune_refusal: str | None = None
+    """Why nothing was pruned when pruning was asked for, or nothing at all."""
+
+    skipped: bool = False
+    """Another process held the apply lock, so it is doing this work."""
+
+    aborted: str | None = None
+    """Why the walk stopped early: an instance fault, which no list of refusals should wear."""
+
+
+async def prune_absent(session: AsyncSession, keep: set[str], *, dry_run: bool = False) -> PruneResult:
+    """Reconcile against a directory: deactivate absent pipelines, delete absent triggers documents.
+
+    The narrowing is by the CURRENT version's provenance: what a directory applied last, a
+    directory may take away. Deactivation pauses nothing retroactively beyond what
+    deactivation always means, and history survives untouched. A triggers document has no
+    history worth keeping, so it is deleted instead, and the cascade takes the schedules and
+    webhooks it owned -- never the pipeline's own inline ones. A schema is never pruned.
+    """
+    rows = await session.execute(
+        sa.select(Pipeline.code)
+        .join(
+            PipelineVersion,
+            sa.and_(
+                PipelineVersion.pipeline_id == Pipeline.id,
+                PipelineVersion.version == Pipeline.current_version,
+            ),
+        )
+        .where(
+            Pipeline.active.is_(True),
+            PipelineVersion.provenance_source == ProvenanceSource.DIRECTORY,
+            Pipeline.code.notin_(keep) if keep else sa.true(),
+        )
+        .order_by(Pipeline.code)
+    )
+    absent = [str(code) for (code,) in rows.all()]
+    documents = await delete_absent_trigger_documents(session, keep, dry_run=dry_run)
+    if not dry_run:
+        for code in absent:
+            await set_active(session, code, active=False)
+    return PruneResult(pruned=absent, trigger_documents_removed=documents, dry_run=dry_run)
+
+
+async def apply_directory(
+    sessions: async_sessionmaker[AsyncSession],
+    services: EngineServices,
+    root: Path,
+    *,
+    prune: bool = False,
+    dry_run: bool = False,
+    pause_schedules: bool = False,
+    lock_key: int | None = None,
+) -> DirectorySummary:
+    """Apply every schema and document in the directory, then reconcile when asked to.
+
+    Each one is applied in its own transaction, so one that fails rolls back alone.
+    With a ``lock_key``, the whole walk runs under the same PostgreSQL advisory lock the
+    scheduler uses for leadership: several servers booting together elect one applier, and
+    the others skip with a log line rather than racing it.
+    """
+    summary = DirectorySummary()
+    async with session_scope(sessions) as holder:
+        if lock_key is not None and not await try_lead(holder, lock_key):
+            _logger.info("directory apply skipped: another process holds the lock", root=str(root))
+            summary.skipped = True
+            return summary
+        try:
+            await _walk(sessions, services, root, summary, dry_run=dry_run, pause_schedules=pause_schedules)
+            if prune:
+                await _prune(sessions, summary, dry_run=dry_run)
+        except SQLAlchemyError as error:
+            summary.aborted = str(error).splitlines()[0]
+            _logger.error("directory apply aborted by an instance fault", root=str(root), error=summary.aborted)
+        finally:
+            if lock_key is not None:
+                await release_lead(holder, lock_key)
+    return summary
+
+
+async def _walk(
+    sessions: async_sessionmaker[AsyncSession],
+    services: EngineServices,
+    root: Path,
+    summary: DirectorySummary,
+    *,
+    dry_run: bool,
+    pause_schedules: bool,
+) -> None:
+    """Apply each discovered file into the summary, refusing without stopping.
+
+    Schemas go first, pipeline documents after them, and triggers documents last, so a
+    directory carrying all three converges in one pass: a pipeline is refused while a schema
+    its ``requires`` names is absent, and a triggers document is refused while the pipeline it
+    names is absent. Applying each ahead of what needs it is what makes it present.
+    """
+    documents = discover_documents(root)
+    if not documents:
+        _logger.warning("directory apply found no documents", root=str(root))
+        return
+    walked: dict[str, str] = {}
+    for rank, path in sorted(((_walk_rank(path), path) for path in documents), key=lambda pair: pair[0]):
+        if rank == _RANK_SCHEMA:
+            await _apply_schema(sessions, root, path, summary, dry_run=dry_run)
+        else:
+            await _apply_one(
+                sessions, services, root, path, summary, walked, dry_run=dry_run, pause_schedules=pause_schedules
+            )
+
+
+def _walk_rank(path: Path) -> int:
+    """Peek at a file once to place it in the walk, without holding it against the format.
+
+    A ``.json`` file that parses to an object carrying neither ``format`` nor ``kind`` is a
+    plain JSON Schema when it names ``$schema``, or failing that any of the keywords a schema
+    is written with. One that cannot be read at all sorts with the pipelines, so it is refused
+    where its own path says it should be.
+    """
+    try:
+        raw = safe_load(path.read_text())
+    except (OSError, ValueError, yaml.YAMLError):
+        return _RANK_PIPELINE
+    if not isinstance(raw, dict):
+        return _RANK_PIPELINE
+    body = cast("JsonMap", raw)
+    if body.get("kind") == KIND_TRIGGERS:
+        return _RANK_TRIGGERS
+    unenveloped = path.suffix == ".json" and "format" not in body and "kind" not in body
+    if unenveloped and ("$schema" in body or any(keyword in body for keyword in _SCHEMA_KEYWORDS)):
+        return _RANK_SCHEMA
+    return _RANK_PIPELINE
+
+
+async def _apply_schema(
+    sessions: async_sessionmaker[AsyncSession],
+    root: Path,
+    path: Path,
+    summary: DirectorySummary,
+    *,
+    dry_run: bool,
+) -> None:
+    """Store one JSON Schema the directory carries, in its own transaction."""
+    ref = path.relative_to(root).as_posix()
+    try:
+        body = cast("JsonMap", safe_load(path.read_text()))
+        check_valid_schema(body)
+        code, name, description = resolve_identity(
+            body, code=None, name=None, description=None, fallback_code=path.stem
+        )
+    except (OSError, ValueError, yaml.YAMLError, SchemaRefused) as error:
+        summary.refused.append(RefusedDocument(path=ref, message=str(error)))
+        _logger.warning("directory schema refused", path=ref, error=str(error))
+        return
+    async with session_scope(sessions) as session:
+        existing = await find_schema(session, code)
+        if existing is None:
+            summary.applied.append(code)
+        elif existing.body == body and existing.name == name and existing.description == description:
+            summary.unchanged.append(code)
+        else:
+            summary.updated.append(code)
+        if not dry_run:
+            await store_schema(session, body, fallback_code=path.stem)
+    summary.schemas.append(code)
+
+
+async def _apply_one(
+    sessions: async_sessionmaker[AsyncSession],
+    services: EngineServices,
+    root: Path,
+    path: Path,
+    summary: DirectorySummary,
+    walked: dict[str, str],
+    *,
+    dry_run: bool,
+    pause_schedules: bool,
+) -> None:
+    """Apply one document into the summary."""
+    ref = path.relative_to(root).as_posix()
+    try:
+        definition = load_text(path.read_text())
+    except (OSError, DocumentError) as error:
+        summary.refused.append(RefusedDocument(path=ref, message=str(error)))
+        _logger.warning("directory document refused", path=ref, error=str(error))
+        return
+    if definition.code in walked:
+        message = f"code '{definition.code}' was already applied from {walked[definition.code]} in this walk"
+        summary.refused.append(RefusedDocument(path=ref, code=definition.code, message=message))
+        _logger.warning("directory document refused", path=ref, pipeline=definition.code, error=message)
+        return
+    walked[definition.code] = ref
+    carried = carried_refusal(definition)
+    if carried is not None:
+        summary.refused.append(RefusedDocument(path=ref, code=definition.code, message=carried))
+        _logger.warning("directory document refused", path=ref, pipeline=definition.code, error=carried)
+        return
+    try:
+        async with session_scope(sessions) as session:
+            result = await apply_document(
+                session,
+                services,
+                definition,
+                provenance=Provenance(source=ProvenanceSource.DIRECTORY, ref=ref),
+                dry_run=dry_run,
+                pause_schedules=pause_schedules,
+            )
+    except SQLAlchemyError:
+        # A database fault is the instance's, not this document's: refusing 59 documents
+        # one by one is how a schema drift hides. The walk stops and says so once.
+        raise
+    except Exception as error:  # noqa: BLE001 - one broken document must not stop the walk
+        summary.refused.append(RefusedDocument(path=ref, code=definition.code, message=str(error)))
+        _logger.warning("directory document refused", path=ref, pipeline=definition.code, error=str(error))
+        return
+    plan = result.plan
+    if plan.action is PlanAction.INVALID:
+        first = plan.issues[0].message if plan.issues else "invalid"
+        summary.refused.append(RefusedDocument(path=ref, code=plan.code, message=first))
+        _logger.warning("directory document refused", path=ref, pipeline=plan.code, error=first)
+    elif plan.action is PlanAction.CREATE:
+        summary.applied.append(plan.code)
+    elif plan.action is PlanAction.UPDATE:
+        summary.updated.append(plan.code)
+    else:
+        summary.unchanged.append(plan.code)
+
+
+async def _prune(sessions: async_sessionmaker[AsyncSession], summary: DirectorySummary, *, dry_run: bool) -> None:
+    """Reconcile after the walk: deactivate what the directory no longer holds.
+
+    Schema codes are dropped from what is kept: they name no pipeline, and no schema is pruned.
+    """
+    codes = {
+        *summary.applied,
+        *summary.updated,
+        *summary.unchanged,
+        *(refused.code for refused in summary.refused if refused.code is not None),
+    }
+    if not codes:
+        summary.prune_refusal = "the directory held no documents, and an empty directory never prunes"
+        _logger.warning("directory prune refused", reason=summary.prune_refusal)
+        return
+    keep = codes - set(summary.schemas)
+    async with session_scope(sessions) as session:
+        result = await prune_absent(session, keep, dry_run=dry_run)
+    summary.pruned = result.pruned
+    summary.trigger_documents_removed = result.trigger_documents_removed