lorebase-store 0.2.3__py3-none-any.whl

lorebase_store/bundle.py

@@ -0,0 +1,1830 @@
+"""Portable Domain bundles: export a Domain to a directory or tarball, import it elsewhere.
+
+A bundle holds the versioned content of one Domain: every Revision reachable from a ref with its
+parents and changeset, the tree nodes and objects those Revisions reach, the refs and their
+log, the provenance entries, and a readable `schemas/<branch>.yaml` per branch head. It does
+not hold the audit log, principals, grants, proposals, documents, vault or embeddings (see
+docs/operations.md). Materialized heads are not stored: import rebuilds them from the trees.
+
+Layout (the `.jsonl` names carry `.gz` when `store.bundle.compression` is gzip)::
+
+    manifest.json      format version, Lorebase version, counts, heads, sha256 per file
+    domain.json        the Domain row
+    refs.json          refs (kind, head, version, protection, materialization, materialized)
+    ref_log.jsonl      head movements of every ref
+    revisions.jsonl    id, root, generation, parents, author, message, origin, stats, created_at
+    changesets.jsonl   element changes of each Revision
+    provenance.jsonl   provenance entries
+    nodes.jsonl        tree nodes: hash and base64 body
+    objects.jsonl      objects: hash, type and canonical body
+    vault.jsonl        only with vault values: token, binding, value or erased (see below)
+    vault_lineage.jsonl     only with vault values: key changes of vault Records
+    vault_tombstones.jsonl  only with vault values: fingerprints of erased values
+    schemas/<ref>.yaml schema at the head of each branch (informational)
+
+Revisions are ordered by generation, so a parent always precedes its children. Everything is
+written and read in batches: memory is bounded by `store.bundle.batch_rows` and by one tree
+level while marking. Export reads one repeatable-read snapshot. Import verifies the sha256 of
+every file, recomputes every Revision id, node hash and object hash, checks that every
+reachable node and object is present, and runs in one transaction: a failing import leaves
+nothing behind.
+
+Revision ids include the Domain id, so a bundle can only be imported under the id it was
+exported with.
+
+Values of vault Attributes travel as the references Record objects hold: by default the vault
+is not exported, so after an import they read as unknown (masked), and import refuses a Record
+whose vault Attribute holds anything but a well-formed reference (design 13.5).
+
+An export with `include_vault_values` also writes the vault sections
+(`lorebase_store.vault.transfer`): for every token the exported Record objects reference, the
+Kind, Attribute and Record key its vault row binds it to and the opened value, or `erased`; the
+Domain's key changes (`vault_lineage`); and its tombstones with the token key they were made
+under. The files are listed in the manifest with their sha256 like the others, and objects keep
+their hashes. Such an export needs the vault keys and holds personal data in plain text (files
+and tarball are created readable by their owner only). It appends `bundle.export_plain_started`
+to the audit log before reading any value and `bundle.export_plain` (Domain, heads and counts,
+never values) when done; when either cannot be written the output is removed. Its snapshot is
+taken while the Domain's vault lock is held shared, so no erasure is in flight when it is taken;
+the lock is let go once the snapshot exists, before any file is written, and an erasure starting
+meanwhile waits at most `vault.erase_lock_timeout_ms`. Entries carry the row's `moved_to`.
+Tombstones recorded before they kept their `fingerprint_key` are exported as made under the
+current token key with the first scheme, and the manifest says how many (`vault_notes`). The
+audit actor is what the caller gives: the command runs with the owner's database credentials and
+vault keys, and is as trustworthy as whoever holds them.
+
+Import needs the vault keys and the token key every tombstone was made under (by id). It stores
+the key changes (each between two Records of its Kind the bundle holds) and tombstones, then
+each entry again, sealed under the target's keys, after checking that every Record object
+holding its token holds it in that Attribute under the bound key or one linked to it by the key
+changes, that its `moved_to` follows a key change from its key, and that the value is valid for
+a definition of the Attribute in the bundle's schemas. A value a tombstone holds is handled as
+`vault.erased_values` says; erased entries stay erased. A `bundle.import_vault` audit entry
+records the counts (values, erased, tombstones, key changes, groups of more than
+`MAX_LINKED_KEYS` linked keys) in the same transaction. A target Domain that already has vault
+rows, tombstones or key changes is not empty.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import base64
+import gzip
+import hashlib
+import importlib.metadata
+import itertools
+import json
+import logging
+import os
+import shutil
+import stat
+import sys
+import tarfile
+import tempfile
+import time
+from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
+from contextlib import AsyncExitStack, contextmanager
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path, PurePosixPath
+from typing import IO, Any, BinaryIO, Protocol
+from urllib.parse import quote
+
+from sqlalchemy import ARRAY, LargeBinary, bindparam, text
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
+
+from lorebase_core.canonical import ObjectType, from_hex, node_hash
+from lorebase_core.canonical import hex as to_hex
+from lorebase_core.model import Attribute, Kind, dump_schema, vault_token
+from lorebase_core.revision import (
+    Author,
+    Origin,
+    Revision,
+    RevisionStats,
+    format_timestamp,
+    next_generation,
+    revision_id,
+)
+from lorebase_core.tree import decode_node
+from lorebase_store.auth import AuditRepository, NewAuditEntry
+from lorebase_store.bulk import OnConflict, bulk_write
+from lorebase_store.gc import Marker
+from lorebase_store.materialize import rebuild, verify
+from lorebase_store.materialize.rebuild import load_schema
+from lorebase_store.models.enums import Materialization, RefKind
+from lorebase_store.nodestore import NodeRepository
+from lorebase_store.objectstore import OBJECT_TYPES_BY_CODE, PgObjectStore, encode_object
+from lorebase_store.repositories.changesets import CHANGESETS
+from lorebase_store.repositories.domains import DomainNotFoundError, DomainRepository, DomainRow
+from lorebase_store.repositories.provenance import PROVENANCE
+from lorebase_store.repositories.refs import REF_LOG, REFS, RefRepository
+from lorebase_store.repositories.revisions import REVISION_PARENTS, REVISIONS, RevisionRepository
+from lorebase_store.repositories.scope import scope_to_domain
+from lorebase_store.runtime import StoreRuntime, default_runtime
+from lorebase_store.settings import BundleSettings
+from lorebase_store.vault import VaultKeys
+from lorebase_store.vault.store import MAX_LINKED_KEYS
+from lorebase_store.vault.transfer import (
+    LEGACY_SCHEME,
+    Holder,
+    KeyLinks,
+    LineageEntry,
+    TombstoneEntry,
+    VaultEntry,
+    VaultTransferError,
+    holders_problem,
+    lineage_of,
+    parse_entry,
+    parse_lineage,
+    parse_tombstone,
+    read_entries,
+    resolved_legacy,
+    restore_entries,
+    restore_lineage,
+    restore_tombstones,
+    tombstones_of,
+    unknown_entry,
+    value_problem,
+)
+
+log = logging.getLogger(__name__)
+
+FORMAT = "lorebase-domain-bundle"
+FORMAT_VERSION = 1
+MANIFEST = "manifest.json"
+SCHEMAS_DIR = "schemas"
+TAR_SUFFIX = ".tar"
+VAULT = "vault"
+VAULT_LINEAGE = "vault_lineage"
+VAULT_TOMBSTONES = "vault_tombstones"
+_VAULT_SECTIONS = (VAULT, VAULT_LINEAGE, VAULT_TOMBSTONES)
+EXPORT_PLAIN_STARTED_ACTION = "bundle.export_plain_started"
+"""Audit action written before an export holding vault values reads any value."""
+EXPORT_PLAIN_ACTION = "bundle.export_plain"
+"""Audit action of a finished export holding vault values."""
+IMPORT_VAULT_ACTION = "bundle.import_vault"
+"""Audit action of an import of vault values (counts, never values)."""
+_VAULT_LOCK_SHARED = text(
+    "SELECT pg_advisory_xact_lock_shared(hashtextextended('lorebase.vault/' || :domain, 0))"
+)
+BUNDLE_ACTOR = "system:bundle"
+"""Default audit actor of the bundle command."""
+PLAIN_TEXT_WARNING = (
+    "warning: {path} holds personal data in plain text (vault values); keep it as such and "
+    "delete it once imported\n"
+)
+
+_JSONL = ("revisions", "changesets", "provenance", "ref_log", "nodes", "objects")
+_PLAIN = ("domain.json", "refs.json")
+_HASH_CHUNK = 1 << 20
+_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
+_DIRECTORY = getattr(os, "O_DIRECTORY", 0)
+_NONBLOCK = getattr(os, "O_NONBLOCK", 0)
+
+
+class BundleError(Exception):
+    """The bundle cannot be written, is malformed, or fails verification."""
+
+
+class DomainNotEmptyError(BundleError):
+    """The target Domain already has Revisions or refs."""
+
+
+class VaultNotConfiguredError(BundleError):
+    """Vault values are asked for or carried, but the vault keys are not configured."""
+
+
+@dataclass(slots=True)
+class BundleReport:
+    """What an export or import moved."""
+
+    domain_id: str
+    path: str
+    revisions: int = 0
+    changesets: int = 0
+    provenance: int = 0
+    nodes: int = 0
+    objects: int = 0
+    refs: int = 0
+    ref_log: int = 0
+    schemas: int = 0
+    vault_values: int = 0
+    vault_erased: int = 0
+    vault_lineage: int = 0
+    vault_legacy_tombstones: int = 0
+    vault_oversized_links: int = 0
+    vault_tombstones: int = 0
+    rebuilt_branches: list[str] = field(default_factory=list)
+    seconds: float = 0.0
+
+    def as_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+# --------------------------------------------------------------------------------------
+# Files
+
+
+def _jsonl_name(base: str, compression: str) -> str:
+    return f"{base}.jsonl.gz" if compression == "gzip" else f"{base}.jsonl"
+
+
+def _dumps(value: Any) -> bytes:
+    return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode()
+
+
+def _create(path: Path) -> BinaryIO:
+    """A new file only its owner may read or write."""
+    return os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | _NOFOLLOW, 0o600), "wb")
+
+
+def _sha256(path: Path) -> tuple[str, int]:
+    digest = hashlib.sha256()
+    size = 0
+    with path.open("rb") as handle:
+        while chunk := handle.read(_HASH_CHUNK):
+            digest.update(chunk)
+            size += len(chunk)
+    return digest.hexdigest(), size
+
+
+class _Output:
+    """The files of a bundle being written into a directory."""
+
+    def __init__(self, root: Path, compression: str, level: int) -> None:
+        self.root = root
+        self.compression = compression
+        self.level = level
+        self.files: dict[str, dict[str, Any]] = {}
+
+    @contextmanager
+    def jsonl(self, base: str, *, private: bool = False) -> Iterator[_LineWriter]:
+        """Lines of `base`; `private` creates the file readable by its owner only."""
+        name = _jsonl_name(base, self.compression)
+        path = self.root / name
+        raw = _create(path) if private else path.open("wb")
+        stream: _Sink = raw
+        if self.compression == "gzip":
+            stream = gzip.GzipFile(
+                filename="", mode="wb", fileobj=raw, compresslevel=self.level, mtime=0
+            )
+        writer = _LineWriter(stream)
+        try:
+            yield writer
+        finally:
+            stream.close()
+            raw.close()
+        self._register(name, path, writer.records)
+
+    def plain(self, name: str, content: bytes, records: int | None = None) -> None:
+        path = self.root / name
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_bytes(content)
+        self._register(name, path, records)
+
+    def _register(self, name: str, path: Path, records: int | None) -> None:
+        digest, size = _sha256(path)
+        entry: dict[str, Any] = {"sha256": digest, "bytes": size}
+        if records is not None:
+            entry["records"] = records
+        self.files[name] = entry
+
+
+class _Sink(Protocol):
+    def write(self, data: bytes, /) -> Any: ...
+
+    def close(self) -> None: ...
+
+
+class _LineWriter:
+    def __init__(self, stream: _Sink) -> None:
+        self._stream = stream
+        self.records = 0
+
+    def write(self, value: Any) -> None:
+        self._stream.write(_dumps(value) + b"\n")
+        self.records += 1
+
+
+def _read_jsonl(root: Path, name: str, limits: BundleSettings) -> Iterator[dict[str, Any]]:
+    """The records of a `.jsonl` file, read line by line within the store.bundle limits."""
+    with _open_file(root, name) as raw:
+        handle: gzip.GzipFile | BinaryIO = (
+            gzip.GzipFile(fileobj=raw, mode="rb") if name.endswith(".gz") else raw
+        )
+        size = 0
+        number = 0
+        try:
+            while line := handle.readline(limits.max_line_bytes + 1):
+                number += 1
+                size += len(line)
+                _check_size(name, size, 0, limits)
+                if len(line) > limits.max_line_bytes and not line.endswith(b"\n"):
+                    raise BundleError(
+                        f"{name} line {number} is longer than store.bundle.max_line_bytes "
+                        f"({limits.max_line_bytes})"
+                    )
+                try:
+                    value = json.loads(line)
+                except ValueError as error:
+                    raise BundleError(f"{name} line {number}: not JSON") from error
+                if not isinstance(value, dict):
+                    raise BundleError(f"{name} line {number}: not an object")
+                yield value
+        except (OSError, EOFError) as error:
+            raise BundleError(f"{name} cannot be read: {error}") from error
+        finally:
+            handle.close()
+
+
+def _batches[T](items: Iterable[T], size: int) -> Iterator[tuple[T, ...]]:
+    iterator = iter(items)
+    while batch := tuple(itertools.islice(iterator, size)):
+        yield batch
+
+
+# --------------------------------------------------------------------------------------
+# Export
+
+
+def _bytes(name: str) -> Any:
+    return bindparam(name, type_=ARRAY(LargeBinary()))
+
+
+_REVISIONS = text(
+    """
+    SELECT r.id, r.root, r.generation, r.author, r.on_behalf_of, r.message, r.origin, r.stats,
+           r.created_at
+    FROM revisions r JOIN lb_gc_revs m ON m.id = r.id
+    WHERE r.domain_id = :d AND (r.generation, r.id) > (:g, :i)
+    ORDER BY r.generation, r.id LIMIT :n
+    """
+)
+_PARENTS = text(
+    "SELECT revision_id, parent_id FROM revision_parents "
+    "WHERE domain_id = :d AND revision_id = ANY(CAST(:ids AS bytea[])) "
+    "ORDER BY revision_id, position"
+).bindparams(_bytes("ids"))
+_CHANGESETS = text(
+    """
+    SELECT c.revision_id, c.partition, c.element_key, c.old_hash, c.new_hash, c.provenance
+    FROM changesets c JOIN lb_gc_revs m ON m.id = c.revision_id
+    WHERE c.domain_id = :d AND (c.revision_id, c.partition, c.element_key) > (:r, :p, :k)
+    ORDER BY c.revision_id, c.partition, c.element_key LIMIT :n
+    """
+)
+_PROVENANCE = text(
+    "SELECT id, origin_type, origin_ref, actor, at FROM provenance "
+    "WHERE domain_id = :d AND id > :last ORDER BY id LIMIT :n"
+)
+_REF_LOG = text(
+    "SELECT ref_name, seq, old_head, new_head, reason, actor, at FROM ref_log "
+    "WHERE domain_id = :d AND (ref_name, seq) > (:r, :s) ORDER BY ref_name, seq LIMIT :n"
+)
+_NODES = text(
+    "SELECT n.hash, n.body FROM tree_nodes n JOIN lb_gc_nodes m ON m.hash = n.hash "
+    "WHERE n.domain_id = :d AND n.hash > :last ORDER BY n.hash LIMIT :n"
+)
+_OBJECTS = text(
+    "SELECT o.hash, o.object_type, o.body FROM objects o JOIN lb_gc_objects m ON m.hash = o.hash "
+    "WHERE o.domain_id = :d AND o.hash > :last ORDER BY o.hash LIMIT :n"
+)
+_REFS = text(
+    """
+    SELECT r.name, r.kind::text AS kind, r.head, r.protected, r.materialization::text AS
+           materialization, r.version, r.created_at, r.created_by,
+           EXISTS (SELECT 1 FROM mat_schema s WHERE s.domain_id = r.domain_id
+                   AND s.branch = r.name) AS materialized
+    FROM refs r WHERE r.domain_id = :d AND r.deleted_at IS NULL AND r.kind <> 'snapshot'
+    ORDER BY r.name
+    """
+)
+
+
+def _hex(value: bytes | None) -> str | None:
+    return None if value is None else to_hex(bytes(value))
+
+
+def _timestamp(value: datetime) -> str:
+    return format_timestamp(value)
+
+
+async def export_domain(
+    engine: AsyncEngine,
+    domain_id: str,
+    destination: str | Path,
+    *,
+    runtime: StoreRuntime | None = None,
+    include_vault_values: bool = False,
+    actor: str = BUNDLE_ACTOR,
+) -> BundleReport:
+    """Write a bundle of `domain_id` to `destination`: a new directory, or a `.tar` file.
+
+    Reads one consistent snapshot; commits made meanwhile are not part of the bundle.
+    `include_vault_values` also writes the opened vault values in plain text and appends an
+    audit entry for `actor` (module doc); it raises `VaultNotConfiguredError` without the
+    vault keys. When such an export fails, nothing of it is left.
+    """
+    runtime = runtime or default_runtime()
+    keys = runtime.vault if include_vault_values else None
+    if include_vault_values and keys is None:
+        raise VaultNotConfiguredError(
+            "exporting vault values needs vault.token_key and vault.encryption_key"
+        )
+    target = Path(destination)
+    started = time.monotonic()
+    as_tar = target.suffix == TAR_SUFFIX
+    _check_destination(target, as_tar)
+    if keys is not None:
+        try:
+            await _audit(engine, actor, EXPORT_PLAIN_STARTED_ACTION, domain_id, {})
+        except BaseException:
+            if not as_tar:
+                shutil.rmtree(target, ignore_errors=True)
+            raise
+    if as_tar:
+        with tempfile.TemporaryDirectory(dir=target.parent, prefix=".bundle-") as work:
+            report, heads = await _export(engine, domain_id, Path(work), runtime, str(target), keys)
+            if keys is not None:
+                await _audit_plain_export(engine, report, heads, actor)
+            _pack(Path(work), target, private=keys is not None)
+    else:
+        try:
+            report, heads = await _export(engine, domain_id, target, runtime, str(target), keys)
+            if keys is not None:
+                await _audit_plain_export(engine, report, heads, actor)
+        except BaseException:
+            if keys is not None:
+                shutil.rmtree(target, ignore_errors=True)
+            raise
+    report.seconds = time.monotonic() - started
+    return report
+
+
+async def _audit_plain_export(
+    engine: AsyncEngine, report: BundleReport, heads: Mapping[str, str], actor: str
+) -> None:
+    """The audit entry of a finished export holding vault values: counts, never values."""
+    details = {
+        "heads": dict(heads),
+        "values": report.vault_values,
+        "erased": report.vault_erased,
+        "lineage": report.vault_lineage,
+        "tombstones": report.vault_tombstones,
+    }
+    await _audit(engine, actor, EXPORT_PLAIN_ACTION, report.domain_id, details)
+
+
+async def _audit(
+    engine: AsyncEngine, actor: str, action: str, domain_id: str, details: Mapping[str, Any]
+) -> None:
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    async with sessions() as session, session.begin():
+        await AuditRepository(session).append(
+            NewAuditEntry(
+                actor=actor, action=action, domain_id=domain_id, target=domain_id, details=details
+            )
+        )
+
+
+def _check_destination(target: Path, as_tar: bool) -> None:
+    if as_tar:
+        if target.exists():
+            raise BundleError(f"{target} exists")
+        target.parent.mkdir(parents=True, exist_ok=True)
+    else:
+        if target.exists() and (not target.is_dir() or any(target.iterdir())):
+            raise BundleError(f"{target} exists and is not an empty directory")
+        target.mkdir(parents=True, exist_ok=True)
+
+
+def _pack(work: Path, target: Path, *, private: bool = False) -> None:
+    """Pack `work` into `target`; `private` makes it readable by its owner only."""
+    partial = target.with_name(target.name + ".partial")
+    try:
+        with (
+            _create(partial) if private else partial.open("wb") as sink,
+            tarfile.open(fileobj=sink, mode="w") as archive,
+        ):
+            for path in sorted(work.rglob("*")):
+                if path.is_file():
+                    archive.add(path, arcname=path.relative_to(work).as_posix())
+    except BaseException:
+        partial.unlink(missing_ok=True)
+        raise
+    partial.rename(target)
+
+
+async def _export(
+    engine: AsyncEngine,
+    domain_id: str,
+    root: Path,
+    runtime: StoreRuntime,
+    label: str,
+    keys: VaultKeys | None,
+) -> tuple[BundleReport, dict[str, str]]:
+    settings = runtime.settings
+    batch = settings.bundle.batch_rows
+    out = _Output(root, settings.bundle.compression, settings.bundle.compression_level)
+    report = BundleReport(domain_id, label)
+    sessions = async_sessionmaker(engine, expire_on_commit=False)
+    async with AsyncExitStack() as stack:
+        guard = None
+        if keys is not None:
+            # The vault lock is held shared from another connection while the snapshot below is
+            # taken: an erasure in flight ends first, and one starting meanwhile waits (at most
+            # vault.erase_lock_timeout_ms). The snapshot fixes what is read; the lock is let go
+            # as soon as it exists, before any file is written.
+            guard = await stack.enter_async_context(engine.connect())
+            await guard.execute(_VAULT_LOCK_SHARED, {"domain": domain_id})
+        session = await stack.enter_async_context(sessions())
+        await stack.enter_async_context(session.begin())
+        await session.execute(text("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"))
+        await scope_to_domain(session, domain_id)  # the first query: it takes the snapshot
+        if guard is not None:
+            await guard.close()
+        domain = await DomainRepository(session).get(domain_id)
+        if domain is None:
+            raise DomainNotFoundError(domain_id)
+        refs = [dict(row) for row in (await session.execute(_REFS, {"d": domain_id})).mappings()]
+        marker = Marker(session, domain_id, batch=settings.gc.mark_batch)
+        await marker.prepare()
+        await marker.mark(attempts=False)
+        marked = await marker.stats()
+        if marked.missing_nodes:
+            raise BundleError(f"{marked.missing_nodes} reachable tree nodes are missing")
+
+        out.plain("domain.json", _dumps(_domain_json(domain)))
+        out.plain("refs.json", _dumps([_ref_json(ref) for ref in refs]), len(refs))
+        report.refs = len(refs)
+        report.revisions = await _export_revisions(session, out, domain_id, batch)
+        report.changesets = await _export_changesets(session, out, domain_id, batch)
+        report.provenance = await _export_provenance(session, out, domain_id, batch)
+        report.ref_log = await _export_ref_log(session, out, domain_id, batch)
+        report.nodes = await _export_nodes(session, out, domain_id, batch)
+        holders: dict[str, set[Holder]] | None = None if keys is None else {}
+        report.objects = await _export_objects(session, out, domain_id, batch, holders)
+        for name, got, want in (
+            ("revisions", report.revisions, marked.revisions),
+            ("nodes", report.nodes, marked.nodes),
+            ("objects", report.objects, marked.objects),
+        ):
+            if got != want:
+                raise BundleError(f"expected {want} {name} but the store holds {got}")
+        report.schemas = await _export_schemas(session, out, domain_id, refs, runtime)
+        if keys is not None and holders is not None:
+            await _export_vault(session, out, domain_id, batch, holders, keys, report)
+        await marker.drop()
+
+    heads = {ref["name"]: to_hex(bytes(ref["head"])) for ref in refs}
+    counts = {
+        "revisions": report.revisions,
+        "changesets": report.changesets,
+        "provenance": report.provenance,
+        "ref_log": report.ref_log,
+        "nodes": report.nodes,
+        "objects": report.objects,
+        "refs": report.refs,
+    }
+    if keys is not None:
+        counts[VAULT] = report.vault_values + report.vault_erased
+        counts[VAULT_LINEAGE] = report.vault_lineage
+        counts[VAULT_TOMBSTONES] = report.vault_tombstones
+    manifest = {
+        "format": FORMAT,
+        "format_version": FORMAT_VERSION,
+        "lorebase_version": _lorebase_version(),
+        "domain_id": domain_id,
+        "created_at": format_timestamp(datetime.now(UTC)),
+        "compression": out.compression,
+        "heads": heads,
+        "counts": counts,
+        "files": out.files,
+    }
+    if report.vault_legacy_tombstones:
+        manifest["vault_notes"] = {
+            "legacy_tombstones": report.vault_legacy_tombstones,
+            "legacy_fingerprint_key": f"{keys.token_key_id}/{LEGACY_SCHEME}" if keys else None,
+            "note": "tombstones recorded without a fingerprint_key, exported as made under the "
+            "current token key with the first fingerprint scheme",
+        }
+    (root / MANIFEST).write_bytes(json.dumps(manifest, indent=2, sort_keys=True).encode() + b"\n")
+    return report, heads
+
+
+def _lorebase_version() -> str:
+    try:
+        return importlib.metadata.version("lorebase-store")
+    except importlib.metadata.PackageNotFoundError:
+        return "unknown"
+
+
+def _domain_json(domain: DomainRow) -> dict[str, Any]:
+    return {
+        "id": domain.id,
+        "display_name": domain.display_name,
+        "description": domain.description,
+        "default_branch": domain.default_branch,
+        "leaf_size": domain.leaf_size,
+        "settings": domain.settings,
+        "created_at": _timestamp(domain.created_at),
+        "created_by": domain.created_by,
+    }
+
+
+def _ref_json(ref: Mapping[str, Any]) -> dict[str, Any]:
+    return {
+        "name": ref["name"],
+        "kind": ref["kind"],
+        "head": to_hex(bytes(ref["head"])),
+        "protected": ref["protected"],
+        "materialization": ref["materialization"],
+        "version": ref["version"],
+        "created_at": _timestamp(ref["created_at"]),
+        "created_by": ref["created_by"],
+        "materialized": ref["materialized"],
+    }
+
+
+async def _export_revisions(session: AsyncSession, out: _Output, domain: str, batch: int) -> int:
+    after: tuple[int, bytes] = (0, b"")
+    with out.jsonl("revisions") as lines:
+        while True:
+            rows = (
+                await session.execute(
+                    _REVISIONS, {"d": domain, "g": after[0], "i": after[1], "n": batch}
+                )
+            ).all()
+            if not rows:
+                return lines.records
+            ids = [bytes(row.id) for row in rows]
+            parents: dict[bytes, list[str]] = {}
+            for revision, parent in await session.execute(_PARENTS, {"d": domain, "ids": ids}):
+                parents.setdefault(bytes(revision), []).append(to_hex(bytes(parent)))
+            for row in rows:
+                lines.write(
+                    {
+                        "id": to_hex(bytes(row.id)),
+                        "root": to_hex(bytes(row.root)),
+                        "generation": row.generation,
+                        "parents": parents.get(bytes(row.id), []),
+                        "author": row.author,
+                        "on_behalf_of": row.on_behalf_of,
+                        "message": row.message,
+                        "origin": row.origin,
+                        "stats": row.stats,
+                        "created_at": _timestamp(row.created_at),
+                    }
+                )
+            after = (int(rows[-1].generation), bytes(rows[-1].id))
+
+
+async def _export_changesets(session: AsyncSession, out: _Output, domain: str, batch: int) -> int:
+    after: tuple[bytes, str, str] = (b"", "", "")
+    with out.jsonl("changesets") as lines:
+        while True:
+            rows = (
+                await session.execute(
+                    _CHANGESETS,
+                    {"d": domain, "r": after[0], "p": after[1], "k": after[2], "n": batch},
+                )
+            ).all()
+            if not rows:
+                return lines.records
+            for row in rows:
+                lines.write(
+                    {
+                        "revision": to_hex(bytes(row.revision_id)),
+                        "partition": row.partition,
+                        "key": row.element_key,
+                        "old": _hex(row.old_hash),
+                        "new": _hex(row.new_hash),
+                        "provenance": row.provenance,
+                    }
+                )
+            last = rows[-1]
+            after = (bytes(last.revision_id), last.partition, last.element_key)
+
+
+async def _export_provenance(session: AsyncSession, out: _Output, domain: str, batch: int) -> int:
+    last = ""
+    with out.jsonl("provenance") as lines:
+        while True:
+            rows = (
+                await session.execute(_PROVENANCE, {"d": domain, "last": last, "n": batch})
+            ).all()
+            if not rows:
+                return lines.records
+            for row in rows:
+                lines.write(
+                    {
+                        "id": row.id,
+                        "origin_type": row.origin_type,
+                        "origin_ref": row.origin_ref,
+                        "actor": row.actor,
+                        "at": _timestamp(row.at),
+                    }
+                )
+            last = rows[-1].id
+
+
+async def _export_ref_log(session: AsyncSession, out: _Output, domain: str, batch: int) -> int:
+    after: tuple[str, int] = ("", 0)
+    with out.jsonl("ref_log") as lines:
+        while True:
+            rows = (
+                await session.execute(
+                    _REF_LOG, {"d": domain, "r": after[0], "s": after[1], "n": batch}
+                )
+            ).all()
+            if not rows:
+                return lines.records
+            for row in rows:
+                lines.write(
+                    {
+                        "ref": row.ref_name,
+                        "seq": row.seq,
+                        "old": _hex(row.old_head),
+                        "new": to_hex(bytes(row.new_head)),
+                        "reason": row.reason,
+                        "actor": row.actor,
+                        "at": _timestamp(row.at),
+                    }
+                )
+            after = (rows[-1].ref_name, int(rows[-1].seq))
+
+
+async def _export_nodes(session: AsyncSession, out: _Output, domain: str, batch: int) -> int:
+    last = b""
+    with out.jsonl("nodes") as lines:
+        while True:
+            rows = (await session.execute(_NODES, {"d": domain, "last": last, "n": batch})).all()
+            if not rows:
+                return lines.records
+            for row in rows:
+                lines.write(
+                    {
+                        "hash": to_hex(bytes(row.hash)),
+                        "body": base64.b64encode(bytes(row.body)).decode("ascii"),
+                    }
+                )
+            last = bytes(rows[-1].hash)
+
+
+async def _export_objects(
+    session: AsyncSession,
+    out: _Output,
+    domain: str,
+    batch: int,
+    holders: dict[str, set[Holder]] | None = None,
+) -> int:
+    """Write the marked objects; with `holders`, collect the vault tokens of Record objects."""
+    last = b""
+    with out.jsonl("objects") as lines:
+        while True:
+            rows = (await session.execute(_OBJECTS, {"d": domain, "last": last, "n": batch})).all()
+            if not rows:
+                return lines.records
+            for row in rows:
+                object_type = OBJECT_TYPES_BY_CODE[row.object_type]
+                line = {
+                    "hash": to_hex(bytes(row.hash)),
+                    "type": object_type.value,
+                    "body": row.body,
+                }
+                if holders is not None and object_type is ObjectType.RECORD:
+                    _collect_holders(line, holders)
+                lines.write(line)
+            last = bytes(rows[-1].hash)
+
+
+def _collect_holders(line: Mapping[str, Any], holders: dict[str, set[Holder]]) -> None:
+    """Add the Record of a Record object as a holder of each vault token it holds."""
+    body = line.get("body") or {}
+    values = body.get("values") or {}
+    if not isinstance(values, dict):
+        return
+    for attribute_id, value in values.items():
+        token = vault_token(value)
+        if token is not None:
+            holders.setdefault(token, set()).add(
+                Holder(str(body.get("kind")), str(attribute_id), str(body.get("key")))
+            )
+
+
+async def _export_vault(
+    session: AsyncSession,
+    out: _Output,
+    domain: str,
+    batch: int,
+    holders: Mapping[str, set[Holder]],
+    keys: VaultKeys,
+    report: BundleReport,
+) -> None:
+    """Write the key changes, the tombstones and the opened value (or `erased`) of every token
+    the exported objects hold, bound as its vault row is. Every Record holding a token must
+    hold it in the bound Attribute under the bound key or one linked to it (a key change on an
+    erased reference keeps the token)."""
+    lineage = await lineage_of(session, domain)
+    links = KeyLinks()
+    with out.jsonl(VAULT_LINEAGE, private=True) as lines:
+        for item in lineage:
+            links.link(item)
+            lines.write(item.as_json())
+    report.vault_lineage = len(lineage)
+    with out.jsonl(VAULT_TOMBSTONES, private=True) as lines:
+        async for found in tombstones_of(session, domain, batch):
+            for tombstone in found:
+                if tombstone.fingerprint_key is None:
+                    report.vault_legacy_tombstones += 1
+                lines.write(resolved_legacy(tombstone, keys.token_key_id).as_json())
+            report.vault_tombstones += len(found)
+    tokens = sorted(holders)
+    with out.jsonl(VAULT, private=True) as lines:
+        for start in range(0, len(tokens), batch):
+            chunk = tokens[start : start + batch]
+            try:
+                stored = await read_entries(session, domain, chunk, keys)
+            except VaultTransferError as error:
+                raise BundleError(str(error)) from error
+            for token in chunk:
+                entry = stored.get(token) or unknown_entry(token, holders[token])
+                problem = holders_problem(
+                    token,
+                    entry.kind_id,
+                    entry.attribute_id,
+                    entry.record_key,
+                    holders[token],
+                    links,
+                )
+                if problem is not None:
+                    raise BundleError(problem)
+                if entry.erased:
+                    report.vault_erased += 1
+                else:
+                    report.vault_values += 1
+                lines.write(entry.as_json())
+
+
+async def _export_schemas(
+    session: AsyncSession,
+    out: _Output,
+    domain_id: str,
+    refs: Sequence[Mapping[str, Any]],
+    runtime: StoreRuntime,
+) -> int:
+    settings = runtime.settings
+    nodes = NodeRepository(
+        session, domain_id, writes=settings.writes, batch=settings.nodes.prefetch_batch
+    )
+    revisions = RevisionRepository(session, domain_id)
+    count = 0
+    for ref in refs:
+        if ref["kind"] != RefKind.BRANCH.value:
+            continue
+        head = await revisions.require(bytes(ref["head"]))
+        schema, _ = await load_schema(
+            session, domain_id, await nodes.load_tree(head.root), runtime=runtime
+        )
+        name = f"{SCHEMAS_DIR}/{quote(ref['name'], safe='')}.yaml"
+        out.plain(name, dump_schema(schema).encode())
+        count += 1
+    return count
+
+
+# --------------------------------------------------------------------------------------
+# Import
+
+
+def _check_size(name: str, size: int, total: int, limits: BundleSettings) -> None:
+    if size > limits.max_member_bytes:
+        raise BundleError(
+            f"{name} is larger than store.bundle.max_member_bytes ({limits.max_member_bytes})"
+        )
+    if total > limits.max_total_bytes:
+        raise BundleError(
+            f"the bundle exceeds store.bundle.max_total_bytes ({limits.max_total_bytes}) at {name}"
+        )
+
+
+def _check_entries(name: str, entries: int, limits: BundleSettings) -> None:
+    if entries > limits.max_entries:
+        raise BundleError(
+            f"the bundle has more entries than store.bundle.max_entries ({limits.max_entries}) "
+            f"at {name}"
+        )
+
+
+class _Meter:
+    """Bytes read from the files of one bundle, checked against the size limits as they come."""
+
+    def __init__(self, limits: BundleSettings) -> None:
+        self.limits = limits
+        self.total = 0
+
+    def chunks(self, name: str, source: IO[bytes]) -> Iterator[bytes]:
+        size = 0
+        while chunk := source.read(_HASH_CHUNK):
+            size += len(chunk)
+            self.total += len(chunk)
+            _check_size(name, size, self.total, self.limits)
+            yield chunk
+
+
+def _open_file(root: Path, name: str) -> BinaryIO:
+    """Open a regular file of a bundle directory without following a link at any level."""
+    parts = PurePosixPath(name).parts
+    try:
+        folder = os.open(root, os.O_RDONLY | _DIRECTORY)
+        try:
+            for part in parts[:-1]:
+                inner = os.open(
+                    part, os.O_RDONLY | _DIRECTORY | _NOFOLLOW | _NONBLOCK, dir_fd=folder
+                )
+                os.close(folder)
+                folder = inner
+            descriptor = os.open(parts[-1], os.O_RDONLY | _NOFOLLOW | _NONBLOCK, dir_fd=folder)
+        finally:
+            os.close(folder)
+    except OSError as error:
+        raise BundleError(
+            f"{name} cannot be opened (links are not followed): {error.strerror}"
+        ) from error
+    if not stat.S_ISREG(os.fstat(descriptor).st_mode):
+        os.close(descriptor)
+        raise BundleError(f"{name} is not a regular file")
+    return os.fdopen(descriptor, "rb")
+
+
+def _read_document(root: Path, name: str, limits: BundleSettings) -> Any:
+    """A whole `.json` file of a bundle, at most `store.bundle.max_line_bytes` long."""
+    with _open_file(root, name) as handle:
+        data = handle.read(limits.max_line_bytes + 1)
+    if len(data) > limits.max_line_bytes:
+        raise BundleError(
+            f"{name} is larger than store.bundle.max_line_bytes ({limits.max_line_bytes})"
+        )
+    try:
+        return json.loads(data)
+    except ValueError as error:
+        raise BundleError(f"{name} is not valid JSON") from error
+
+
+def _member_parts(member: tarfile.TarInfo, archive: str) -> tuple[str, ...]:
+    """The path of a tarball member, refused unless it is a file or directory of the layout."""
+    label = f"{member.name!r} in {archive}"
+    if member.issym():
+        raise BundleError(f"{label} is a symbolic link")
+    if member.islnk():
+        raise BundleError(f"{label} is a hard link")
+    if not (member.isreg() or member.isdir()):
+        raise BundleError(f"{label} is not a regular file or directory")
+    path = PurePosixPath(member.name)
+    if path.is_absolute():
+        raise BundleError(f"{label} has an absolute path")
+    if ".." in path.parts:
+        raise BundleError(f"{label} leaves the bundle directory")
+    parts = path.parts
+    expected = (
+        parts == (SCHEMAS_DIR,)
+        if member.isdir()
+        else len(parts) == 1 or (len(parts) == 2 and parts[0] == SCHEMAS_DIR)
+    )
+    if not expected:
+        raise BundleError(f"unexpected entry {label}")
+    return parts
+
+
+def _extract(tarball: Path, into: Path, limits: BundleSettings) -> None:
+    """Unpack a bundle tarball: only files and directories with the bundle's names, within
+    the store.bundle entry and size limits, counting the bytes actually read."""
+    meter = _Meter(limits)
+    entries = 0
+    declared = 0
+    try:
+        with tarfile.open(tarball, "r:") as archive:
+            for member in archive:
+                entries += 1
+                label = f"{member.name!r} in {tarball.name}"
+                _check_entries(label, entries, limits)
+                parts = _member_parts(member, tarball.name)
+                target = into.joinpath(*parts)
+                if member.isdir():
+                    target.mkdir(exist_ok=True)
+                    continue
+                declared += member.size
+                _check_size(label, member.size, declared, limits)
+                source = archive.extractfile(member)
+                if source is None:
+                    raise BundleError(f"{label} cannot be read")
+                target.parent.mkdir(exist_ok=True)
+                try:
+                    descriptor = os.open(
+                        target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | _NOFOLLOW, 0o600
+                    )
+                except FileExistsError as error:
+                    raise BundleError(f"{label} appears twice") from error
+                with source, os.fdopen(descriptor, "wb") as sink:
+                    for chunk in meter.chunks(label, source):
+                        sink.write(chunk)
+    except tarfile.TarError as error:
+        raise BundleError(f"{tarball.name} is not a readable tar file: {error}") from error
+    except OSError as error:
+        raise BundleError(f"{tarball.name} cannot be unpacked: {error.strerror}") from error
+
+
+def _scan(root: Path, limits: BundleSettings) -> set[str]:
+    """The files of a bundle directory. Links and special files are refused, never followed."""
+    files: set[str] = set()
+    entries = 0
+    total = 0
+    pending = [PurePosixPath()]
+    while pending:
+        folder = pending.pop()
+        with os.scandir(root.joinpath(*folder.parts)) as listing:
+            for entry in listing:
+                relative = folder / entry.name
+                name = relative.as_posix()
+                entries += 1
+                _check_entries(name, entries, limits)
+                if entry.is_symlink():
+                    raise BundleError(f"{name} is a symbolic link")
+                if entry.is_dir(follow_symlinks=False):
+                    pending.append(relative)
+                    continue
+                if not entry.is_file(follow_symlinks=False):
+                    raise BundleError(f"{name} is not a regular file or directory")
+                size = entry.stat(follow_symlinks=False).st_size
+                total += size
+                _check_size(name, size, total, limits)
+                files.add(name)
+    return files
+
+
+@contextmanager
+def _opened(source: Path, limits: BundleSettings) -> Iterator[Path]:
+    if source.is_dir():
+        yield source
+    elif source.is_file() and source.suffix == TAR_SUFFIX:
+        with tempfile.TemporaryDirectory(prefix="lorebase-bundle-") as work:
+            _extract(source, Path(work), limits)
+            yield Path(work)
+    else:
+        raise BundleError(f"{source} is neither a bundle directory nor a {TAR_SUFFIX} file")
+
+
+def read_manifest(root: Path, limits: BundleSettings | None = None) -> dict[str, Any]:
+    """The manifest of an opened bundle directory, checked for format and version."""
+    limits = limits or BundleSettings()
+    if not os.path.lexists(root / MANIFEST):
+        raise BundleError(f"{MANIFEST} not found")
+    manifest = _read_document(root, MANIFEST, limits)
+    if not isinstance(manifest, dict):
+        raise BundleError(f"{MANIFEST} is not a JSON object")
+    if manifest.get("format") != FORMAT:
+        raise BundleError(f"not a Lorebase Domain bundle (format {manifest.get('format')!r})")
+    version = manifest.get("format_version")
+    if version != FORMAT_VERSION:
+        raise BundleError(
+            f"bundle format version {version!r} is not supported (this release reads "
+            f"{FORMAT_VERSION})"
+        )
+    for key in ("domain_id", "compression", "counts", "files", "heads"):
+        if key not in manifest:
+            raise BundleError(f"{MANIFEST} lacks {key!r}")
+    return manifest
+
+
+def _verify_files(root: Path, manifest: Mapping[str, Any], limits: BundleSettings) -> None:
+    files: Mapping[str, Mapping[str, Any]] = manifest["files"]
+    present = _scan(root, limits) - {MANIFEST}
+    expected = set(files)
+    if present != expected:
+        raise BundleError(
+            f"files differ from the manifest: missing {sorted(expected - present)}, "
+            f"unexpected {sorted(present - expected)}"
+        )
+    meter = _Meter(limits)
+    for name, entry in files.items():
+        digest = hashlib.sha256()
+        size = 0
+        with _open_file(root, name) as handle:
+            for chunk in meter.chunks(name, handle):
+                digest.update(chunk)
+                size += len(chunk)
+        if digest.hexdigest() != entry["sha256"] or size != entry["bytes"]:
+            raise BundleError(f"{name} does not match its sha256 in the manifest")
+
+
+@dataclass(slots=True)
+class _Import:
+    session: AsyncSession
+    root: Path
+    manifest: Mapping[str, Any]
+    domain_id: str
+    runtime: StoreRuntime
+    batch: int
+    report: BundleReport
+    actor: str = BUNDLE_ACTOR
+
+    def lines(self, base: str) -> Iterator[dict[str, Any]]:
+        name = _jsonl_name(base, self.manifest["compression"])
+        return _read_jsonl(self.root, name, self.runtime.settings.bundle)
+
+    def document(self, name: str) -> Any:
+        return _read_document(self.root, name, self.runtime.settings.bundle)
+
+    def expect(self, name: str, got: int) -> None:
+        want = self.manifest["counts"][name]
+        if got != want:
+            raise BundleError(f"{name}: manifest says {want}, bundle holds {got}")
+
+
+async def import_bundle(
+    engine: AsyncEngine,
+    source: str | Path,
+    *,
+    domain_id: str | None = None,
+    runtime: StoreRuntime | None = None,
+    verify_heads: bool | None = None,
+    actor: str = BUNDLE_ACTOR,
+) -> BundleReport:
+    """Import a bundle (directory or `.tar`) into `domain_id`, which must not exist yet or be
+    empty (no Revisions, no refs). It defaults to, and must equal, the bundle's Domain id.
+    A bundle carrying vault values appends a `bundle.import_vault` audit entry for `actor` in
+    the same transaction.
+
+    Every file hash, Revision id, node hash and object hash is verified, materialized heads are
+    rebuilt for the refs that had one, and with `verify_heads` (default
+    `store.bundle.verify_after_import`) each rebuilt branch is checked against its tree. All
+    in one transaction. Raises `BundleError` (the transaction is rolled back).
+
+    Reading is bounded by the `store.bundle` limits (entries, file and total sizes, line
+    length); links and special files are refused. The content itself is trusted: authors,
+    provenance, the ref log, changesets and Domain settings are imported as written.
+    """
+    runtime = runtime or default_runtime()
+    limits = runtime.settings.bundle
+    started = time.monotonic()
+    path = Path(source)
+    with _opened(path, limits) as root:
+        manifest = read_manifest(root, limits)
+        target = domain_id or manifest["domain_id"]
+        if target != manifest["domain_id"]:
+            raise BundleError(
+                f"the bundle is of Domain {manifest['domain_id']!r}; Revision ids include the "
+                f"Domain id, so it cannot be imported as {target!r}"
+            )
+        _verify_files(root, manifest, limits)
+        settings = runtime.settings
+        report = BundleReport(target, str(path))
+        sessions = async_sessionmaker(engine, expire_on_commit=False)
+        async with sessions() as session, session.begin():
+            await scope_to_domain(session, target)
+            context = _Import(
+                session, root, manifest, target, runtime, settings.bundle.batch_rows, report, actor
+            )
+            await _prepare_domain(context)
+            scanned = await _import_objects(context)
+            if scanned is not None:
+                await _import_vault(context, scanned)
+            await _import_nodes(context)
+            await _import_revisions(context)
+            await _import_changesets(context)
+            await _import_provenance(context)
+            refs = await _import_refs(context)
+            await _import_ref_log(context)
+            await _check_integrity(context)
+            await _materialize(
+                context,
+                refs,
+                settings.bundle.verify_after_import if verify_heads is None else verify_heads,
+            )
+    report.seconds = time.monotonic() - started
+    return report
+
+
+async def _prepare_domain(c: _Import) -> None:
+    domains = DomainRepository(c.session)
+    data = c.document("domain.json")
+    if data["id"] != c.domain_id:
+        raise BundleError("domain.json names a different Domain than the manifest")
+    if await domains.get(c.domain_id) is not None:
+        counts = (
+            await c.session.execute(
+                text(
+                    "SELECT (SELECT count(*) FROM revisions WHERE domain_id = :d), "
+                    "(SELECT count(*) FROM refs WHERE domain_id = :d), "
+                    "(SELECT count(*) FROM vault WHERE domain_id = :d), "
+                    "(SELECT count(*) FROM vault_tombstones WHERE domain_id = :d), "
+                    "(SELECT count(*) FROM vault_lineage WHERE domain_id = :d)"
+                ),
+                {"d": c.domain_id},
+            )
+        ).one()
+        if any(counts):
+            raise DomainNotEmptyError(
+                f"Domain {c.domain_id!r} already has {counts[0]} Revisions, {counts[1]} refs, "
+                f"{counts[2]} vault rows, {counts[3]} vault tombstones and {counts[4]} vault "
+                "key changes"
+            )
+        return
+    await domains.create(
+        DomainRow(
+            id=data["id"],
+            display_name=data["display_name"],
+            description=data["description"],
+            default_branch=data["default_branch"],
+            leaf_size=data["leaf_size"],
+            settings=data["settings"],
+            created_at=datetime.fromisoformat(data["created_at"]),
+            created_by=data["created_by"],
+        )
+    )
+
+
+def _vault_attributes(c: _Import) -> dict[str, set[str]]:
+    """Kind id -> ids of Attributes that are vault Attributes in any schema of the bundle."""
+    found: dict[str, set[str]] = {}
+    for line in c.lines("objects"):
+        if line.get("type") != ObjectType.SCHEMA_KIND.value:
+            continue
+        body = line.get("body") or {}
+        attributes = body.get("attributes") or {}
+        ids = {k for k, a in attributes.items() if isinstance(a, dict) and a.get("vault") is True}
+        if ids:
+            found.setdefault(str(body.get("id")), set()).update(ids)
+    return found
+
+
+def _check_vault_values(line: dict[str, Any], vaulted: dict[str, set[str]]) -> None:
+    """A Record's vault Attributes hold references only: plain values would never be
+    erasable."""
+    body = line.get("body") or {}
+    values = body.get("values") or {}
+    for attribute_id in vaulted.get(str(body.get("kind")), ()):
+        value = values.get(attribute_id)
+        if value is not None and vault_token(value) is None:
+            raise BundleError(
+                f"object {line.get('hash')}: vault attribute {attribute_id} holds a value, "
+                "not a vault reference"
+            )
+
+
+def _carries_vault(c: _Import) -> bool:
+    """Whether the bundle holds vault values: the vault, key change and tombstone sections are
+    all listed, each with its count, or none is."""
+    found = set()
+    for section in _VAULT_SECTIONS:
+        listed = _jsonl_name(section, c.manifest["compression"]) in c.manifest["files"]
+        if listed != (section in c.manifest["counts"]):
+            raise BundleError(f"{MANIFEST} lists the {section} file and its count inconsistently")
+        found.add(listed)
+    if len(found) != 1:
+        raise BundleError(f"{MANIFEST} lists some vault sections but not all")
+    return found.pop()
+
+
+def _vault_definitions(c: _Import) -> dict[tuple[str, str], list[Attribute]]:
+    """(Kind id, Attribute id) -> the distinct definitions of each vault Attribute in the
+    bundle's schemas."""
+    found: dict[tuple[str, str], dict[str, Attribute]] = {}
+    for line in c.lines("objects"):
+        if line.get("type") != ObjectType.SCHEMA_KIND.value:
+            continue
+        try:
+            kind = Kind.model_validate(line.get("body"))
+        except ValueError as error:
+            raise BundleError(f"object {line.get('hash')}: {error}") from error
+        for attribute in kind.attributes.values():
+            if attribute.in_vault:
+                definitions = found.setdefault((kind.id, attribute.id), {})
+                definitions.setdefault(attribute.model_dump_json(), attribute)
+    return {key: list(definitions.values()) for key, definitions in found.items()}
+
+
+@dataclass(slots=True)
+class _VaultScan:
+    """What the Record objects of a bundle carrying vault values show: the Records holding
+    each token, and every (Kind id, key) of a Record in any Revision."""
+
+    holders: dict[str, set[Holder]] = field(default_factory=dict)
+    records: set[tuple[str, str]] = field(default_factory=set)
+
+
+async def _import_objects(c: _Import) -> _VaultScan | None:
+    """Store the objects; when the bundle carries vault values, what its Record objects show
+    (`_VaultScan`)."""
+    store = PgObjectStore(c.session, c.domain_id, writes=c.runtime.settings.writes, batch=c.batch)
+    vaulted = _vault_attributes(c)
+    scan = _VaultScan() if _carries_vault(c) else None
+    total = 0
+    for batch in _batches(c.lines("objects"), c.batch):
+        encoded = []
+        for line in batch:
+            if line.get("type") == ObjectType.RECORD.value:
+                if vaulted:
+                    _check_vault_values(line, vaulted)
+                if scan is not None:
+                    _collect_holders(line, scan.holders)
+                    body = line.get("body") or {}
+                    scan.records.add((str(body.get("kind")), str(body.get("key"))))
+            try:
+                item = encode_object(ObjectType(line["type"]), line["body"])
+            except (ValueError, KeyError, TypeError) as error:
+                raise BundleError(f"object {line.get('hash')}: {error}") from error
+            if to_hex(item.hash) != line["hash"]:
+                raise BundleError(f"object {line['hash']} does not match its content")
+            encoded.append(item)
+        await store.put_many(encoded)
+        total += len(batch)
+    c.report.objects = total
+    c.expect("objects", total)
+    return scan
+
+
+async def _import_vault(c: _Import, scan: _VaultScan) -> None:
+    """Store the key changes, the tombstones and the vault entries (module doc), and append
+    the `bundle.import_vault` audit entry (counts, never values).
+
+    A key change must link two Records of its Kind that the bundle holds. Each entry must be
+    well formed, appear once, hold a value valid for a definition of its Attribute in the
+    bundle's schemas, and every Record holding its token must hold it in that Attribute under
+    its key or a key linked to it by the key changes; its `moved_to` must follow a key change
+    from its key. Values are checked against the tombstones (the bundle's) as
+    `vault.erased_values` says. Groups of more than `MAX_LINKED_KEYS` linked keys are counted
+    for an administrator to look at, as erasure flags them.
+    """
+    holders = scan.holders
+    keys = c.runtime.vault
+    if keys is None:
+        raise VaultNotConfiguredError(
+            "the bundle holds vault values; importing them needs vault.token_key and "
+            "vault.encryption_key"
+        )
+    at = datetime.now(UTC)
+    links, edges = await _import_lineage(c, scan.records, at)
+    c.report.vault_oversized_links = links.oversized(MAX_LINKED_KEYS)
+    await _import_tombstones(c, keys, at)
+    definitions = _vault_definitions(c)
+    seen: set[str] = set()
+    for batch in _batches(enumerate(c.lines(VAULT), start=1), c.batch):
+        entries: list[VaultEntry] = []
+        for number, line in batch:
+            try:
+                entry = parse_entry(line)
+            except VaultTransferError as error:
+                raise BundleError(f"vault line {number}: {error}") from error
+            problem = holders_problem(
+                entry.token,
+                entry.kind_id,
+                entry.attribute_id,
+                entry.record_key,
+                holders.get(entry.token, ()),
+                links,
+            )
+            if problem is None and entry.moved_to is not None:
+                edge = LineageEntry(entry.kind_id, entry.moved_to, entry.record_key)
+                if edge not in edges:
+                    problem = (
+                        f"{entry.token} moved to {entry.moved_to!r}, but no key change of the "
+                        f"bundle leads there from {entry.record_key!r}"
+                    )
+            if problem is None and not entry.erased:
+                problem = value_problem(
+                    entry, definitions.get((entry.kind_id, entry.attribute_id), [])
+                )
+            if problem is not None:
+                raise BundleError(f"vault line {number}: {problem}")
+            if entry.token in seen:
+                raise BundleError(f"vault line {number}: {entry.token} appears twice")
+            seen.add(entry.token)
+            entries.append(entry)
+            if entry.erased:
+                c.report.vault_erased += 1
+            else:
+                c.report.vault_values += 1
+        try:
+            tombstoned = await restore_entries(
+                c.session,
+                c.domain_id,
+                entries,
+                keys,
+                erased_values=c.runtime.vault_settings.erased_values,
+                at=at,
+            )
+        except VaultTransferError as error:
+            raise BundleError(str(error)) from error
+        c.report.vault_values -= tombstoned
+        c.report.vault_erased += tombstoned
+    c.expect(VAULT, len(seen))
+    report = c.report
+    await AuditRepository(c.session).append(
+        NewAuditEntry(
+            actor=c.actor,
+            action=IMPORT_VAULT_ACTION,
+            domain_id=c.domain_id,
+            target=c.domain_id,
+            details={
+                "values": report.vault_values,
+                "erased": report.vault_erased,
+                "tombstones": report.vault_tombstones,
+                "lineage_edges": report.vault_lineage,
+                "oversized_lineage_groups": report.vault_oversized_links,
+            },
+            at=at,
+        )
+    )
+
+
+async def _import_lineage(
+    c: _Import, records: Collection[tuple[str, str]], at: datetime
+) -> tuple[KeyLinks, set[LineageEntry]]:
+    """The key changes, each between two Records of its Kind that the bundle holds."""
+    links = KeyLinks()
+    seen: set[LineageEntry] = set()
+    for batch in _batches(enumerate(c.lines(VAULT_LINEAGE), start=1), c.batch):
+        entries: list[LineageEntry] = []
+        for number, line in batch:
+            try:
+                entry = parse_lineage(line)
+            except VaultTransferError as error:
+                raise BundleError(f"vault_lineage line {number}: {error}") from error
+            if entry in seen:
+                raise BundleError(f"vault_lineage line {number}: the key change appears twice")
+            missing = [
+                key
+                for key in (entry.previous_key, entry.record_key)
+                if (entry.kind_id, key) not in records
+            ]
+            if missing:
+                raise BundleError(
+                    f"vault_lineage line {number}: no Record {missing[0]!r} of Kind "
+                    f"{entry.kind_id} in the bundle"
+                )
+            seen.add(entry)
+            links.link(entry)
+            entries.append(entry)
+        await restore_lineage(c.session, c.domain_id, entries, at=at)
+    c.report.vault_lineage = len(seen)
+    c.expect(VAULT_LINEAGE, len(seen))
+    return links, seen
+
+
+async def _import_tombstones(c: _Import, keys: VaultKeys, at: datetime) -> None:
+    """Tombstones are taken as made: only when the target has the token key they were made
+    under (by id)."""
+    total = 0
+    for batch in _batches(enumerate(c.lines(VAULT_TOMBSTONES), start=1), c.batch):
+        entries: list[TombstoneEntry] = []
+        for number, line in batch:
+            try:
+                entries.append(parse_tombstone(line, keys.token_key_ids))
+            except VaultTransferError as error:
+                raise BundleError(f"vault_tombstones line {number}: {error}") from error
+        await restore_tombstones(c.session, c.domain_id, entries, at=at)
+        total += len(entries)
+    c.report.vault_tombstones = total
+    c.expect(VAULT_TOMBSTONES, total)
+
+
+async def _import_nodes(c: _Import) -> None:
+    repository = NodeRepository(
+        c.session,
+        c.domain_id,
+        writes=c.runtime.settings.writes,
+        batch=c.runtime.settings.nodes.prefetch_batch,
+    )
+    total = 0
+    for batch in _batches(c.lines("nodes"), c.batch):
+        nodes: dict[bytes, bytes] = {}
+        for line in batch:
+            body = base64.b64decode(line["body"], validate=True)
+            digest = from_hex(line["hash"])
+            if node_hash(body) != digest:
+                raise BundleError(f"tree node {line['hash']} does not match its content")
+            decode_node(body)  # rejects malformed nodes
+            nodes[digest] = body
+        await repository.put_many(nodes)
+        total += len(batch)
+    c.report.nodes = total
+    c.expect("nodes", total)
+
+
+async def _import_revisions(c: _Import) -> None:
+    revisions = RevisionRepository(c.session, c.domain_id)
+    writes = c.runtime.settings.writes
+    total = 0
+    previous = 0
+    for batch in _batches(c.lines("revisions"), c.batch):
+        local: dict[bytes, int] = {}
+        parsed: list[tuple[dict[str, Any], Revision, bytes]] = []
+        for line in batch:
+            revision = _revision(c.domain_id, line)
+            digest = revision_id(revision)
+            if to_hex(digest) != line["id"]:
+                raise BundleError(f"Revision {line['id']} does not match its content")
+            if line["generation"] < previous:
+                raise BundleError("revisions.jsonl is not ordered by generation")
+            previous = line["generation"]
+            parsed.append((line, revision, digest))
+        outside = {p for _, rev, _ in parsed for p in rev.parents} - {d for _, _, d in parsed}
+        known = await revisions.generations(outside)
+        rows: list[dict[str, Any]] = []
+        parent_rows: list[dict[str, Any]] = []
+        for line, revision, digest in parsed:
+            try:
+                generation = next_generation(local.get(p) or known[p] for p in revision.parents)
+            except KeyError as error:
+                raise BundleError(
+                    f"Revision {line['id']} comes before its parent {to_hex(error.args[0])}"
+                ) from error
+            if generation != line["generation"]:
+                raise BundleError(f"Revision {line['id']} has the wrong generation")
+            local[digest] = generation
+            rows.append(
+                {
+                    "domain_id": c.domain_id,
+                    "id": digest,
+                    "root": revision.root,
+                    "generation": generation,
+                    "author": revision.author.principal,
+                    "on_behalf_of": revision.author.on_behalf_of,
+                    "message": revision.message,
+                    "origin": revision.origin.model_dump(),
+                    "stats": revision.stats.model_dump(),
+                    "created_at": revision.created_at,
+                }
+            )
+            parent_rows.extend(
+                {
+                    "domain_id": c.domain_id,
+                    "revision_id": digest,
+                    "position": position,
+                    "parent_id": parent,
+                }
+                for position, parent in enumerate(revision.parents)
+            )
+        await bulk_write(c.session, REVISIONS, rows, writes=writes, on_conflict=OnConflict.ERROR)
+        await bulk_write(
+            c.session, REVISION_PARENTS, parent_rows, writes=writes, on_conflict=OnConflict.ERROR
+        )
+        total += len(batch)
+    c.report.revisions = total
+    c.expect("revisions", total)
+
+
+def _revision(domain_id: str, line: Mapping[str, Any]) -> Revision:
+    try:
+        return Revision(
+            parents=tuple(from_hex(p) for p in line["parents"]),
+            root=from_hex(line["root"]),
+            domain=domain_id,
+            author=Author(principal=line["author"], on_behalf_of=line["on_behalf_of"]),
+            message=line["message"],
+            origin=Origin(**line["origin"]),
+            created_at=datetime.fromisoformat(line["created_at"]),
+            stats=RevisionStats(**line["stats"]),
+        )
+    except (ValueError, KeyError, TypeError) as error:
+        raise BundleError(f"Revision {line.get('id')}: {error}") from error
+
+
+def _from_hex(value: str | None) -> bytes | None:
+    return None if value is None else from_hex(value)
+
+
+async def _import_changesets(c: _Import) -> None:
+    writes = c.runtime.settings.writes
+    total = 0
+    for batch in _batches(c.lines("changesets"), c.batch):
+        rows = [
+            {
+                "domain_id": c.domain_id,
+                "revision_id": from_hex(line["revision"]),
+                "partition": line["partition"],
+                "element_key": line["key"],
+                "old_hash": _from_hex(line["old"]),
+                "new_hash": _from_hex(line["new"]),
+                "provenance": line["provenance"] or None,
+            }
+            for line in batch
+        ]
+        await bulk_write(c.session, CHANGESETS, rows, writes=writes, on_conflict=OnConflict.ERROR)
+        total += len(rows)
+    c.report.changesets = total
+    c.expect("changesets", total)
+
+
+async def _import_provenance(c: _Import) -> None:
+    writes = c.runtime.settings.writes
+    total = 0
+    for batch in _batches(c.lines("provenance"), c.batch):
+        rows = [
+            {
+                "domain_id": c.domain_id,
+                "id": line["id"],
+                "origin_type": line["origin_type"],
+                "origin_ref": line["origin_ref"],
+                "actor": line["actor"],
+                "at": datetime.fromisoformat(line["at"]),
+            }
+            for line in batch
+        ]
+        await bulk_write(c.session, PROVENANCE, rows, writes=writes, on_conflict=OnConflict.ERROR)
+        total += len(rows)
+    c.report.provenance = total
+    c.expect("provenance", total)
+
+
+async def _import_refs(c: _Import) -> list[dict[str, Any]]:
+    refs: list[dict[str, Any]] = c.document("refs.json")
+    heads = c.manifest["heads"]
+    rows = []
+    for ref in refs:
+        if heads.get(ref["name"]) != ref["head"]:
+            raise BundleError(f"ref {ref['name']!r} disagrees with the manifest heads")
+        rows.append(
+            {
+                "domain_id": c.domain_id,
+                "name": ref["name"],
+                "kind": RefKind(ref["kind"]),
+                "head": from_hex(ref["head"]),
+                "protected": ref["protected"],
+                "materialization": _imported_materialization(ref["materialization"]),
+                "version": ref["version"],
+                "created_at": datetime.fromisoformat(ref["created_at"]),
+                "created_by": ref["created_by"],
+            }
+        )
+    await bulk_write(
+        c.session, REFS, rows, writes=c.runtime.settings.writes, on_conflict=OnConflict.ERROR
+    )
+    c.report.refs = len(rows)
+    c.expect("refs", len(rows))
+    return refs
+
+
+def _imported_materialization(value: str) -> Materialization:
+    """Overlay branches come back as full copies: import rebuilds every head from its tree."""
+    found = Materialization(value)
+    return Materialization.FULL if found is Materialization.OVERLAY else found
+
+
+async def _import_ref_log(c: _Import) -> None:
+    writes = c.runtime.settings.writes
+    total = 0
+    for batch in _batches(c.lines("ref_log"), c.batch):
+        rows = [
+            {
+                "domain_id": c.domain_id,
+                "ref_name": line["ref"],
+                "seq": line["seq"],
+                "old_head": _from_hex(line["old"]),
+                "new_head": from_hex(line["new"]),
+                "reason": line["reason"],
+                "actor": line["actor"],
+                "at": datetime.fromisoformat(line["at"]),
+            }
+            for line in batch
+        ]
+        await bulk_write(c.session, REF_LOG, rows, writes=writes, on_conflict=OnConflict.ERROR)
+        total += len(rows)
+    c.report.ref_log = total
+    c.expect("ref_log", total)
+
+
+async def _check_integrity(c: _Import) -> None:
+    """Every Revision is reachable, every reachable node and object is stored."""
+    marker = Marker(c.session, c.domain_id, batch=c.runtime.settings.gc.mark_batch)
+    await marker.prepare()
+    await marker.mark(attempts=False)
+    stats = await marker.stats()
+    missing_objects = (
+        await c.session.execute(
+            text(
+                "SELECT count(*) FROM lb_gc_objects m WHERE NOT EXISTS "
+                "(SELECT 1 FROM objects o WHERE o.domain_id = :d AND o.hash = m.hash)"
+            ),
+            {"d": c.domain_id},
+        )
+    ).scalar_one()
+    await marker.drop()
+    if stats.missing_nodes or missing_objects:
+        raise BundleError(
+            f"the bundle lacks {stats.missing_nodes} tree nodes and {missing_objects} objects "
+            "that its Revisions reach"
+        )
+    if stats.revisions != c.report.revisions:
+        raise BundleError(
+            f"{c.report.revisions - stats.revisions} Revisions are not reachable from any ref"
+        )
+
+
+async def _materialize(c: _Import, refs: Sequence[Mapping[str, Any]], check: bool) -> None:
+    for ref in refs:
+        if ref["kind"] != RefKind.BRANCH.value or not ref["materialized"]:
+            continue
+        await rebuild(c.session, c.domain_id, ref["name"], runtime=c.runtime)
+        c.report.rebuilt_branches.append(ref["name"])
+        if check:
+            result = await verify(c.session, c.domain_id, ref["name"], runtime=c.runtime)
+            if not result.ok:
+                raise BundleError(
+                    f"branch {ref['name']!r} fails verification after rebuild: "
+                    f"{result.divergences[:3]}"
+                )
+    heads = RefRepository(c.session, c.domain_id)
+    for ref in refs:  # the refs must still be exactly the bundle's
+        stored = await heads.require(ref["name"])
+        if stored.head != from_hex(ref["head"]) or stored.version != ref["version"]:
+            raise BundleError(f"ref {ref['name']!r} changed while importing")
+    schemas = len([name for name in c.manifest["files"] if name.startswith(SCHEMAS_DIR + "/")])
+    c.report.schemas = schemas
+
+
+# --------------------------------------------------------------------------------------
+# Command line
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="python -m lorebase_store.bundle",
+        description="Export and import portable Domain bundles. Reads the database and "
+        "store.bundle settings from the Lorebase configuration.",
+    )
+    commands = parser.add_subparsers(dest="command", required=True)
+    export = commands.add_parser("export", help="Write a bundle of a Domain.")
+    export.add_argument("--domain", required=True, help="Domain id.")
+    export.add_argument(
+        "--out", required=True, type=Path, help="New directory, or a path ending in .tar."
+    )
+    export.add_argument(
+        "--include-vault-values",
+        action="store_true",
+        help="Also write the vault values, in plain text (needs the vault keys; audited).",
+    )
+    export.add_argument(
+        "--actor",
+        default=BUNDLE_ACTOR,
+        help="Who exports, recorded in the audit entry of --include-vault-values.",
+    )
+    load = commands.add_parser("import", help="Import a bundle into an empty Domain.")
+    load.add_argument("--in", dest="source", required=True, type=Path, help="Directory or .tar.")
+    load.add_argument("--domain", default=None, help="Target Domain id (must equal the bundle's).")
+    load.add_argument(
+        "--no-verify", action="store_true", help="Skip verifying rebuilt branches against trees."
+    )
+    load.add_argument(
+        "--actor",
+        default=BUNDLE_ACTOR,
+        help="Who imports, recorded in the audit entry of a bundle with vault values.",
+    )
+    return parser
+
+
+async def _main(args: argparse.Namespace) -> int:
+    from lorebase_platform.settings import load_settings
+    from lorebase_store.db import create_engine
+    from lorebase_store.settings import database_settings
+
+    settings = load_settings()
+    runtime = StoreRuntime.from_loaded(settings)
+    engine = create_engine(database_settings(settings), pooled=False, statement_timeout_ms=0)
+    try:
+        if args.command == "export":
+            report = await export_domain(
+                engine,
+                args.domain,
+                args.out,
+                runtime=runtime,
+                include_vault_values=args.include_vault_values,
+                actor=args.actor,
+            )
+            if args.include_vault_values:
+                sys.stderr.write(PLAIN_TEXT_WARNING.format(path=args.out))
+        else:
+            report = await import_bundle(
+                engine,
+                args.source,
+                domain_id=args.domain,
+                runtime=runtime,
+                verify_heads=False if args.no_verify else None,
+                actor=args.actor,
+            )
+    except (BundleError, DomainNotFoundError) as error:
+        sys.stderr.write(f"{error}\n")
+        return 1
+    finally:
+        await engine.dispose()
+    sys.stdout.write(json.dumps(report.as_dict()) + "\n")
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    return asyncio.run(_main(_parser().parse_args(argv)))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())