smart-data-engine-sdk 0.1.0.dev0__py3-none-any.whl

sde/placement.py

@@ -0,0 +1,818 @@
+"""The placement map: where every group lives, and where every operation goes.
+
+The map is the only instruction this library takes from outside. It says which engine holds each
+group, what the physical layout there is, and - optionally - which materialisation each operation
+shape should be routed to. Because it decides where data is written, it is the one input that
+refuses rather than degrades: a bad signature, a mismatched model version or an unknown contract
+version all stop the library from starting.
+
+Two things about signing are worth stating plainly, because open-sourcing the library changes how
+they read.
+
+Publishing this code does not weaken the map. The public key lives here, the private key lives in
+the control plane, and a reader seeing that maps are verified is reassured rather than informed of a
+weakness. A signature is not a secret.
+
+And a map with no signature at all is a *valid* map. That is the no-account mode: write a map by
+hand, point the library at it, and everything works with no key, no account and no network. It is a
+documented, supported way to use this library rather than a gap - which is also the honest answer to
+anyone asking what happens if they stop paying us. What is refused is the middle case: a map that
+carries a signature, thereby claiming to come from us, when we have no key to check that claim
+against.
+"""
+
+from __future__ import annotations
+
+import base64
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field, replace
+from typing import Any
+
+from .canonical import canonical_bytes
+from .errors import MapError
+from .logging import log
+from .model import LogicalModel
+
+__all__ = [
+    "BACKFILL_TABLE",
+    "RESERVED_TABLES",
+    "GroupPlacement",
+    "Materialization",
+    "PhysicalLayout",
+    "PlacementMap",
+    "load_map",
+]
+
+
+@dataclass(frozen=True)
+class PhysicalLayout:
+    """What the group looks like inside one engine.
+
+    This is ours to choose and ours to change, which is the entire reason the client's code never
+    names a table. Everything here is derived by the planner and simply obeyed by the library.
+    """
+
+    tables: Mapping[str, str]
+    columns: Mapping[str, Mapping[str, str]]
+    indexes: Sequence[Mapping[str, Any]] = field(default_factory=tuple)
+    partition_by: Mapping[str, str] = field(default_factory=dict)
+
+    def table_for(self, entity: str) -> str:
+        try:
+            return self.tables[entity]
+        except KeyError:
+            raise MapError(
+                f"the layout has no table for {entity!r}. The map claims to place a group that "
+                "contains this entity, so this is a defect in the map rather than something the "
+                "library can work around."
+            ) from None
+
+
+@dataclass(frozen=True)
+class Materialization:
+    """One physical copy of a group in one engine."""
+
+    id: str
+    engine: str
+    layout: PhysicalLayout
+    lag_budget_ms: int | None = None
+
+    @property
+    def is_source(self) -> bool:
+        return self.lag_budget_ms is None
+
+
+@dataclass(frozen=True)
+class GroupPlacement:
+    group: str
+    source: Materialization
+    derived: tuple[Materialization, ...] = ()
+    also_write: tuple[Materialization, ...] = ()
+    """Derived copies that writes are **also** sent to, additionally and never authoritatively.
+
+    This is how a migration reaches the library, and the reason there is no phase name anywhere in
+    the map. A library does not need to know what ``DUAL_WRITE`` means; it needs to know where
+    writes go and where reads go, and both of those were already things a map says. Putting the
+    phase in the document as well would be a second representation of a fact the fan-out and the
+    routing table already carry - and a signed document with two representations of one fact is one
+    that can contradict itself.
+
+    Empty for every map that is not mid-migration, and the key is absent rather than empty then, for
+    the reason ``partition_by`` is: absent says "not doing this", and an empty list says "considered
+    and chose none", which is a stronger claim and a false one.
+    """
+
+    def all(self) -> tuple[Materialization, ...]:
+        return (self.source, *self.derived)
+
+    def by_id(self, mat_id: str) -> Materialization:
+        for mat in self.all():
+            if mat.id == mat_id:
+                return mat
+        raise MapError(
+            f"group {self.group!r} has no materialisation {mat_id!r}, but the routing table points "
+            "at it. The map is internally inconsistent."
+        )
+
+
+@dataclass(frozen=True)
+class PlacementMap:
+    contract: int
+    model_version: str
+    map_version: int
+    groups: Mapping[str, GroupPlacement]
+    routing: Mapping[str, str]
+    signed: bool
+    verified_with: str | None = None
+    """Which of the caller's public keys verified this map, by the caller's own name for it.
+
+    ``None`` for an unsigned map and also for one verified against a bare key, where there was no
+    name to report; :attr:`signed` tells those apart. A default is allowed here because this field
+    is *reported* and never acted on - nothing routes or writes differently because of it - unlike
+    the dialect in a materialisation plan, which has no default precisely because a wrong one
+    produces a document that verifies and cannot be applied.
+
+    Public because a rotation needs it. Adding a key is safe and reversible; **removing** one is
+    the irreversible half, and a client cannot know when the old key is safe to drop without
+    seeing which key the maps they are actually receiving were signed with.
+    """
+
+    def placement_of(self, group: str) -> GroupPlacement:
+        try:
+            return self.groups[group]
+        except KeyError:
+            raise MapError(
+                f"no placement for group {group!r}. Every group in the model needs one: a group "
+                "with "
+                "nowhere to live is not a slow path, it is an unanswerable operation."
+            ) from None
+
+
+MAP_CONTRACT = 3
+"""The placement map's format version, which is not the IR's - see :data:`sde.model.CONTRACT`.
+
+Two was for ``also_write``, which a contract-1 library would ignore while a contract-2 one honours
+it: the same document, two different sets of engines written to, and the difference decided by
+which version happens to be installed. That is a loosening, and section 11 of the contract says a
+loosening bumps the number.
+
+Three since 7 September 2026, and the reason is not a new key. The rendered ClickHouse timestamp
+moved from `DateTime64(3)` to `DateTime64(6)` so that it matches PostgreSQL, which changed what
+this library believes the two dialects keep - and therefore whether it accepts a `also_write` map
+between them. A contract-2 library refuses a fan-out that a contract-3 one performs, on the same
+document, which is the loosening section 11 says bumps the number.
+"""
+
+ALSO_WRITE_SINCE = 2
+"""The contract that introduced ``also_write``.
+
+A document declaring an earlier contract and carrying the key is refused. That is a *tightening* -
+no bump - and it exists because of who makes the mistake: a producer that grew the key and forgot to
+raise the number, which is exactly what happened on the control-plane side of the very change that
+added it. Honouring the key anyway would mean an old library ignoring the fan-out while a new one
+performs it, on the same document.
+"""
+
+MAP_CONTRACT_FLOOR = 1
+"""The oldest map format this library still reads.
+
+**Backwards compatible, forwards strict**, and the asymmetry is knowledge rather than kindness: we
+know exactly what contract 1 was, and every contract-1 document is a valid contract-2 one with a key
+absent - which reads as "no dual write", a complete and correct meaning. What came *after* this
+library cannot be known, so a higher number is refused rather than interpreted, which is the case
+the original message argued for.
+
+The alternative - strict equality, as this library did until contract 2 - couples a library upgrade
+to a control-plane action: a client would have to be issued a new map before they could start. For
+the no-account mode that is worse than inconvenient, because there is nobody to issue them one and
+the promise was that hand-writing a map is enough.
+"""
+
+WATERMARK_TABLE = "sde_map_state"
+"""The table this library keeps its map bookkeeping in - see :mod:`sde.watermark`.
+
+Defined here rather than there because the reservation is a property of the map format, and because
+``watermark.py`` imports this module: a layout naming this table is refused when the map is loaded.
+
+A client entity called ``SdeMapState`` would derive this table name from the model, and the refusal
+below catches that too - at map load, which for a map we issue means at issuance, since `issue()`
+parses its own output with this parser. The message names the fix, which is to rename the entity.
+"""
+
+BACKFILL_TABLE = "sde_backfill_state"
+"""The table this library keeps its backfill progress in - see :mod:`sde.migration`.
+
+The second of these, and the reason the refusal below is written over a set rather than over one
+name: a bookkeeping table whose name is not reserved is a table a client's model can collide with,
+and the collision is silent in the direction that matters - we would read their rows as progress and
+write progress into their table.
+"""
+
+RESERVED_TABLES: Mapping[str, str] = {
+    WATERMARK_TABLE: (
+        "the highest map version applied against an engine, which is what stops an older map "
+        "from being loaded over a newer one"
+    ),
+    BACKFILL_TABLE: (
+        "how many rows of each entity a migration has copied into an engine, which is what lets "
+        "an interrupted backfill resume instead of starting over"
+    ),
+}
+"""Table names this library owns, and what each one holds.
+
+A mapping rather than a set so the refusal can say what the collision would break. "This name is
+reserved" sends the reader to our source; "this name holds the backfill marker" tells them why
+their entity cannot have it.
+"""
+
+_AUTO = object()
+
+
+def _layout(raw: Mapping[str, Any], where: str) -> PhysicalLayout | object:
+    """Parse a layout, or report that it asked to be derived.
+
+    ``{"auto": true}`` is what makes a hand-written map a few lines rather than a full schema
+    written out by hand. Without it the no-account mode from requirement 12.5 would be technically
+    true and practically unusable, which is the same as not having it.
+
+    It derives a **PostgreSQL** layout, always, and that is a limit rather than a default worth
+    changing quietly. A layout has no dialect in it and the map names an engine by *name*, not by
+    dialect - deliberately, since reasoning from an engine's name is what this library refuses
+    everywhere - so there is nothing here from which the right dialect could be known. A
+    hand-written map for ClickHouse or for the orderbook engine therefore needs an explicit layout,
+    and both of those engines refuse a PostgreSQL-derived one rather than applying it: ClickHouse
+    because it carries indexes it has no B-tree for, the orderbook engine because the table is not
+    the one name its storage has. Fails closed, in other words, but the message names the symptom.
+    Making ``auto`` dialect-aware means a new key in a signed document, which is a loosening of the
+    format and so a contract bump in every language at once - see section 11.
+    """
+    if not isinstance(raw, dict):
+        raise MapError(f"{where}: layout must be an object")
+    if raw.get("auto") is True:
+        if len(raw) != 1:
+            raise MapError(
+                f"{where}: a layout is either auto or explicit, not both. Two sources of truth for "
+                "a "
+                "schema is how a column ends up existing in one place and not the other."
+            )
+        return _AUTO
+    tables = raw.get("tables")
+    if not isinstance(tables, dict) or not tables:
+        raise MapError(
+            f"{where}: layout needs a non-empty 'tables' mapping entity -> table name, "
+            'or {"auto": true} to have one derived from the model'
+        )
+    for name, holds in RESERVED_TABLES.items():
+        reserved = sorted(entity for entity, table in tables.items() if str(table) == name)
+        if reserved:
+            raise MapError(
+                f"{where}: {reserved} would be stored in a table called {name!r}, which this "
+                f"library keeps its own bookkeeping in - {holds}. A client table under that name "
+                f"would be read as bookkeeping and written to as bookkeeping. Rename the table; "
+                f"the name is yours to choose everywhere else."
+            )
+    # `partition_by` is refused rather than ignored, and the refusal is the whole point: this key
+    # is parsed here, the control plane emits it when non-empty, and **no renderer has ever applied
+    # it** - a layout declaring it produced an unpartitioned table and said nothing. Nothing
+    # populates it today, so no issued map has carried one, but a hand-written map legally may and
+    # the no-account mode is a documented mode. Rendering it instead would mean designing two
+    # dialect-specific features with no requirement behind them and interpolating a caller's SQL
+    # fragment into DDL. Fails closed until partitioning exists; §7a and `errors/037`.
+    if raw.get("partition_by"):
+        raise MapError(
+            f"{where}: this layout declares partition_by, and no library renders it - the table "
+            f"would be created unpartitioned and nothing would say so. A storage decision in a "
+            f"signed document that is silently dropped is worse than a refusal, so this refuses "
+            f"until partitioning is implemented. Remove the key to apply the rest of the layout."
+        )
+    return PhysicalLayout(
+        tables=dict(tables),
+        columns={k: dict(v) for k, v in (raw.get("columns") or {}).items()},
+        indexes=tuple(raw.get("indexes") or ()),
+        partition_by=dict(raw.get("partition_by") or {}),
+    )
+
+
+def _also_write(
+    raw: Any,
+    where: str,
+    *,
+    contract: int,
+    source: Materialization,
+    derived: tuple[Materialization, ...],
+) -> tuple[Materialization, ...]:
+    """The derived copies writes are additionally sent to. Four refusals, each with its failure.
+
+    Validated here rather than at the first write, for the reason the routing table was moved here:
+    a map is a document handed over once and obeyed for months, so a defect in it should be found
+    when it arrives and not by the request that happens to touch it. During a migration that request
+    is a write, and the failure mode is a write that goes to one engine when the map says two.
+    """
+    if raw is None:
+        return ()
+    if contract < ALSO_WRITE_SINCE:
+        raise MapError(
+            f"{where}: this map declares format contract {contract} and uses 'also_write', which "
+            f"contract {ALSO_WRITE_SINCE} introduced. Refused rather than honoured, and the reason "
+            f"is who makes this mistake: a producer that grew the key and forgot to raise the "
+            f"number. Honouring it would mean a contract-1 library ignoring the fan-out while a "
+            f"contract-2 one performs it - the same document, two sets of engines written to - "
+            f"which is the whole failure the version exists to prevent."
+        )
+    if not isinstance(raw, list) or not raw:
+        raise MapError(
+            f"{where}: 'also_write' is a non-empty list of materialisation ids, or absent. Absent "
+            f"means writes go to the source alone; an empty list would be a claim that fan-out was "
+            f"considered and none chosen, which is a stronger thing to say and not what the "
+            f"planner means when it omits the key."
+        )
+    by_id = {m.id: m for m in derived}
+    out: list[Materialization] = []
+    seen: set[str] = set()
+    for entry in raw:
+        if not isinstance(entry, str):
+            raise MapError(f"{where}: 'also_write' holds materialisation ids, not {entry!r}")
+        if entry == source.id:
+            raise MapError(
+                f"{where}: 'also_write' names the source {entry!r}. The source is where writes "
+                f"already land - listing it would either write the row twice or read as though the "
+                f"source were somehow optional, and both are worse than the refusal."
+            )
+        if entry in seen:
+            raise MapError(
+                f"{where}: 'also_write' names {entry!r} twice. Two identical fan-out targets is "
+                f"either a duplicated row or a document nobody meant to write."
+            )
+        if entry not in by_id:
+            raise MapError(
+                f"{where}: 'also_write' names {entry!r}, which is not a derived materialisation of "
+                f"this group. It has {sorted(by_id)}. A fan-out target that does not exist is a "
+                f"write with nowhere to go, and during a migration that is a row the copy never "
+                f"receives."
+            )
+        seen.add(entry)
+        out.append(by_id[entry])
+    return tuple(out)
+
+
+def _materialization(raw: Mapping[str, Any], where: str, *, source: bool) -> Materialization:
+    for required in ("id", "engine", "layout"):
+        if required not in raw:
+            raise MapError(f"{where}: materialisation is missing {required!r}")
+    lag = raw.get("lag_budget_ms")
+    if source and lag is not None:
+        raise MapError(
+            f"{where}: the source materialisation cannot have a lag budget. The source is where "
+            "writes land, so it is by definition not behind anything."
+        )
+    if not source and lag is None:
+        raise MapError(
+            f"{where}: a derived materialisation needs lag_budget_ms. Without it nobody - not the "
+            "client, not the monitoring - can tell whether it is healthy or hours behind."
+        )
+    layout = _layout(raw["layout"], where)
+    return Materialization(
+        id=str(raw["id"]),
+        engine=str(raw["engine"]),
+        layout=layout,  # type: ignore[arg-type]
+        lag_budget_ms=None if lag is None else int(lag),
+    )
+
+
+def _key_set(public_key: bytes | Mapping[str, bytes]) -> tuple[tuple[str, bytes], ...]:
+    """Normalise what the caller supplied into named keys, refusing what cannot be a key.
+
+    A bare ``bytes`` is the ordinary case and keeps the empty name, because the caller gave us no
+    name to report back. A mapping is the rotation case, and the names are the caller's own: what a
+    rotation runbook needs is not a key but something to *remove*, and "remove k1" is only an
+    instruction if k1 is what they wrote in their configuration.
+
+    A key of the wrong length is refused here rather than left to fail verification. It is the
+    client's configuration, and a rotation that silently did not take effect - because the new key
+    was pasted a byte short - is the failure mode this whole mechanism exists to avoid. Before this
+    it raised a bare ``ValueError`` out of ``load_map`` on the application's start path.
+    """
+    pairs: tuple[tuple[str, bytes], ...]
+    if isinstance(public_key, (bytes, bytearray)):
+        pairs = (("", bytes(public_key)),)
+    else:
+        if not public_key:
+            raise MapError(
+                "an empty set of public keys was supplied. That is not the no-account mode - a map "
+                "with no signature is - it is a configuration that can verify nothing. Pass the "
+                "keys, or load an unsigned map."
+            )
+        pairs = tuple(sorted((str(k), bytes(v)) for k, v in public_key.items()))
+    for name, value in pairs:
+        if len(value) != 32:
+            raise MapError(
+                f"the public key {name or '(unnamed)'!r} is {len(value)} bytes and an Ed25519 "
+                f"public key is 32. Refused here rather than at verification: a key pasted a byte "
+                f"short would look like a map that does not verify, and the two have completely "
+                f"different fixes."
+            )
+    return pairs
+
+
+def _verify_signature(
+    raw: Mapping[str, Any], public_key: bytes | Mapping[str, bytes]
+) -> str | None:
+    """Verify, and report **which** of the caller's keys did it.
+
+    Returns the caller's own name for the key that verified, or ``None`` when a bare key was
+    supplied and there was no name to report. That does not collide with an unsigned map:
+    :attr:`PlacementMap.signed` tells those two apart, and one field meaning two things is how a
+    value ends up standing for four (see the coordinator's three-state answer in the engine).
+
+    **Every key is tried, and ``key_id`` only decides the order.** The signature block is excluded
+    from the signed payload in its entirety, so ``key_id`` is not covered by the signature and
+    anybody can edit it. Making it authoritative would therefore be trusting an attacker's
+    annotation; making it *authenticated* would mean changing what the payload is, which recomputes
+    every signature ever issued and is a contract bump in every language at once. As an ordering it
+    costs nothing to be wrong about: a bad hint makes verification try the keys in a worse order
+    and reach the same answer.
+
+    The cost of accepting a set is real and belongs written down here rather than in a document:
+    **an old key verifies for as long as the client keeps it.** This library has no clock - it is
+    never told when a map was issued, and reads the wall clock exactly once in the whole package -
+    so it cannot expire a key, and that same absence is what makes "you stop paying and nothing
+    breaks" true. Revocation is therefore the client removing a key from their own configuration,
+    which is why the id that verified is reported: a client who cannot see which key is in use
+    cannot know when the old one is safe to drop.
+    """
+    try:
+        from cryptography.exceptions import InvalidSignature
+        from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
+    except ImportError as exc:  # pragma: no cover - depends on the install extra
+        raise MapError(
+            "this map is signed, but signature verification needs the 'signed' extra: "
+            "pip install 'smart-data-engine-sdk[signed]'. The base install stays dependency-free "
+            "on purpose, because a library that goes into someone's application should not drag in "
+            "cryptography unless it is actually verifying something."
+        ) from exc
+
+    signature = raw["signature"]
+    if not isinstance(signature, dict) or signature.get("alg") != "ed25519":
+        raise MapError("only ed25519 signatures are understood")
+    try:
+        value = base64.b64decode(signature["value"], validate=True)
+    except Exception as exc:
+        raise MapError("the signature is not valid base64") from exc
+
+    claimed = signature.get("key_id")
+    claimed = str(claimed) if isinstance(claimed, str) else None
+    keys = _key_set(public_key)
+    if claimed is None:
+        ordered = list(keys)
+    else:
+        hinted = [pair for pair in keys if pair[0] == claimed]
+        ordered = hinted + [pair for pair in keys if pair[0] != claimed]
+
+    payload = canonical_bytes({k: v for k, v in raw.items() if k != "signature"})
+    for name, key in ordered:
+        try:
+            Ed25519PublicKey.from_public_bytes(key).verify(value, payload)
+        except InvalidSignature:
+            continue
+        return name or None
+
+    tried = [name or "(unnamed)" for name, _ in ordered]
+    raise MapError(
+        f"the map's signature does not verify against any of the {len(ordered)} key(s) supplied "
+        f"({tried}); the map says it was signed with {claimed!r}. This is refused rather than "
+        f"warned about: the map decides where your data is written. If a rotation is in progress, "
+        f"the key named above is the one to add - and while both are configured, maps signed with "
+        f"either are accepted."
+    )
+
+
+def load_map(
+    raw: Mapping[str, Any],
+    *,
+    model: LogicalModel | None = None,
+    public_key: bytes | Mapping[str, bytes] | None = None,
+    require_signature: bool = False,
+) -> PlacementMap:
+    """Parse and validate a placement map, and put the outcome on the record either way.
+
+    ``model`` is optional only so that the conformance vectors can exercise parsing on its own; in
+    an application it is always passed, because a map for a different model version has to be
+    refused rather than half-applied.
+
+    Both outcomes are logged, and the pair is the point. Until this wrapper existed the only line
+    a map load produced was ``sde.map.unsigned``, which fired **only** when the map had no
+    signature - so the one thing this library said about a map was said exclusively to clients
+    without an account. That is the shape requirement 12.6 forbids, arrived at from the other
+    direction: not a complaint anybody wrote, but a line that exists in one mode because nobody
+    wrote the other one.
+
+    So the event is the same in both modes and the mode is a **field**. That is what the closed
+    vocabulary is for - structured fields, and no interpolated values in the event name - and it
+    means the account-free mode emits no event the account mode does not. ``forward_only`` is the
+    single thing that actually differs; the sentence explaining it lives on
+    :attr:`Session.rollback_protection`, where a client can read it rather than grep for it.
+
+    A refusal is logged too, because ``MapError`` reaches the application while the log reaches the
+    operator, and those are different people at four in the morning. Its ``reason`` is the
+    engine-facing message, which in this library is always about structure - contract versions,
+    model versions, group, engine, table and materialisation names - and never about a value: there
+    is no code path by which one of the client's rows reaches a map.
+    """
+    try:
+        placement = _parse_map(
+            raw, model=model, public_key=public_key, require_signature=require_signature
+        )
+    except MapError as exc:
+        log("sde.map.rejected", error=type(exc).__name__, reason=str(exc)[:200])
+        raise
+    log(
+        "sde.map.loaded",
+        model_version=placement.model_version,
+        map_version=placement.map_version,
+        signed=placement.signed,
+        forward_only=placement.signed,
+        key=placement.verified_with,
+    )
+    return placement
+
+
+def _parse_map(
+    raw: Mapping[str, Any],
+    *,
+    model: LogicalModel | None = None,
+    public_key: bytes | Mapping[str, bytes] | None = None,
+    require_signature: bool = False,
+) -> PlacementMap:
+    """The parse itself. Separate so that the reporting above wraps every refusal in it, including
+    the ones added later by somebody who has not read this file."""
+    if not isinstance(raw, dict):
+        raise MapError("a placement map is an object")
+
+    contract = raw.get("contract")
+    if not isinstance(contract, int) or isinstance(contract, bool):
+        raise MapError(
+            f"this map declares format contract {contract!r}, which is not a version number. "
+            f"This library reads {MAP_CONTRACT_FLOOR} to {MAP_CONTRACT}."
+        )
+    if contract > MAP_CONTRACT:
+        raise MapError(
+            f"this map declares format contract {contract} and this library implements "
+            f"{MAP_CONTRACT}. Refusing rather than guessing: a version this library does not know "
+            f"may say something with a key it has never heard of, and the difference between two "
+            f"contract versions is exactly the kind of thing that would otherwise be interpreted "
+            f"as a missing field meaning zero. Upgrade the library."
+        )
+    if contract < MAP_CONTRACT_FLOOR:
+        raise MapError(
+            f"this map declares format contract {contract} and the oldest this library still "
+            f"reads is {MAP_CONTRACT_FLOOR}."
+        )
+
+    model_version = raw.get("model_version")
+    if not isinstance(model_version, str) or not model_version:
+        raise MapError("the map does not say which model version it is for")
+    if model is not None and model.version != model_version:
+        raise MapError(
+            f"this map is for model version {model_version} and your declared model is "
+            f"{model.version}. Something changed in your entities; ask for a new map rather than "
+            "running against this one, because the difference cannot be guessed."
+        )
+
+    map_version = raw.get("map_version")
+    if not isinstance(map_version, int) or isinstance(map_version, bool) or map_version < 1:
+        raise MapError(
+            f"this map declares map_version {map_version!r}, which is not a version number. It "
+            "used to be read as int(document.get('map_version', 0)), so a document without one "
+            "loaded as version 0 - and this is the number that decides whether an older map is "
+            "being replayed over a newer one against the client's own engines."
+        )
+
+    signature_present = "signature" in raw and raw["signature"] is not None
+    if require_signature and not signature_present:
+        raise MapError("a signature was required and this map has none")
+    if signature_present:
+        if public_key is None:
+            raise MapError(
+                "this map is signed, which is a claim that it came from us, and no public key was "
+                "provided to check that claim. Either pass the key, or use an unsigned map - an "
+                "unsigned map is a supported mode, an unverifiable claim is not."
+            )
+        verified_with = _verify_signature(raw, public_key)
+    else:
+        verified_with = None
+
+    groups_raw = raw.get("groups")
+    if not isinstance(groups_raw, dict) or not groups_raw:
+        raise MapError("the map places no groups")
+
+    groups: dict[str, GroupPlacement] = {}
+    # By name, not in the document's order. A map with two defects has to refuse the same way in
+    # every language (format-contract §8a): `errors/019` carries a reserved table name in one group
+    # and an auto-plus-explicit layout in another, and it pinned the first only because that group
+    # is written earlier in the file and both of our runtimes iterate an object in insertion order.
+    # A third implementation in Go, whose maps iterate in a randomised order, failed that vector in
+    # 5 of 20 runs. The routing loop below has sorted since it was written; this one had not.
+    for name in sorted(groups_raw):
+        body = groups_raw[name]
+        where = f"group {name!r}"
+        if not isinstance(body, dict) or "source" not in body:
+            raise MapError(f"{where}: needs a 'source' materialisation")
+        source = _materialization(body["source"], f"{where}.source", source=True)
+        derived = tuple(
+            _materialization(d, f"{where}.derived[{i}]", source=False)
+            for i, d in enumerate(body.get("derived") or ())
+        )
+        ids = [m.id for m in (source, *derived)]
+        if len(set(ids)) != len(ids):
+            raise MapError(f"{where}: two materialisations share an id")
+
+        groups[name] = GroupPlacement(
+            group=name,
+            source=source,
+            derived=derived,
+            also_write=_also_write(
+                body.get("also_write"),
+                where,
+                contract=contract,
+                source=source,
+                derived=derived,
+            ),
+        )
+
+    routing = raw.get("routing") or {}
+    if not isinstance(routing, dict):
+        raise MapError("'routing' must be a mapping from shape id to materialisation id")
+
+    if model is not None:
+        model_groups = {g.name: g for g in _model_groups(model)}
+        missing = sorted(set(model_groups) - set(groups))
+        if missing:
+            raise MapError(
+                f"the map does not place these groups: {missing}. Every group in the model needs a "
+                "home before anything can run."
+            )
+        # And the other direction. Checking only one of the two is how a map placing a group this
+        # model does not have reached `model_groups[name]` below and came out as a bare KeyError -
+        # a library exception with no explanation, on the path whose entire job is to explain.
+        unknown = sorted(set(groups) - set(model_groups))
+        if unknown:
+            raise MapError(
+                f"the map places groups this model does not have: {unknown}. The model version "
+                "matched, so this is not a stale map: the two sides derived colocation groups "
+                "differently, and a group nobody declared has no entities to hold."
+            )
+        groups = {
+            name: _resolve_auto(placement, model, model_groups[name])
+            for name, placement in groups.items()
+        }
+        for placement in groups.values():
+            _refuse_shadowing(placement)
+        _check_routing_targets(routing, groups, model)
+    elif any(m.layout is _AUTO for p in groups.values() for m in p.all()):
+        raise MapError(
+            'a layout asked to be derived with {"auto": true}, but no model was supplied to derive '
+            "it from. Pass model= to load_map()."
+        )
+    else:
+        _check_routing_targets(routing, groups, None)
+
+    return PlacementMap(
+        contract=contract,
+        model_version=model_version,
+        map_version=map_version,
+        groups=groups,
+        routing={str(k): str(v) for k, v in routing.items()},
+        signed=signature_present,
+        verified_with=verified_with,
+    )
+
+
+def _refuse_shadowing(placement: GroupPlacement) -> None:
+    """Refuse a derived copy that is secretly the source.
+
+    Two materialisations of one group in the same engine must not name the same tables. Found by a
+    test rather than by thinking: with an auto layout on both, the source and a derived copy derive
+    identical table names, so a copy placed in the same engine simply *is* the source. Reads would
+    appear to work, the lag would always measure zero, and the second copy would exist only in the
+    map. Refused rather than warned about, precisely because it looks like it works.
+
+    Checked after auto layouts are resolved, since before that there are no table names to compare.
+    """
+    source_tables = set(placement.source.layout.tables.values())
+    for candidate in placement.derived:
+        if candidate.engine != placement.source.engine:
+            continue
+        shared = sorted(source_tables & set(candidate.layout.tables.values()))
+        if shared:
+            raise MapError(
+                f"group {placement.group!r}: materialisation {candidate.id!r} is in the same "
+                "engine "
+                f"as the source and reuses its tables {shared}. That is not a copy of the group, "
+                "it is the original with a second name in the map, so its lag would always read as "
+                "zero and a read routed to it would silently be a read of the source."
+            )
+
+
+def _resolve_auto(placement: GroupPlacement, model: LogicalModel, group: Any) -> GroupPlacement:
+    """Fill in any layout that asked to be derived.
+
+    Derivation lives in :mod:`sde.layout` and is the boring total function from a model to a schema
+    that stores it. The interesting choices - indexes worth their cost, partitioning, a second
+    materialisation - are the planner's and are not in this repository.
+    """
+    from .layout import default_layout
+
+    def fill(mat: Materialization) -> Materialization:
+        if mat.layout is not _AUTO:
+            return mat
+        return Materialization(
+            id=mat.id,
+            engine=mat.engine,
+            layout=default_layout(model, group),
+            lag_budget_ms=mat.lag_budget_ms,
+        )
+
+    # `replace` rather than a fresh GroupPlacement, and this is a fix rather than a preference.
+    # Reconstructing it listed the fields that existed when it was written, so `also_write` was
+    # dropped here the moment it was added: the map parsed, the fan-out came back empty, and writes
+    # went to one engine while the signed document said two. Silently, on the write path, during a
+    # migration - the worst failure this feature has. It is the third time in this project that a
+    # value has been computed and lost at a boundary (`provisional`, then `basis`), so the shape of
+    # the fix matters more than the fix: with `replace`, a field added later travels whether or not
+    # anybody remembers this function.
+    return replace(
+        placement,
+        source=fill(placement.source),
+        derived=tuple(fill(m) for m in placement.derived),
+        also_write=tuple(fill(m) for m in placement.also_write),
+    )
+
+
+def _model_groups(model: LogicalModel) -> tuple[Any, ...]:
+    # Imported late: groups imports model, and model must not import groups.
+    from .groups import colocation_groups
+
+    return colocation_groups(model)
+
+
+def _model_shapes(model: LogicalModel) -> tuple[Any, ...]:
+    # Late for the same reason: shapes imports groups, which imports model.
+    from .shapes import enumerate_shapes
+
+    return enumerate_shapes(model)
+
+
+def _check_routing_targets(
+    routing: Mapping[str, Any],
+    groups: Mapping[str, GroupPlacement],
+    model: LogicalModel | None,
+) -> None:
+    """Every routing entry must name a materialisation that exists, in the shape's own group.
+
+    This used to be checked in ``GroupPlacement.by_id`` - that is, at the first read that routed
+    through the broken entry. Deferring it there turns a mistake in a document we hand over into an
+    error inside the client's request path, at a moment nobody can predict: only the shapes routing
+    through that entry fail, so a staging run that never issues those operations is green and the
+    map looks applied. Everything here is decidable at load, so it is decided at load.
+
+    Two levels, because ``model`` is optional. Without a model: the target has to be an id declared
+    somewhere in this map. With one: it has to be declared in the group the shape belongs to, which
+    is the check that matters - ids are only unique *within* a group, so a target that exists in
+    some other group would otherwise read the entity out of a copy that does not hold it.
+    """
+    declared = {name: {m.id for m in placement.all()} for name, placement in groups.items()}
+    everywhere = {mat_id for ids in declared.values() for mat_id in ids}
+
+    for shape_id, target in sorted(routing.items()):
+        if not isinstance(target, str):
+            raise MapError(
+                f"the routing entry for shape {shape_id!r} is not a materialisation id. Routing "
+                "maps a shape id to one id, and anything else is a table nobody can look up."
+            )
+        if target not in everywhere:
+            raise MapError(
+                f"the routing table sends shape {shape_id!r} to materialisation {target!r}, and no "
+                f"group in this map declares one with that id. The map is internally inconsistent: "
+                f"declared ids are {sorted(everywhere)}."
+            )
+
+    if model is None:
+        return
+
+    shapes = {shape.id: shape for shape in _model_shapes(model)}
+    for shape_id, target in sorted(routing.items()):
+        shape = shapes.get(shape_id)
+        if shape is None:
+            raise MapError(
+                f"the routing table has an entry for shape {shape_id!r}, and this model does not "
+                "produce that shape. The model version matched, so the two sides enumerated shapes "
+                "differently - which is the divergence that puts one library's write in a table "
+                "another library never looks at."
+            )
+        if target not in declared.get(shape.group, frozenset()):
+            raise MapError(
+                f"the routing table sends shape {shape_id!r} - which belongs to group "
+                f"{shape.group!r} - to materialisation {target!r}, which that group does not "
+                f"declare. Materialisation ids are unique only within a group, so this would read "
+                f"{shape.entity} out of a copy that does not hold it."
+            )