smart-data-engine-sdk 0.1.0__py3-none-any.whl

sde/index_build.py

@@ -0,0 +1,313 @@
+"""Load the signed authorization to build indexes on the tables in force, in place.
+
+A model's design that only adds indexes to a group's source used to run as a relayout: a fresh copy
+with the index, then a cutover that copies and compares every row while the source's writes are
+frozen. Measured on PostgreSQL 15.19 and ClickHouse 24.8.14.39 (24 September 2026), that pause is
+linear in the table - 22.5 s at 300 000 rows - and a table of about 400 000 rows exceeds the 30 s
+cutover budget and rolls back. Building the index on the live table moves no row and pauses no
+write.
+
+This authorization binds that build: the exact map in force and the next map, which differs from it
+only by indexes added to one group's source. Nothing else may change, because nothing else can
+change without a copy - and the tables, the write generation and every running process stay as they
+are, which is what lets the build run without a barrier.
+
+Protocol 2 also removes indexes the map in force declares, for a design that drops an index nobody
+reads or replaces one with another. The removed ones leave the next map, the others keep their
+order, and the new ones follow. The operator removes them only after its decision, one resumable
+step each: ``DROP INDEX CONCURRENTLY`` pauses no write (measured, like the build), and a process on
+the map in force that still declares a removed index reports it missing and keeps serving rows.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from collections.abc import Mapping
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Any
+
+from .canonical import CanonicalError, canonical_bytes
+from .cutover import _hex, _record, _signature
+from .errors import MapError, MigrationRefused
+from .generation import GENERATIONS_SINCE, check_map_project, json_numbers
+from .model import LogicalModel
+from .placement import PlacementMap, _verify_signature, load_map
+
+INDEX_PROTOCOL = 1
+"""Indexes added to one group's source, built on the tables in force; no copy, no cutover."""
+
+INDEX_CHANGE_PROTOCOL = 2
+"""Indexes added to and removed from one group's source, in place; at least one removed."""
+
+_PROTOCOLS = (INDEX_PROTOCOL, INDEX_CHANGE_PROTOCOL)
+
+MAX_BUILD_BUDGET_MS = 86_400_000
+"""A day. The budget bounds a build on a server that stopped answering; nothing is paused while it
+runs, so it is not a pause budget and it is deliberately far longer than a cutover's."""
+
+_FIELDS = {
+    "kind",
+    "protocol",
+    "index_id",
+    "project_id",
+    "group",
+    "current",
+    "prepared",
+    "build_budget_ms",
+    "signature",
+}
+
+
+_SUBJECT = "index build"
+
+
+def index_build_name(index_id: str, position: int) -> str:
+    """The physical name of the ``position``-th index an in-place build adds (one-based)."""
+    _hex(index_id, 32, "index_id", _SUBJECT)
+    if type(position) is not int or not 1 <= position <= 999999:
+        raise MigrationRefused("index build position must be an integer from 1 through 999999")
+    return f"sde_i_{index_id}_{position:06d}"
+
+
+@dataclass(frozen=True)
+class IndexPlan:
+    index_id: str
+    project_id: str
+    group: str
+    current: PlacementMap
+    prepared: PlacementMap
+    build_budget_ms: int
+    added: tuple[Mapping[str, Any], ...]
+    """The new index definitions, in position order, exactly as the prepared map carries them."""
+    verified_with: str | None
+    removed: tuple[Mapping[str, Any], ...] = ()
+    """Protocol 2: the definitions in force the next map drops, in their order in force."""
+    fingerprint: str | None = field(default=None, init=False)
+    _document: bytes = field(default=b"", init=False, repr=False)
+
+    def _loaded(self) -> None:
+        if self.fingerprint is None or not self._document:
+            raise MigrationRefused("an index build requires an immutable loaded authorization")
+
+    def as_record(self) -> dict[str, Any]:
+        self._loaded()
+        value: dict[str, Any] = json.loads(self._document)
+        return value
+
+    @property
+    def protocol(self) -> int:
+        return int(self.as_record()["protocol"])
+
+    def prepared_payload(self) -> bytes:
+        return canonical_bytes(self.as_record()["prepared"])
+
+    def check_current(self, current: PlacementMap) -> None:
+        self._loaded()
+        check_map_project(current, self.project_id)
+        if not current.signed or current.fingerprint != self.current.fingerprint:
+            raise MigrationRefused("index build authorization does not name the signed current map")
+
+
+@dataclass(frozen=True, init=False)
+class IndexReceipt:
+    _payload: bytes
+
+    def __init__(self, record: Mapping[str, Any]) -> None:
+        object.__setattr__(
+            self,
+            "_payload",
+            json.dumps(
+                record, ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False
+            ).encode("utf-8"),
+        )
+
+    def as_record(self) -> dict[str, Any]:
+        record: dict[str, Any] = json.loads(self._payload)
+        return record
+
+
+def _names(document: Mapping[str, Any]) -> set[str]:
+    """Every table and index name any layout of a map document uses."""
+    found: set[str] = set()
+    for raw_group in document["groups"].values():
+        for material in (raw_group["source"], *raw_group.get("derived", [])):
+            layout = material["layout"]
+            found.update(str(table) for table in layout.get("tables", {}).values())
+            found.update(str(index["name"]) for index in layout.get("indexes", []) or [])
+    return found
+
+
+def _load(
+    raw: Mapping[str, Any],
+    model: LogicalModel,
+    project_id: str,
+    public_key: bytes | Mapping[str, bytes],
+) -> IndexPlan:
+    body = _record(json_numbers(deepcopy(raw)), "authorization", _SUBJECT)
+    if set(body) != _FIELDS:
+        raise MigrationRefused("index build authorization has missing or unknown fields")
+    if (
+        type(body["protocol"]) is not int
+        or body["protocol"] not in _PROTOCOLS
+        or body["kind"] != "sde-index"
+    ):
+        raise MigrationRefused("unsupported index build authorization kind or protocol")
+    protocol = int(body["protocol"])
+    identity = _hex(body["index_id"], 32, "index_id", _SUBJECT)
+    local = _hex(body["project_id"], 32, "project_id", _SUBJECT)
+    if local != project_id:
+        raise MigrationRefused("index build authorization belongs to another local project")
+    group = body["group"]
+    if not isinstance(group, str) or not group:
+        raise MigrationRefused("index build group must be a nonempty string")
+    budget = body["build_budget_ms"]
+    if type(budget) is not int or not 1 <= budget <= MAX_BUILD_BUDGET_MS:
+        raise MigrationRefused(
+            f"index build budget must be an integer from 1 through {MAX_BUILD_BUDGET_MS} ms"
+        )
+    _signature(body, _SUBJECT)
+    verified = _verify_signature(body, public_key)
+    maps = []
+    for name in ("current", "prepared"):
+        document = _record(body[name], name, _SUBJECT)
+        _signature(document, _SUBJECT)
+        for raw_group in _record(document.get("groups"), "groups", _SUBJECT).values():
+            value = _record(raw_group, "group", _SUBJECT)
+            copies = value.get("derived", [])
+            if not isinstance(copies, list):
+                raise MigrationRefused("index build derived copies must be an array")
+            for raw_material in (value.get("source"), *copies):
+                raw_layout = _record(
+                    _record(raw_material, "materialization", _SUBJECT).get("layout"),
+                    "layout",
+                    _SUBJECT,
+                )
+                if raw_layout.get("auto"):
+                    raise MigrationRefused("index build maps need explicit physical layouts")
+        parsed = load_map(document, model=model, public_key=public_key, require_signature=True)
+        if parsed.contract < GENERATIONS_SINCE:
+            raise MigrationRefused(
+                f"index build protocol {protocol} requires map contract "
+                f"{GENERATIONS_SINCE} or later"
+            )
+        check_map_project(parsed, project_id)
+        for placement in parsed.groups.values():
+            for material in placement.all():
+                if not material.layout.tables or not material.layout.columns:
+                    raise MigrationRefused("index build maps need explicit physical layouts")
+        maps.append(parsed)
+    current, prepared = maps
+    # The prepared map may raise the contract, because an index method first appears here - a
+    # BRIN index under a contract-4 current map. It may not lower it: a lower number would tell an
+    # older library it may ignore keys that the current map already relies on.
+    if prepared.contract < current.contract:
+        raise MigrationRefused("an index build cannot lower the placement map contract")
+    if current.map_version >= prepared.map_version:
+        raise MigrationRefused("an index build must allocate a newer prepared map")
+    if group not in current.groups or set(current.groups) != set(prepared.groups):
+        raise MigrationRefused("an index build cannot add or remove colocation groups")
+    old, new = current.groups[group], prepared.groups[group]
+    if old.derived or old.also_write or new.derived or new.also_write:
+        raise MigrationRefused("an index build begins and ends with a source-only group")
+    if new.write_epoch != old.write_epoch:
+        # Nothing is fenced: the tables stay and every running process keeps writing to them.
+        raise MigrationRefused("an index build keeps the source's write generation")
+    current_raw, prepared_raw = body["current"], body["prepared"]
+    old_group, new_group = current_raw["groups"][group], prepared_raw["groups"][group]
+    if set(old_group) != {"source", "write_epoch"} or set(new_group) != {"source", "write_epoch"}:
+        raise MigrationRefused("an index build group is a source and its write generation only")
+
+    def without_indexes(material: Mapping[str, Any]) -> dict[str, Any]:
+        layout = {key: value for key, value in material["layout"].items() if key != "indexes"}
+        return {**material, "layout": layout}
+
+    if canonical_bytes(without_indexes(old_group["source"])) != canonical_bytes(
+        without_indexes(new_group["source"])
+    ):
+        raise MigrationRefused(
+            "an index build changes nothing about the source but its indexes; a new key order, "
+            "partition, table or engine is a relayout or a move"
+        )
+    in_force = list(old_group["source"]["layout"].get("indexes", []) or [])
+    after = list(new_group["source"]["layout"].get("indexes", []) or [])
+    if protocol == INDEX_PROTOCOL:
+        kept, removed = in_force, []
+        if len(after) <= len(kept) or canonical_bytes(after[: len(kept)]) != canonical_bytes(kept):
+            raise MigrationRefused(
+                "an index build keeps every index in force, in order, and adds at least one after "
+                "them"
+            )
+    else:
+        remaining = {str(index.get("name")) for index in after}
+        kept = [index for index in in_force if str(index.get("name")) in remaining]
+        removed = [index for index in in_force if str(index.get("name")) not in remaining]
+        if not removed:
+            raise MigrationRefused("index build protocol 2 removes at least one index in force")
+        if canonical_bytes(after[: len(kept)]) != canonical_bytes(kept):
+            raise MigrationRefused(
+                "an index change keeps the other indexes in force, in order, before the new ones"
+            )
+    added = after[len(kept) :]
+    for position, index in enumerate(added, start=1):
+        if index.get("name") != index_build_name(identity, position):
+            raise MigrationRefused(
+                "new indexes need fresh names bound to the index build id and their position"
+            )
+    if _names(current_raw) & {str(index["name"]) for index in added}:
+        raise MigrationRefused("an index build cannot reuse a name the current map uses")
+    stable = {
+        key: value
+        for key, value in current_raw.items()
+        if key not in {"signature", "map_version", "groups", "contract"}
+    }
+    changed = {
+        key: value
+        for key, value in prepared_raw.items()
+        if key not in {"signature", "map_version", "groups", "contract"}
+    }
+    if canonical_bytes(stable) != canonical_bytes(changed) or dict(current.routing) != dict(
+        prepared.routing
+    ):
+        raise MigrationRefused("an index build cannot change routing or other map attributes")
+    for other in current.groups:
+        if other != group and canonical_bytes(current_raw["groups"][other]) != canonical_bytes(
+            prepared_raw["groups"][other]
+        ):
+            raise MigrationRefused("an index build cannot change an unaffected group")
+    gone = {str(index["name"]) for index in removed}
+    plan = IndexPlan(
+        identity,
+        local,
+        group,
+        current,
+        prepared,
+        budget,
+        # From the loaded maps, which freeze nested structures, not from the caller's dictionaries.
+        tuple(new.source.layout.indexes[len(kept) :]),
+        verified,
+        tuple(index for index in old.source.layout.indexes if str(index["name"]) in gone),
+    )
+    object.__setattr__(plan, "_document", canonical_bytes(body))
+    object.__setattr__(
+        plan,
+        "fingerprint",
+        hashlib.sha256(
+            canonical_bytes({key: value for key, value in body.items() if key != "signature"})
+        ).hexdigest(),
+    )
+    return plan
+
+
+def load_index_plan(
+    raw: Mapping[str, Any],
+    *,
+    model: LogicalModel,
+    project_id: str,
+    public_key: bytes | Mapping[str, bytes],
+) -> IndexPlan:
+    try:
+        return _load(raw, model, project_id, public_key)
+    except (MapError, CanonicalError, ValueError, TypeError, KeyError) as exc:
+        raise MigrationRefused(f"index build authorization refused: {exc}") from exc

sde/index_operator.py

@@ -0,0 +1,347 @@
+"""Durable in-place index builds using only customer-local connections.
+
+The build pauses nothing: writers on the map in force keep writing to the same tables while the
+index is built beside them, and the next map differs only by declaring it. So the operation holds no
+barrier and raises no generation - it refuses to start next to anybody else's barrier instead - and
+its two terminal outcomes are ``built`` (the next map is published) and ``abandoned`` (this build's
+own indexes are removed and the map in force stays). Without a decision a recovery builds on:
+unlike a cutover, where the source stands frozen until a decision, here nothing waits, and an index
+that gets built is harmless.
+"""
+
+from __future__ import annotations
+
+import json
+from functools import partial
+from time import monotonic_ns
+from typing import TYPE_CHECKING, Any
+
+from .errors import MigrationRefused
+from .index_build import IndexPlan, IndexReceipt, load_index_plan
+from .local_cutover import CutoverRecoveryRequired
+from .physical import METHODS_BY_DIALECT, index_method, refuse_findings
+from .placement import WATERMARK_TABLE
+
+if TYPE_CHECKING:
+    from .local_cutover import LocalCutover
+
+
+def _snapshot(operator: LocalCutover, plan: IndexPlan, state: dict[str, Any]) -> dict[str, Any]:
+    from .engines._index_build import NativeIndexBuild
+
+    plan.check_current(operator.active_map())
+    placement = plan.current.groups[plan.group]
+    source = placement.source
+    if source.engine not in operator.engines:
+        raise MigrationRefused("an index build is missing its configured local engine binding")
+    native = operator.native[source.engine]
+    for index in plan.added:
+        if index_method(index) not in METHODS_BY_DIALECT.get(native.dialect, ()):
+            raise MigrationRefused(
+                f"{native.dialect} cannot build a {index_method(index)} index in place"
+            )
+    engine = operator.engines[source.engine]
+    # The design in force, read back before any DDL: a build beside a table that already differs
+    # from its map would be refused only at publication, after the work.
+    keys = {entity: operator.model.entity(entity).key for entity in source.layout.tables}
+    refuse_findings(engine.validate_schema(source.layout, keys=keys), MigrationRefused)
+    tables: dict[str, dict[str, str]] = {}
+    builder = NativeIndexBuild(native)
+    for entity, table in sorted(source.layout.tables.items()):
+        identity = native.identity(table)
+        observed = engine.write_fence(table, project_id=operator.project_id).state()
+        if not observed.complete or observed.epoch != placement.write_epoch or observed.holds:
+            # A build beside another operation's barrier would publish a map that operation's
+            # own next map does not know about.
+            raise MigrationRefused(
+                "an index build needs the current generation without another barrier"
+            )
+        tables[entity] = identity.as_record()
+        for index in plan.added:
+            if index["entity"] == entity:
+                status, reason = builder.inspect(identity, index)
+                if status == "foreign":
+                    raise MigrationRefused(reason)
+        for index in plan.removed:
+            # Before any DDL, as for a build beside a table that differs from its map: removing
+            # an index the table does not hold as declared would publish a map about another table.
+            if index["entity"] == entity and builder.declared(identity, index) not in (
+                "ready",
+                "unfinished",
+            ):
+                raise MigrationRefused(
+                    "an index the map in force declares is not on its table as declared; "
+                    "inspect the table before changing its indexes"
+                )
+    allowed: dict[str, set[str]] = {name: {WATERMARK_TABLE} for name in operator.engines}
+    for placed in plan.current.groups.values():
+        for material in placed.all():
+            allowed[material.engine].update(material.layout.tables.values())
+    bindings = {}
+    for name, driver in operator.native.items():
+        value = operator.engines[name].map_watermark()
+        if value is not None and value > plan.current.map_version:
+            raise MigrationRefused("a newer map was adopted before this index build")
+        driver.qualify([WATERMARK_TABLE], sorted(allowed[name]))
+        bindings[name] = {
+            "endpoint": list(driver.endpoint()),
+            "watermark_identity": driver.identity(WATERMARK_TABLE).as_record(),
+            "principals": dict(driver.principal_ids),
+            "allowed_tables": sorted(allowed[name]),
+        }
+    return {
+        "kind": "index",
+        "plan_id": plan.index_id,
+        "plan_fingerprint": plan.fingerprint,
+        "plan": plan.as_record(),
+        "phase": "index_prepared",
+        "pending_hold": None,
+        "decision": None,
+        "bindings": bindings,
+        "engine": source.engine,
+        "epoch": placement.write_epoch,
+        "tables": tables,
+        "indexes": [
+            {"entity": str(index["entity"]), "name": str(index["name"]), "index": dict(index)}
+            for index in plan.added
+        ],
+        "removed": [
+            {"entity": str(index["entity"]), "name": str(index["name"]), "index": dict(index)}
+            for index in plan.removed
+        ],
+    }
+
+
+def _recheck(operator: LocalCutover, execution: dict[str, Any]) -> None:
+    """Everything a publication stands on: bindings, logins, tables and a generation to itself."""
+    from .engines._operator import TableIdentity
+
+    for name, binding in execution["bindings"].items():
+        native = operator.native[name]
+        if list(native.endpoint()) != binding["endpoint"]:
+            raise MigrationRefused("an index build binding names another native database")
+        expected_watermark = TableIdentity(**binding["watermark_identity"])
+        if native.identity(WATERMARK_TABLE).physical_key != expected_watermark.physical_key:
+            raise MigrationRefused("an index build's namespace or watermark identity changed")
+        native.qualify([WATERMARK_TABLE], binding["allowed_tables"])
+        if native.principal_ids != binding["principals"]:
+            raise MigrationRefused("a runtime login identity changed during the index build")
+    engine = operator.engines[execution["engine"]]
+    native = operator.native[execution["engine"]]
+    for record in execution["tables"].values():
+        expected = TableIdentity(**record)
+        if native.identity(expected.name).physical_key != expected.physical_key:
+            raise MigrationRefused("a table was replaced during its index build")
+        observed = engine.write_fence(expected.name, project_id=operator.project_id).state()
+        if not observed.complete or observed.epoch != execution["epoch"] or observed.holds:
+            raise MigrationRefused("another barrier or generation appeared during the index build")
+
+
+def _finish(
+    operator: LocalCutover, state: dict[str, Any], plan: IndexPlan, *, recovered: bool
+) -> IndexReceipt:
+    from .engines._index_build import NativeIndexBuild
+    from .engines._operator import TableIdentity
+
+    execution = state["execution"]
+    builder = NativeIndexBuild(operator.native[execution["engine"]])
+    if execution["decision"] == "abandoned":
+        # Removing our own indexes publishes nothing, so it needs only the same database - each
+        # drop checks its table's identity - and neither the logins nor a barrier-free table.
+        # That is what keeps abandonment available when the build cannot finish.
+        binding = execution["bindings"][execution["engine"]]
+        if list(operator.native[execution["engine"]].endpoint()) != binding["endpoint"]:
+            raise MigrationRefused("an index build binding names another native database")
+    else:
+        _recheck(operator, execution)
+    rows = [
+        (TableIdentity(**execution["tables"][row["entity"]]), row["index"])
+        for row in execution["indexes"]
+    ]
+    if execution["decision"] is None:
+        for table, index in rows:
+            operator._step(
+                state, "index_build_" + str(index["name"]), partial(builder.build, table, index)
+            )
+
+        def qualify() -> None:
+            # The whole next layout, read back: a build that left the right index beside a table
+            # that no longer matches would still be publishing a map the engine does not hold.
+            source = plan.prepared.groups[plan.group].source
+            keys = {entity: operator.model.entity(entity).key for entity in source.layout.tables}
+            refuse_findings(
+                operator.engines[execution["engine"]].validate_schema(source.layout, keys=keys),
+                MigrationRefused,
+            )
+            for table, index in rows:
+                if builder.status(table, index) != "ready":
+                    raise MigrationRefused("an index is not ready for publication")
+            _recheck(operator, execution)
+
+        operator._step(state, "index_qualify", qualify)
+        execution["decision"] = "built"
+        operator.store.write(state)
+        operator._after_step("index_decision")
+    if execution["decision"] == "built":
+
+        def watermarks() -> None:
+            for engine in operator.engines.values():
+                observed = engine.map_watermark()
+                if observed is not None and observed > plan.prepared.map_version:
+                    raise MigrationRefused("a newer map appeared while finishing the index build")
+                engine.record_map_version(
+                    plan.prepared.map_version, model_version=operator.model.version
+                )
+
+        operator._step(state, "index_watermarks", watermarks)
+
+        def publish() -> None:
+            current = operator.active_map()
+            if current.fingerprint not in (plan.current.fingerprint, plan.prepared.fingerprint):
+                raise MigrationRefused("another active map replaced the index build instruction")
+            operator.store.publish(plan.prepared_payload())
+
+        operator._step(state, "index_publish", publish)
+        # Removals follow the decision and the publication: each is a step of its own, resumed
+        # after a crash, and a process still on the map in force only reports a removed index
+        # missing. Before this point nothing of the map in force was touched.
+        for row in execution.get("removed", ()):
+            table = TableIdentity(**execution["tables"][row["entity"]])
+            operator._step(
+                state,
+                "index_remove_" + str(row["name"]),
+                partial(builder.remove, table, row["index"]),
+            )
+        outcome, active = "built", plan.prepared
+    else:
+        for table, index in rows:
+            operator._step(
+                state, "index_drop_" + str(index["name"]), partial(builder.drop, table, index)
+            )
+        if operator.active_map().fingerprint != plan.current.fingerprint:
+            raise MigrationRefused("an abandoned index build found another active map")
+        outcome, active = "abandoned", plan.current
+    receipt: dict[str, Any] = {
+        "protocol": plan.protocol,
+        "index_id": plan.index_id,
+        "index_fingerprint": plan.fingerprint,
+        "project_id": operator.project_id,
+        "group": plan.group,
+        "outcome": outcome,
+        "map_version": active.map_version,
+        "map_fingerprint": active.fingerprint,
+        "indexes": [
+            {
+                "engine": execution["engine"],
+                "entity": row["entity"],
+                "name": row["name"],
+                "table": execution["tables"][row["entity"]],
+            }
+            for row in execution["indexes"]
+        ],
+        "elapsed_ms": operator._elapsed(),
+        "recovered": recovered,
+    }
+    if plan.protocol == 2:
+        # Abandoned before its decision, a change removed nothing: the rows say what was removed.
+        receipt["removed"] = [
+            {
+                "engine": execution["engine"],
+                "entity": row["entity"],
+                "name": row["name"],
+                "table": execution["tables"][row["entity"]],
+            }
+            for row in execution.get("removed", ())
+            if outcome == "built"
+        ]
+    state["indexes"][plan.index_id] = {
+        "plan_fingerprint": plan.fingerprint,
+        "plan": plan.as_record(),
+        "receipt": receipt,
+    }
+    if outcome == "built":
+        state["active_map"] = json.loads(plan.prepared_payload())
+    state["execution"] = None
+    operator.store.write(state)
+    operator._after_step("index_complete")
+    return IndexReceipt(receipt)
+
+
+def _completed(operator: LocalCutover, state: dict[str, Any], plan: IndexPlan) -> IndexReceipt:
+    complete = state.get("indexes", {}).get(plan.index_id)
+    assert complete is not None
+    if complete["plan_fingerprint"] != plan.fingerprint:
+        raise MigrationRefused("the completed index build id names another authorization")
+    operator.store.confirm()
+    return IndexReceipt(complete["receipt"])
+
+
+def execute_index(operator: LocalCutover, plan: IndexPlan) -> IndexReceipt:
+    with operator.store.lock():
+        state = operator.store.read()
+        if plan.index_id in state.get("indexes", {}):
+            return _completed(operator, state, plan)
+        if state["execution"] is not None:
+            raise CutoverRecoveryRequired("an unfinished local operation requires resume")
+        execution = _snapshot(operator, plan, state)
+        state.setdefault("stages", {})
+        state.setdefault("indexes", {})
+        state["execution"] = execution
+        operator.store.write(state)
+        operator._after_step("index_prepared")
+        operator._started, operator._enforce_budget = monotonic_ns(), False
+        try:
+            return _finish(operator, state, plan, recovered=False)
+        except Exception as exc:
+            raise CutoverRecoveryRequired(
+                "the local index build needs recovery; preserve its project state"
+            ) from exc
+
+
+def _stored_plan(operator: LocalCutover, execution: dict[str, Any]) -> IndexPlan:
+    plan = load_index_plan(
+        execution["plan"],
+        model=operator.model,
+        project_id=operator.project_id,
+        public_key=operator.keys,
+    )
+    if plan.fingerprint != execution["plan_fingerprint"]:
+        raise MigrationRefused("index build recovery names another signed authorization")
+    return plan
+
+
+def resume_index(operator: LocalCutover, state: dict[str, Any]) -> IndexReceipt:
+    plan = _stored_plan(operator, state["execution"])
+    operator._alarm.arm(plan.build_budget_ms)
+    operator._started, operator._enforce_budget = monotonic_ns(), False
+    try:
+        return _finish(operator, state, plan, recovered=True)
+    except Exception as exc:
+        raise CutoverRecoveryRequired(
+            "index build recovery remains incomplete; preserve its state"
+        ) from exc
+
+
+def abandon_index(operator: LocalCutover) -> IndexReceipt:
+    with operator.store.lock():
+        state = operator.store.read()
+        execution = state["execution"]
+        if execution is None or execution.get("kind") != "index":
+            raise MigrationRefused("there is no unfinished index build to abandon")
+        if execution["decision"] == "built":
+            raise MigrationRefused(
+                "a built index cannot be abandoned: its next map is decided; resume to publish it"
+            )
+        plan = _stored_plan(operator, execution)
+        operator._alarm.arm(plan.build_budget_ms)
+        if execution["decision"] is None:
+            execution["decision"] = "abandoned"
+            operator.store.write(state)
+            operator._after_step("index_abandoned")
+        operator._started, operator._enforce_budget = monotonic_ns(), False
+        try:
+            return _finish(operator, state, plan, recovered=True)
+        except Exception as exc:
+            raise CutoverRecoveryRequired(
+                "abandoning the index build remains incomplete; preserve its state"
+            ) from exc