stackbox 0.1.0__py3-none-any.whl

stackbox/router.py

@@ -0,0 +1,1049 @@
+"""Placement, fallback, circuit breaking and reconciliation."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any
+
+from stackbox import ledger as L
+from stackbox.backends import (
+    GONE_STATES,
+    GW_ID_KEY,
+    MANAGED_KEY,
+    PIN_KEY,
+    ROUTER_KEYS,
+    Backend,
+    BackendError,
+    Created,
+    CreateFailure,
+    ReconcileError,
+    gateway_instance_id,
+)
+from stackbox.config import RouterConfig
+
+logger = logging.getLogger(__name__)
+
+
+# Providers that delete asynchronously keep listing a sandbox briefly after a delete.
+DELETE_GRACE_S = 60.0
+
+# Quota discovery. When a backend refuses a create for quota or capacity (429-class)
+# while holding N sandboxes, its limit becomes N: creates beyond it go elsewhere (or
+# wait) without paying for another refusal (Daytona takes ~3.4 s to refuse). The
+# limit then grows by one every QUOTA_PROBE_S without a refusal, so it follows the
+# quota back up. A refusal while holding nothing (an account-wide limit, or sandboxes
+# still being torn down) instead skips the backend for 1 s, doubling per consecutive
+# refusal, up to SATURATION_MAX_BACKOFF_S.
+SATURATION_MAX_BACKOFF_S = 8.0
+QUOTA_PROBE_S = 15.0
+
+_SATURATED: Any = object()  # _attempt result: the backend refused for capacity
+
+
+class PlacementError(Exception):
+    def __init__(
+        self,
+        status: int,
+        code: str,
+        message: str,
+        details: Any = None,
+        hint: str | None = None,
+    ):
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.details = details
+        self.hint = hint
+
+
+@dataclass
+class Breaker:
+    consecutive_failures: int = 0
+    open_until: float = 0.0
+
+    def is_open(self) -> bool:
+        return time.monotonic() < self.open_until
+
+
+@dataclass
+class BackendStats:
+    create_latencies_ms: list[float] = field(default_factory=list)
+    placements: int = 0
+    cold_starts: int = 0
+    saturations: int = 0
+
+
+class StartupModel:
+    """Expected seconds from create request to a usable sandbox, per (backend, image).
+
+    An exponentially weighted moving average of observed creates. Cold starts (the
+    create included an image build) are tracked separately so one slow first boot
+    does not make a backend look slow once the image is prepared.
+    """
+
+    def __init__(self, alpha: float):
+        self.alpha = alpha
+        self.warm: dict[tuple[str, str | None], float] = {}
+        self.cold: dict[tuple[str, str | None], float] = {}
+        self.samples: dict[tuple[str, str | None], int] = {}
+
+    def observe(
+        self, backend: str, image: str | None, seconds: float, cold: bool
+    ) -> None:
+        table = self.cold if cold else self.warm
+        key = (backend, image)
+        prev = table.get(key)
+        table[key] = (
+            seconds if prev is None else self.alpha * seconds + (1 - self.alpha) * prev
+        )
+        if not cold:
+            self.samples[key] = self.samples.get(key, 0) + 1
+
+    def expected(self, backend: Backend, image: str | None, support: str) -> float:
+        key = (backend.name, image)
+        if support == "cold":
+            return self.cold.get(key, backend.cfg.cold_start_prior_s)
+        if key in self.warm:
+            return self.warm[key]
+        return backend.cfg.startup_prior_s
+
+
+def parse_cpu(v: Any) -> float | None:
+    if v is None:
+        return None
+    s = str(v).strip()
+    try:
+        return float(s[:-1]) / 1000 if s.endswith("m") else float(s)
+    except ValueError:
+        return None
+
+
+_MEM_UNITS = {
+    "Ki": 2**-20,
+    "Mi": 2**-10,
+    "Gi": 1,
+    "Ti": 2**10,
+    "K": 1e3 / 2**30,
+    "M": 1e6 / 2**30,
+    "G": 1e9 / 2**30,
+}
+
+
+def parse_memory_gib(v: Any) -> float | None:
+    if v is None:
+        return None
+    s = str(v).strip()
+    for unit, factor in sorted(_MEM_UNITS.items(), key=lambda kv: -len(kv[0])):
+        if s.endswith(unit):
+            try:
+                return float(s[: -len(unit)]) * factor
+            except ValueError:
+                return None
+    try:
+        return float(s) / 2**30
+    except ValueError:
+        return None
+
+
+STRATEGIES = ("priority", "cost", "latency", "balanced")
+
+
+@dataclass
+class Request:
+    """What placement needs to know about a create, including per-request routing
+    preferences (OpenRouter-style: strategy, an ordered allow-list, a price ceiling)."""
+
+    image: str | None
+    cpu: float | None
+    memory_gib: float | None
+    strategy: str | None = None
+    providers: list[str] | None = None
+    max_price: float | None = None
+
+
+def parse_preferences(
+    headers: dict[str, str], metadata: dict[str, str]
+) -> tuple[str | None, list[str] | None, float | None]:
+    """X-Stackbox-* headers, else stackbox.dev/* metadata. Raises PlacementError(400)."""
+    from stackbox.backends.base import MAX_PRICE_KEY, PROVIDERS_KEY, STRATEGY_KEY
+
+    def pick(header: str, key: str) -> str | None:
+        v = headers.get(header) or metadata.get(key)
+        return v.strip() if v else None
+
+    strategy = pick("x-stackbox-strategy", STRATEGY_KEY)
+    if strategy is not None and strategy not in STRATEGIES:
+        raise PlacementError(
+            400,
+            "STACKBOX::BAD_PREFERENCE",
+            f"unknown routing strategy '{strategy}'",
+            hint=f"use one of: {', '.join(STRATEGIES)}",
+        )
+    raw = pick("x-stackbox-providers", PROVIDERS_KEY)
+    providers = (
+        [p for p in raw.replace(".", ",").split(",") if p.strip()] if raw else None
+    )
+    providers = [p.strip() for p in providers] if providers else None
+    price = pick("x-stackbox-max-price", MAX_PRICE_KEY)
+    try:
+        max_price = float(price) if price else None
+    except ValueError as exc:
+        raise PlacementError(
+            400,
+            "STACKBOX::BAD_PREFERENCE",
+            f"max price '{price}' is not a number",
+            hint="give USD per sandbox-hour, e.g. X-Stackbox-Max-Price: 0.10",
+        ) from exc
+    return strategy, providers, max_price
+
+
+@dataclass
+class Choice:
+    backend: Backend
+    support: str  # "ready" | "cold"
+    price_per_hour: float
+    expected_startup_s: float
+    score: float = 0.0
+    exploring: bool = False  # ranked optimistically until it has enough samples
+
+
+class Router:
+    def __init__(
+        self,
+        cfg: RouterConfig,
+        ledger: L.Ledger,
+        backends: dict[str, Backend],
+    ):
+        self.cfg = cfg
+        self.ledger = ledger
+        self.backends = backends
+        self.breakers = {name: Breaker() for name in backends}
+        self.saturated_until: dict[str, float] = {name: 0.0 for name in backends}
+        self.saturation_level: dict[str, int] = {name: 0 for name in backends}
+        self.learned_cap: dict[str, tuple[int, float] | None] = {
+            name: None for name in backends
+        }
+        self.stats = {name: BackendStats() for name in backends}
+        self.startup = StartupModel(cfg.routing.ewma_alpha)
+        self.instance_id = gateway_instance_id(ledger)
+        for b in backends.values():
+            b.instance_id = self.instance_id
+        for backend, image, seconds in ledger.startup_samples():
+            if backend in backends:
+                self.startup.observe(backend, image, seconds, cold=False)
+
+    # ------------------------------------------------------------------ create
+    def rank(self, req: Request, pinned: str | None = None) -> list[Choice]:
+        """Backends that can serve the request, best first under the routing strategy.
+
+        Ignores capacity and breakers; _candidates applies those.
+        """
+        known = ", ".join(self.backends)
+        if pinned is not None:
+            if pinned not in self.backends:
+                raise PlacementError(
+                    400,
+                    "STACKBOX::UNKNOWN_BACKEND",
+                    f"unknown backend '{pinned}'",
+                    hint=f"configured backends: {known}",
+                )
+            pool = [self.backends[pinned]]
+        elif req.providers:
+            unknown = [p for p in req.providers if p not in self.backends]
+            if unknown:
+                raise PlacementError(
+                    400,
+                    "STACKBOX::UNKNOWN_BACKEND",
+                    f"unknown provider(s) in preferences: {', '.join(unknown)}",
+                    hint=f"configured backends: {known}",
+                )
+            pool = [self.backends[p] for p in req.providers]
+        else:
+            pool = list(self.backends.values())
+        choices: list[Choice] = []
+        for b in pool:
+            support = b.image_support(req.image, req.cpu, req.memory_gib)
+            if support is None:
+                continue
+            price = b.price_per_hour(req.image, req.cpu, req.memory_gib)
+            if req.max_price is not None and price > req.max_price + 1e-12:
+                continue
+            choices.append(
+                Choice(
+                    backend=b,
+                    support=support,
+                    price_per_hour=price,
+                    expected_startup_s=self.startup.expected(b, req.image, support),
+                )
+            )
+        strategy = req.strategy or self.cfg.routing.strategy
+        if req.providers and req.strategy is None:
+            # An allow-list without a strategy is an order: try them as listed.
+            order = {p: i for i, p in enumerate(req.providers)}
+            choices.sort(key=lambda c: order[c.backend.name])
+            return choices
+        if strategy in ("latency", "balanced") and self.cfg.routing.explore_samples:
+            need = self.cfg.routing.explore_samples
+            unmeasured = [
+                c
+                for c in choices
+                if c.support == "ready"
+                and self.startup.samples.get((c.backend.name, req.image), 0) < need
+            ]
+            measured = [
+                c for c in choices if c not in unmeasured and c.support == "ready"
+            ]
+            if unmeasured and measured:
+                best = min(c.expected_startup_s for c in measured)
+                for c in unmeasured:
+                    c.expected_startup_s = max(0.0, best - 0.01)
+                    c.exploring = True
+        if req.providers:  # with a strategy, list position breaks ties
+            order = {p: i for i, p in enumerate(req.providers)}
+            tie = lambda c: (order[c.backend.name], c.backend.name)
+        else:
+            tie = lambda c: (c.backend.cfg.priority, c.backend.name)
+        if strategy == "priority":
+            choices.sort(key=tie)
+        elif strategy == "cost":
+            choices.sort(
+                key=lambda c: (
+                    round(c.price_per_hour, 6),
+                    c.expected_startup_s,
+                    *tie(c),
+                )
+            )
+        elif strategy == "latency":
+            choices.sort(
+                key=lambda c: (
+                    round(c.expected_startup_s, 3),
+                    c.price_per_hour,
+                    *tie(c),
+                )
+            )
+        else:  # balanced: each term relative to the best candidate on that axis
+            if choices:
+                min_price = min(c.price_per_hour for c in choices)
+                min_start = min(c.expected_startup_s for c in choices)
+                for c in choices:
+                    c.score = self.cfg.routing.cost_weight * (
+                        (c.price_per_hour + 1e-3) / (min_price + 1e-3)
+                    ) + self.cfg.routing.latency_weight * (
+                        (c.expected_startup_s + 0.1) / (min_start + 0.1)
+                    )
+            choices.sort(key=lambda c: (round(c.score, 6), *tie(c)))
+        return choices
+
+    def explain(self, req: Request, pinned: str | None = None) -> list[dict[str, Any]]:
+        """Every backend, in routing order, with why it would or would not get this sandbox."""
+        try:
+            ranked = self.rank(req, pinned)
+        except PlacementError:
+            ranked = []
+        order = {c.backend.name: (i, c) for i, c in enumerate(ranked)}
+        rows: list[dict[str, Any]] = []
+        placed = False
+        for name, b in sorted(
+            self.backends.items(),
+            key=lambda kv: order.get(kv[0], (len(order), None))[0],
+        ):
+            row: dict[str, Any] = {"backend": name, "kind": b.kind, "rank": None}
+            if name in order:
+                i, c = order[name]
+                row.update(
+                    rank=i + 1,
+                    price_per_hour=round(c.price_per_hour, 6),
+                    expected_startup_s=round(c.expected_startup_s, 2),
+                    support=c.support,
+                    exploring=c.exploring,
+                )
+                held, limit = self.ledger.held_count(name), self.limit(b)
+                if self.breakers[name].is_open():
+                    status = "circuit breaker open after repeated failures (tried last)"
+                elif self._is_saturated(name):
+                    left = self.saturated_until[name] - time.monotonic()
+                    status = f"at provider quota; retrying in {left:.0f} s"
+                elif held >= limit:
+                    quota = (
+                        " (learned from provider refusals)"
+                        if limit < b.cfg.max_active
+                        else ""
+                    )
+                    status = f"full: {held}/{limit}{quota}"
+                else:
+                    status = "available" if placed else "next placement goes here"
+                    placed = True
+                if c.support == "cold":
+                    status += "; first start includes an image build"
+                if c.exploring:
+                    status += "; still measuring startup"
+                row["status"] = status
+            else:
+                row["status"] = self._why_excluded(b, req, pinned)
+            rows.append(row)
+        return rows
+
+    def _why_excluded(self, b: Backend, req: Request, pinned: str | None) -> str:
+        if pinned is not None and b.name != pinned:
+            return f"not used: request is pinned to {pinned}"
+        if req.providers and b.name not in req.providers:
+            return "not used: not in the request's provider list"
+        if b.image_support(req.image, req.cpu, req.memory_gib) is not None:
+            price = b.price_per_hour(req.image, req.cpu, req.memory_gib)
+            return f"too expensive: ${price:.3f}/h > max ${req.max_price:.3f}/h"
+        rec = (
+            b.image_record(req.image)
+            if req.image and hasattr(b, "image_record")
+            else None
+        )
+        if rec is not None and rec.status == L.IMAGE_BUILDING:
+            return f"preparing {req.image} (started {time.time() - rec.updated_at:.0f} s ago)"
+        if rec is not None and rec.status == L.IMAGE_FAILED:
+            return f"preparing {req.image} failed: {rec.error}"
+        if (
+            rec is not None
+            and rec.status == L.IMAGE_READY
+            and rec.detail
+            and "cpu" in rec.detail
+        ):
+            return (
+                f"too small: {req.image} is prepared at {rec.detail['cpu']:g} vCPU / "
+                f"{rec.detail['memory_gib']:g} GiB"
+            )
+        return f"{req.image} not prepared here; run `stackbox prepare {req.image}`"
+
+    def _candidates(self, req: Request, pinned: str | None) -> list[Choice]:
+        ordered = self.rank(req, pinned)
+        if pinned is not None and not ordered:
+            raise PlacementError(
+                503,
+                "STACKBOX::IMAGE_NOT_READY",
+                f"backend '{pinned}' cannot serve image {req.image} yet",
+            )
+        # An open breaker demotes a backend instead of excluding it: it is tried
+        # only after every healthy backend is full. Excluding it outright idles its
+        # capacity, which made creates queue behind a backend that still succeeds
+        # most of the time (see bench/RESULTS.md). Each create tries a backend at
+        # most once, so a dead backend costs one fast failure per waiting create.
+        healthy = [c for c in ordered if not self.breakers[c.backend.name].is_open()]
+        demoted = [c for c in ordered if self.breakers[c.backend.name].is_open()]
+        return healthy + demoted
+
+    def _record_failure(self, name: str) -> None:
+        br = self.breakers[name]
+        br.consecutive_failures += 1
+        if br.consecutive_failures >= self.cfg.gateway.breaker_failures:
+            br.open_until = time.monotonic() + self.cfg.gateway.breaker_cooldown_s
+            logger.warning("circuit breaker opened for backend %s", name)
+
+    def _is_saturated(self, name: str) -> bool:
+        return time.monotonic() < self.saturated_until[name]
+
+    def _record_saturation(self, name: str) -> None:
+        self.stats[name].saturations += 1
+        # Running sandboxes only: creates still in flight have not been counted by the
+        # provider yet (counting them first set Daytona's limit to 7, not its real 5).
+        held = self.ledger.active_count(name)
+        if held > 0:
+            self.learned_cap[name] = (held, time.monotonic())
+            logger.info(
+                "backend %s refused at %d held; limiting it to %d", name, held, held
+            )
+            return
+        level = self.saturation_level[name]
+        backoff = min(SATURATION_MAX_BACKOFF_S, 2.0**level)
+        self.saturated_until[name] = time.monotonic() + backoff
+        self.saturation_level[name] = level + 1
+        logger.info(
+            "backend %s at provider capacity; skipping it for %.0fs", name, backoff
+        )
+
+    def limit(self, backend: Backend) -> int:
+        """Sandboxes this backend may hold now: configured max_active, or less if the
+        provider's quota was found to be lower."""
+        learned = self.learned_cap[backend.name]
+        if learned is None:
+            return backend.cfg.max_active
+        cap, since = learned
+        grown = cap + int((time.monotonic() - since) / QUOTA_PROBE_S)
+        if grown >= backend.cfg.max_active:
+            self.learned_cap[backend.name] = None
+            return backend.cfg.max_active
+        return grown
+
+    def _record_success(self, name: str, latency_ms: float) -> None:
+        self.breakers[name] = Breaker()
+        self.saturation_level[name] = 0
+        self.stats[name].create_latencies_ms.append(latency_ms)
+        self.stats[name].placements += 1
+
+    async def create(
+        self,
+        body: dict[str, Any],
+        pin_header: str | None = None,
+        headers: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        gw_id = str(uuid.uuid4())
+        client_metadata: dict[str, str] = dict(body.get("metadata") or {})
+        pinned = pin_header or client_metadata.get(PIN_KEY)
+        limits = body.get("resourceLimits") or {}
+        img = body.get("image")
+        strategy, providers, max_price = parse_preferences(
+            {k.lower(): v for k, v in (headers or {}).items()}, client_metadata
+        )
+        req = Request(
+            image=img.get("uri") if isinstance(img, dict) else img,
+            cpu=parse_cpu(limits.get("cpu")),
+            memory_gib=parse_memory_gib(limits.get("memory")),
+            strategy=strategy,
+            providers=providers,
+            max_price=max_price,
+        )
+        self.ledger.create_intent(
+            gw_id,
+            cpu=req.cpu,
+            memory_gib=req.memory_gib,
+            pinned_backend=pinned,
+            client_metadata=client_metadata,
+            image=req.image,
+        )
+        upstream_body = {
+            **body,
+            "metadata": {
+                **client_metadata,
+                GW_ID_KEY: gw_id,
+                MANAGED_KEY: self.instance_id,
+            },
+        }
+        try:
+            return await self._place(gw_id, upstream_body, req, pinned)
+        except PlacementError as exc:
+            rec = self.ledger.get(gw_id)
+            if rec is not None and rec.state == L.PROVISIONING:
+                self.ledger.set_state(gw_id, L.FAILED, str(exc))
+            raise
+
+    async def _place(
+        self, gw_id: str, body: dict[str, Any], req: Request, pinned: str | None
+    ) -> dict[str, Any]:
+        deadline = time.monotonic() + self.cfg.gateway.queue_timeout_s
+        max_attempts = self.cfg.gateway.max_create_attempts
+        failed: dict[str, str] = {}  # last error per backend for this create
+        attempts = 0
+        while True:
+            if attempts >= max_attempts:
+                raise PlacementError(
+                    503,
+                    "STACKBOX::ALL_BACKENDS_FAILED",
+                    f"provisioning failed {attempts} times: "
+                    + "; ".join(f"{k}: {v}" for k, v in failed.items()),
+                    {"failed": failed, "backends": self.explain(req, pinned)},
+                    hint="check the provider errors above; `stackbox doctor` "
+                    "tests each provider end to end",
+                )
+            # Backends this create has not failed on come first; ones it already
+            # failed on (cleanly, so nothing exists there) are retried only after
+            # those are full. Otherwise one random failure strands the create in
+            # the queue of a full backend while the failed one has free slots.
+            ordered = self._candidates(req, pinned)
+            candidates = [c for c in ordered if c.backend.name not in failed] + [
+                c for c in ordered if c.backend.name in failed
+            ]
+            if not candidates:
+                if time.monotonic() >= deadline:
+                    why = self.explain(req, pinned)
+                    raise PlacementError(
+                        503,
+                        "STACKBOX::NO_BACKEND_FOR_IMAGE",
+                        f"no backend can run {req.image} under these constraints: "
+                        + "; ".join(f"{r['backend']}: {r['status']}" for r in why),
+                        {"backends": why},
+                        hint=f"run `stackbox prepare {req.image}`, or relax "
+                        "X-Stackbox-Providers / X-Stackbox-Max-Price",
+                    )
+                await asyncio.sleep(0.5)  # e.g. waiting for a background prepare
+                continue
+            for choice in candidates:
+                backend = choice.backend
+                if self._is_saturated(backend.name):
+                    continue  # provider said it is at capacity; treat as full
+                if not self.ledger.claim_backend(
+                    gw_id, backend.name, self.limit(backend)
+                ):
+                    continue  # full; try the next one
+                attempts += 1
+                result = await self._attempt(gw_id, backend, body, req)
+                if result is _SATURATED:
+                    attempts -= 1  # a quota refusal is not a failed attempt
+                    continue  # spill to the next backend now
+                if result is not None:
+                    return result
+                failed[backend.name] = self.ledger.get(gw_id).last_error or "failed"  # type: ignore[union-attr]
+                break  # re-evaluate candidates (breakers may have changed)
+            else:
+                # Every candidate was full. Wait for capacity.
+                if time.monotonic() >= deadline:
+                    why = self.explain(req, pinned)
+                    raise PlacementError(
+                        503,
+                        "STACKBOX::NO_CAPACITY",
+                        f"every eligible backend stayed full for {self.cfg.gateway.queue_timeout_s:.0f} s: "
+                        + "; ".join(f"{r['backend']}: {r['status']}" for r in why),
+                        {"backends": why},
+                        hint="raise max_active where your plan allows, add a provider, "
+                        "or raise gateway.queue_timeout_s",
+                    )
+                await asyncio.sleep(0.2)
+
+    async def _attempt(
+        self, gw_id: str, backend: Backend, body: dict[str, Any], req: Request
+    ) -> dict[str, Any] | None:
+        """Returns the client response on success, None if provisioning clearly failed here."""
+        attempt = self.ledger.start_attempt(gw_id, backend.name)
+        t0 = time.monotonic()
+        try:
+            created = await backend.create(body, gw_id=gw_id)
+        except CreateFailure as exc:
+            if exc.kind == "client":
+                self.ledger.finish_attempt(
+                    attempt, L.CLIENT_ERROR, exc.status, str(exc)
+                )
+                self.ledger.set_state(gw_id, L.FAILED, str(exc))
+                raise PlacementError(
+                    exc.status or 400, "STACKBOX::UPSTREAM_REJECTED", str(exc), exc.body
+                )
+            if exc.kind == "clear" and exc.status == 429:
+                self.ledger.finish_attempt(attempt, L.SATURATED, 429, str(exc))
+                self.ledger.release_backend(gw_id, f"{backend.name}: {exc}")
+                self._record_saturation(backend.name)
+                return _SATURATED
+            if exc.kind == "clear":
+                self.ledger.finish_attempt(
+                    attempt, L.FAILED_CLEAR, exc.status, str(exc)
+                )
+                self.ledger.release_backend(gw_id, f"{backend.name}: {exc}")
+                self._record_failure(backend.name)
+                return None
+            # Ambiguous: never retry elsewhere until we know nothing was created here.
+            self.ledger.finish_attempt(attempt, L.AMBIGUOUS, exc.status, str(exc))
+            self._record_failure(backend.name)
+            try:
+                found = await backend.find_by_gw_id(gw_id)
+            except ReconcileError as rexc:
+                self.ledger.set_state(
+                    gw_id, L.UNKNOWN, f"{backend.name}: {exc}; reconcile failed: {rexc}"
+                )
+                raise PlacementError(
+                    503,
+                    "STACKBOX::OUTCOME_UNKNOWN",
+                    "create outcome is unknown; the gateway will reconcile and clean up",
+                ) from rexc
+            if found is not None and not backend.adopt_ambiguous:
+                # It exists but its setup may be incomplete: delete it, then retry.
+                try:
+                    resp = await backend.delete(found["id"])
+                    deleted = resp.status in (200, 202, 204, 404)
+                except BackendError:
+                    deleted = False
+                if not deleted:
+                    self.ledger.set_state(
+                        gw_id, L.UNKNOWN, f"{backend.name}: {exc}; cleanup failed"
+                    )
+                    raise PlacementError(
+                        503,
+                        "STACKBOX::OUTCOME_UNKNOWN",
+                        "create outcome is unknown; the gateway will reconcile and clean up",
+                    )
+                self.ledger.resolve_ambiguous_attempt(gw_id, backend.name, L.DISCARDED)
+                self.ledger.release_backend(
+                    gw_id, f"{backend.name}: {exc} (found and deleted)"
+                )
+                return None
+            if found is None:
+                self.ledger.resolve_ambiguous_attempt(gw_id, backend.name, L.ABSENT)
+                self.ledger.release_backend(
+                    gw_id, f"{backend.name}: {exc} (confirmed absent)"
+                )
+                return None
+            self.ledger.resolve_ambiguous_attempt(gw_id, backend.name, L.ADOPTED)
+            created = Created(sandbox=found)
+        else:
+            self.ledger.finish_attempt(attempt, L.OK, 200)
+            self.startup.observe(
+                backend.name, req.image, time.monotonic() - t0, cold=created.cold
+            )
+            if created.cold:
+                self.stats[backend.name].cold_starts += 1
+        latency_ms = (time.monotonic() - t0) * 1000
+        self.ledger.mark_active(
+            gw_id,
+            created.sandbox["id"],
+            cpu=created.cpu,
+            memory_gib=created.memory_gib,
+            backend_data=created.backend_data,
+        )
+        self._record_success(backend.name, latency_ms)
+        return self.to_client(created.sandbox, gw_id)
+
+    # ---------------------------------------------------------------- helpers
+    def to_client(self, sandbox: dict[str, Any], gw_id: str) -> dict[str, Any]:
+        out = dict(sandbox)
+        out["id"] = gw_id
+        rec = self.ledger.get(gw_id)
+        meta = {**(rec.client_metadata if rec else {}), **(out.get("metadata") or {})}
+        for k in ROUTER_KEYS:
+            meta.pop(k, None)
+        out["metadata"] = meta
+        return out
+
+    def resolve(self, gw_id: str) -> tuple[L.SandboxRecord, Backend]:
+        rec = self.ledger.get(gw_id)
+        if (
+            rec is None
+            or rec.state not in (L.ACTIVE,)
+            or rec.backend is None
+            or rec.upstream_id is None
+        ):
+            raise PlacementError(
+                404, "SANDBOX::NOT_FOUND", f"sandbox {gw_id} not found"
+            )
+        return rec, self.backends[rec.backend]
+
+    async def delete(self, gw_id: str) -> int:
+        rec, backend = self.resolve(gw_id)
+        resp = await backend.delete(rec.upstream_id)  # type: ignore[arg-type]
+        if resp.status in (200, 202, 204, 404):
+            self.ledger.set_state(gw_id, L.DELETED)
+        return resp.status
+
+    # ------------------------------------------------------------- reconcile
+    async def reconcile_once(self) -> dict[str, int]:
+        """Resolve unknown creates, expire vanished sandboxes, delete leaked ones."""
+        counts = {
+            "adopted_and_deleted": 0,
+            "confirmed_absent": 0,
+            "expired": 0,
+            "leaks_deleted": 0,
+        }
+        for rec in self.ledger.in_states(L.UNKNOWN):
+            if rec.backend is None:
+                continue
+            backend = self.backends[rec.backend]
+            try:
+                found = await backend.find_by_gw_id(rec.gw_id)
+            except ReconcileError:
+                continue
+            if found is None:
+                self.ledger.resolve_ambiguous_attempt(rec.gw_id, backend.name, L.ABSENT)
+                self.ledger.set_state(
+                    rec.gw_id, L.FAILED, "ambiguous create confirmed absent"
+                )
+                counts["confirmed_absent"] += 1
+            else:
+                # The client already received an error, so nobody owns this sandbox.
+                self.ledger.resolve_ambiguous_attempt(
+                    rec.gw_id, backend.name, L.ADOPTED
+                )
+                try:
+                    await backend.delete(found["id"])
+                except BackendError:
+                    continue
+                self.ledger.set_state(
+                    rec.gw_id, L.DELETED, "orphan from ambiguous create; deleted"
+                )
+                counts["adopted_and_deleted"] += 1
+
+        for backend in self.backends.values():
+            listed_at = time.time()
+            try:
+                managed = await backend.list_sandboxes({MANAGED_KEY: self.instance_id})
+            except BackendError:
+                continue
+            live = {
+                s["id"]: s
+                for s in managed
+                if (s.get("status") or {}).get("state") not in GONE_STATES
+            }
+            for rec in self.ledger.in_states(L.ACTIVE):
+                if rec.backend != backend.name or rec.upstream_id in live:
+                    continue
+                if rec.ready_at is not None and rec.ready_at < listed_at - 5:
+                    self.ledger.set_state(
+                        rec.gw_id, L.EXPIRED, "no longer present upstream"
+                    )
+                    counts["expired"] += 1
+            for upstream_id, s in live.items():
+                gw_id = (s.get("metadata") or {}).get(GW_ID_KEY)
+                rec = self.ledger.get(gw_id) if gw_id else None
+                if (
+                    rec is not None
+                    and rec.state in L.HOLDING_STATES
+                    and rec.backend == backend.name
+                    and (rec.state != L.ACTIVE or rec.upstream_id == upstream_id)
+                ):
+                    continue  # the sandbox this record owns (or an in-flight create)
+                if (
+                    rec is not None
+                    and rec.state == L.DELETED
+                    and rec.upstream_id == upstream_id
+                    and rec.ended_at is not None
+                    and listed_at - rec.ended_at < DELETE_GRACE_S
+                ):
+                    continue  # deleted moments ago; the provider is still tearing it down
+                created_at = s.get("createdAt")
+                logger.warning(
+                    "deleting leaked sandbox %s on %s (gw_id=%s, created %s)",
+                    upstream_id,
+                    backend.name,
+                    gw_id,
+                    created_at,
+                )
+                try:
+                    await backend.delete(upstream_id)
+                except BackendError:
+                    continue
+                counts["leaks_deleted"] += 1
+        return counts
+
+    async def reconcile_forever(self) -> None:
+        while True:
+            try:
+                counts = await self.reconcile_once()
+                if any(counts.values()):
+                    logger.info("reconcile: %s", counts)
+            except Exception:  # keep the loop alive
+                logger.exception("reconcile failed")
+            await asyncio.sleep(self.cfg.gateway.reconcile_interval_s)
+
+    # ----------------------------------------------------- introspection API
+    def startup_estimates(self, backend: str | None = None) -> list[dict[str, Any]]:
+        keys = set(self.startup.warm) | set(self.startup.cold)
+        rows = []
+        for b, image in sorted(keys, key=str):
+            if backend is not None and b != backend:
+                continue
+            warm, cold = (
+                self.startup.warm.get((b, image)),
+                self.startup.cold.get((b, image)),
+            )
+            rows.append(
+                {
+                    "image": image,
+                    "warm_s": None if warm is None else round(warm, 2),
+                    "cold_s": None if cold is None else round(cold, 2),
+                    "samples": self.startup.samples.get((b, image), 0),
+                }
+            )
+        return rows
+
+    def _cost(
+        self, backend: str | None, vcpu: float, gib: float, seconds: float
+    ) -> float:
+        b = self.backends.get(backend or "")
+        if b is None:
+            return 0.0
+        per_hour = (
+            vcpu * b.cfg.cost_per_vcpu_hour * b.cfg.cpu_billed_fraction
+            + gib * b.cfg.cost_per_gib_hour
+        )
+        return per_hour * seconds / 3600
+
+    def provider_view(self, name: str) -> dict[str, Any]:
+        """GET /v1/providers/{name}: configuration, live capacity and health, learned
+        startup, prepared images, and usage so far."""
+        from stackbox.catalog import PROVIDERS
+
+        b = self.backends[name]
+        cat = PROVIDERS.get(b.kind)
+        lat = sorted(self.stats[name].create_latencies_ms)
+        held, limit = self.ledger.held_count(name), self.limit(b)
+        if self.breakers[name].is_open():
+            status = "breaker_open"
+        elif self._is_saturated(name):
+            status = "at_quota"
+        elif held >= limit:
+            status = "full"
+        else:
+            status = "ok"
+        usage = self.usage(group_by="provider")["providers"].get(name, {})
+        return {
+            "name": name,
+            "kind": b.kind,
+            "label": cat.label if cat else b.kind,
+            "status": status,
+            "priority": b.cfg.priority,
+            "pricing": {
+                "cost_per_vcpu_hour": b.cfg.cost_per_vcpu_hour,
+                "cost_per_gib_hour": b.cfg.cost_per_gib_hour,
+                "cpu_billed_fraction": b.cfg.cpu_billed_fraction,
+                "usd_per_hour_1vcpu_2gib": round(b.price_per_hour(None, 1, 2), 6),
+            },
+            "capacity": {
+                "held": held,
+                "limit": limit,
+                "max_active": b.cfg.max_active,
+                "limit_learned_from_refusals": limit < b.cfg.max_active,
+            },
+            "quota_refusals": self.stats[name].saturations,
+            "placements": self.stats[name].placements,
+            "cold_starts": self.stats[name].cold_starts,
+            "create_s": {
+                "p50": round(lat[len(lat) // 2] / 1000, 3) if lat else None,
+                "p95": round(lat[min(len(lat) - 1, int(len(lat) * 0.95))] / 1000, 3)
+                if lat
+                else None,
+            },
+            "startup": self.startup_estimates(name),
+            "images": [
+                {k: i[k] for k in ("image", "status", "ref", "error")}
+                for i in self.ledger.images()
+                if i["backend"] == name
+            ],
+            "usage": usage,
+        }
+
+    def placement(self, gw_id: str) -> dict[str, Any] | None:
+        """GET /v1/placements/{id}: where a sandbox went, every attempt, and its cost."""
+        rec = self.ledger.get(gw_id)
+        if rec is None:
+            return None
+        atts = [a for a in self.ledger.attempts() if a["gw_id"] == gw_id]
+        ok = next((a for a in atts if a["outcome"] == L.OK), None)
+        row = self._usage_row(rec, ok["started_at"] if ok else None)
+        return {
+            **row,
+            "queued_s": round(atts[0]["started_at"] - rec.requested_at, 3)
+            if atts
+            else None,
+            "create_s": round(ok["finished_at"] - ok["started_at"], 3) if ok else None,
+            "last_error": rec.last_error,
+            "attempts": [
+                {
+                    "provider": a["backend"],
+                    "outcome": a["outcome"],
+                    "http_status": a["http_status"],
+                    "error": a["error"],
+                    "started_at": _iso(a["started_at"]),
+                    "duration_s": None
+                    if a["finished_at"] is None
+                    else round(a["finished_at"] - a["started_at"], 3),
+                }
+                for a in atts
+            ],
+        }
+
+    def _usage_row(
+        self,
+        rec: L.SandboxRecord,
+        started: float | None,
+        since: float | None = None,
+        until: float | None = None,
+    ) -> dict[str, Any]:
+        now = time.time()
+        start = started or rec.ready_at
+        end = rec.ended_at or (now if rec.state in L.HOLDING_STATES else rec.ended_at)
+        seconds = 0.0
+        if start is not None and end is not None:
+            lo, hi = max(start, since or start), min(end, until or end)
+            seconds = max(0.0, hi - lo)
+        vcpu, gib = rec.cpu or 0.0, rec.memory_gib or 0.0
+        return {
+            "id": rec.gw_id,
+            "provider": rec.backend,
+            "image": rec.image,
+            "state": rec.state,
+            "requested_at": _iso(rec.requested_at),
+            "ready_at": _iso(rec.ready_at),
+            "ended_at": _iso(rec.ended_at),
+            "vcpu": vcpu,
+            "memory_gib": gib,
+            "seconds": round(seconds, 1),
+            "est_cost_usd": round(self._cost(rec.backend, vcpu, gib, seconds), 6),
+        }
+
+    def usage(
+        self,
+        since: float | None = None,
+        until: float | None = None,
+        group_by: str = "provider",
+    ) -> dict[str, Any]:
+        """GET /v1/usage: sandbox time and estimated cost (list prices), billed from the
+        start of the successful create to the delete, clipped to [since, until]."""
+        started = {
+            a["gw_id"]: a["started_at"]
+            for a in self.ledger.attempts()
+            if a["outcome"] == L.OK
+        }
+        rows = []
+        for rec in self.ledger.all():
+            if rec.ready_at is None and rec.state not in L.HOLDING_STATES:
+                continue  # never ran
+            row = self._usage_row(rec, started.get(rec.gw_id), since, until)
+            if (
+                since is not None
+                and row["seconds"] == 0
+                and (rec.ended_at or time.time()) < since
+            ):
+                continue
+            rows.append(row)
+        totals = {
+            "sandboxes": len(rows),
+            "sandbox_hours": round(sum(r["seconds"] for r in rows) / 3600, 4),
+            "vcpu_hours": round(sum(r["seconds"] * r["vcpu"] for r in rows) / 3600, 4),
+            "gib_hours": round(
+                sum(r["seconds"] * r["memory_gib"] for r in rows) / 3600, 4
+            ),
+            "est_cost_usd": round(sum(r["est_cost_usd"] for r in rows), 6),
+        }
+        out: dict[str, Any] = {
+            "since": _iso(since),
+            "until": _iso(until),
+            "totals": totals,
+            "note": "estimates from configured list prices, not invoices",
+        }
+        if group_by == "sandbox":
+            out["sandboxes"] = rows
+        else:
+            per: dict[str, dict[str, Any]] = {}
+            for r in rows:
+                if not r["provider"]:
+                    continue
+                agg = per.setdefault(
+                    r["provider"],
+                    {
+                        "sandboxes": 0,
+                        "sandbox_hours": 0.0,
+                        "vcpu_hours": 0.0,
+                        "gib_hours": 0.0,
+                        "est_cost_usd": 0.0,
+                    },
+                )
+                agg["sandboxes"] += 1
+                agg["sandbox_hours"] += r["seconds"] / 3600
+                agg["vcpu_hours"] += r["seconds"] * r["vcpu"] / 3600
+                agg["gib_hours"] += r["seconds"] * r["memory_gib"] / 3600
+                agg["est_cost_usd"] += r["est_cost_usd"]
+            out["providers"] = {
+                k: {
+                    kk: round(vv, 6) if isinstance(vv, float) else vv
+                    for kk, vv in v.items()
+                }
+                for k, v in per.items()
+            }
+        return out
+
+
+def _iso(ts: float | None) -> str | None:
+    if ts is None:
+        return None
+    import datetime as dt
+
+    return (
+        dt.datetime.fromtimestamp(ts, dt.UTC)
+        .isoformat(timespec="seconds")
+        .replace("+00:00", "Z")
+    )