pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/fleet/heartbeat.py

@@ -0,0 +1,194 @@
+"""Beside a worker's calls: the load livekit reads, and the heartbeat the gateway hears."""
+
+import asyncio
+import logging
+import socket
+import time
+from collections.abc import Callable
+from dataclasses import fields
+
+from livekit.agents import AgentServer
+from livekit.agents.worker import ServerOptions
+
+from pinecall.domain.errors import DeclarationRefused, GatewayRefused
+from pinecall.fleet.client import GatewayClient
+from pinecall.fleet.measures import LastMinute
+from pinecall.fleet.roster import HEARTBEAT_S, REFUSED_AT, agent_name
+from pinecall.process.settings import Settings
+from pinecall.wire.rest.fleet import HeartbeatRequest
+
+logger = logging.getLogger(__name__)
+
+
+NO_MEASURE = "livekit's load_fnc option no longer carries a measure of its own"
+
+
+# A worker that starts again under the same name (a pod's container, after its node was reset)
+# registers anew while LiveKit may hold the last process's dead socket for 15-20 minutes and offer
+# it calls. This process's start, to the second, keeps the two apart; the roster keys by the
+# worker's name, so the gateway offers only the newest registration.
+STARTED = f"{time.time():.0f}"
+
+
+A_SLOT_AT_LEAST = "a worker holds at least one call: PINECALL_MAX_JOBS={max_jobs}"
+
+
+# The id livekit's AgentServer holds until LiveKit answers its registration with the real one.
+UNREGISTERED = "unregistered"
+
+
+# The exit a cordoned worker leaves with; its unit's RestartPreventExitStatus= keeps it down.
+CORDONED_EXIT = 3
+
+
+# What LiveKit reads, on LiveKit's scale: its server offers a job under REFUSED_AT and its
+# framework declines at it, so a worker that counts its calls is at REFUSED_AT with every slot
+# taken. The server reads the load the worker last reported (every 2.5 s): a burst faster than
+# that, on a worker's last slots, is declined by the framework and offered to no one else.
+class Load:
+    """The load LiveKit reads: its line times the slots taken, or the machine's CPU share."""
+
+    def __init__(
+        self, max_jobs: int | None, measure: Callable[[AgentServer], float] | None = None
+    ) -> None:
+        """A worker nobody has dispatched to yet."""
+        if max_jobs is not None and max_jobs < 1:
+            raise DeclarationRefused(A_SLOT_AT_LEAST.format(max_jobs=max_jobs))
+        self.max_jobs = max_jobs
+        self.measure = measure or livekits_measure()
+        self.refused = False
+
+    def __call__(self, server: AgentServer) -> float:
+        """The load now, as livekit reads it."""
+        if self.max_jobs is None:
+            return self.announced(self.measure(server))
+        return self.at(len(server.active_jobs))
+
+    def at(self, active: int) -> float:
+        """The load of a worker holding this many calls, when its slots were measured."""
+        return self.announced(REFUSED_AT * active / (self.max_jobs or 1))
+
+    def announced(self, load: float) -> float:
+        """The load, a crossing of LiveKit's line said once each way."""
+        refused = load >= REFUSED_AT
+        if refused != self.refused:
+            self.refused = refused
+            if refused:
+                logger.warning("load %.2f: livekit routes no job here until it falls", load)
+            else:
+                logger.info("load %.2f: livekit routes jobs here again", load)
+        return load
+
+
+class Heartbeats:
+    """The worker's report to its gateway every five seconds, and a cordon's way out."""
+
+    def __init__(
+        self, server: AgentServer, gateway: GatewayClient, settings: Settings, minute: LastMinute
+    ) -> None:
+        """Not beating yet; `leave` is set when the worker must go."""
+        self.server = server
+        self.gateway = gateway
+        self.fleet = settings.fleet
+        self.max_jobs = settings.max_jobs
+        self.name = worker_name_of(settings)
+        self.agent_name = agent_name_of(settings)
+        self.minute = minute
+        self.cordoned = False
+        self.leave = asyncio.Event()
+        # Set once LiveKit registered the worker and the gateway answered a heartbeat uncordoned.
+        self.ready = asyncio.Event()
+
+    def beat(self) -> HeartbeatRequest:
+        """What this worker holds now, and what its calls did in the last minute."""
+        active = len(self.server.active_jobs)
+        last = self.minute.of(asyncio.get_running_loop().time())
+        # The name only once LiveKit registered it: before, a call offered to it waits for nobody.
+        registered = self.server.id != UNREGISTERED
+        return HeartbeatRequest(
+            fleet=self.fleet,
+            worker=self.name,
+            agent_name=self.agent_name if registered else None,
+            active=active,
+            max_jobs=self.max_jobs,
+            load=self.gateways_load(active),
+            draining=self.server.draining,
+            ended=last.ended,
+            failed=last.failed,
+            errors=last.errors,
+            turns=last.turns,
+            first_audio_p95_s=last.first_audio_p95_s,
+        )
+
+    # The gateway's scale: calls over slots, full at 1.0 (roster._refused_at); a worker on CPU
+    # reports the share LiveKit reads, which is the same number on both scales.
+    def gateways_load(self, active: int) -> float:
+        """The load the gateway reads for this worker."""
+        if self.max_jobs is not None:
+            return active / self.max_jobs
+        load_of = self.server.load_fnc
+        return load_of(self.server) if isinstance(load_of, Load) else 0.0
+
+    # A gateway away is said and waited out: the worker keeps its calls meanwhile.
+    async def run(self) -> None:
+        """Beat until cordoned; then ask the worker to leave."""
+        while True:
+            try:
+                existing = await self.gateway.heartbeat(self.beat())
+            except GatewayRefused as refused:
+                logger.warning("heartbeat: %s", refused)
+            else:
+                if existing.cordoned:
+                    logger.warning("cordoned by the gateway: draining, then leaving")
+                    self.cordoned = True
+                    self.leave.set()
+                    return
+                if self.server.id != UNREGISTERED:
+                    self.ready.set()
+            await asyncio.sleep(HEARTBEAT_S)
+
+
+# systemd's Type=notify: `systemctl restart` returns only once this is said, so a release starts
+# the next instance only after this one can take calls. Outside systemd there is nobody to tell.
+async def announced_ready(beats: Heartbeats, notify: str | None) -> bool:
+    """Tell systemd the worker is ready once LiveKit registered it and the gateway heard it."""
+    await beats.ready.wait()
+    if not notify:
+        return False
+    # An address that starts with @ is in the abstract namespace, whose first byte is a zero.
+    address = "\0" + notify[1:] if notify.startswith("@") else notify
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as systemd:
+        systemd.connect(address)
+        systemd.sendall(b"READY=1")
+    logger.info("registered with LiveKit and heard by the gateway: ready")
+    return True
+
+
+# The job's process names its call's worker as the heartbeat does: the same settings, the same host.
+def worker_name_of(settings: Settings) -> str:
+    """What the worker is called in its heartbeats and its calls: its setting, or the short host."""
+    return settings.worker_name or socket.gethostname().split(".")[0]
+
+
+# Its own name, not the fleet's: LiveKit offers a job only to the workers registered under the
+# name a dispatch carries, so the gateway reaches this process and no other (docs/scaling.md).
+def agent_name_of(settings: Settings) -> str:
+    """The name the worker registers under with LiveKit: its fleet's, its own, its start."""
+    return f"{agent_name(settings.fleet, worker_name_of(settings))}.{STARTED}"
+
+
+# livekit's own CPU average, read off its options' default rather than its private class.
+def livekits_measure() -> Callable[[AgentServer], float]:
+    """The load function livekit runs when given none: the machine's CPU over a few seconds."""
+    (declared,) = (option for option in fields(ServerOptions) if option.name == "load_fnc")
+    measure = declared.default
+    if not callable(measure):
+        raise DeclarationRefused(NO_MEASURE)
+
+    def measured(server: AgentServer) -> float:
+        load = measure(server)
+        if not isinstance(load, int | float):
+            raise DeclarationRefused(NO_MEASURE)
+        return float(load)
+
+    return measured

pinecall/fleet/measures.py

@@ -0,0 +1,157 @@
+"""A worker's last minute: what its calls' job processes tell it, and what its heartbeat carries."""
+
+import asyncio
+import math
+import socket
+import tempfile
+from collections import deque
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, override
+
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from pinecall.process.settings import Settings
+from pinecall.wire.rest.calls import BatchedEntry
+
+# What the heartbeat carries is the last minute's, as the roster judges it.
+WINDOW_S = 60.0
+
+
+# A busy minute is thousands of turns: past this many, the oldest are forgotten early.
+MOST_KEPT = 10_000
+
+
+# A p95 is read off this many turns at least; fewer, and the heartbeat carries none.
+TURNS_FOR_A_P95 = 5
+
+
+# How a call that failed ended; one datagram per thing told, and a datagram lost is uncounted.
+AN_ERROR = "error"
+
+
+class JobReport(BaseModel):
+    """One thing a job tells its worker: a turn's first audio, an error, or how its call ended."""
+
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    first_audio_s: float | None = None
+    error: bool = False
+    ended: str | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class Minute:
+    """A worker's calls over the last minute: ended, failed, error entries, and first audio."""
+
+    ended: int
+    # Of the calls that ended, those that ended in an error.
+    failed: int
+    errors: int
+    turns: int
+    first_audio_p95_s: float | None
+
+
+class LastMinute:
+    """What the worker's job processes told it, kept for a minute, in the worker's main process."""
+
+    def __init__(self) -> None:
+        """Nothing heard yet."""
+        self.first_audio: deque[tuple[float, float]] = deque(maxlen=MOST_KEPT)
+        self.errors: deque[float] = deque(maxlen=MOST_KEPT)
+        self.ended: deque[tuple[float, bool]] = deque(maxlen=MOST_KEPT)
+
+    def heard(self, message: bytes, now: float) -> None:
+        """One datagram from a job; one that does not read is dropped."""
+        try:
+            report = JobReport.model_validate_json(message)
+        except ValidationError:
+            return
+        if report.first_audio_s is not None:
+            self.first_audio.append((now, report.first_audio_s))
+        if report.error:
+            self.errors.append(now)
+        if report.ended is not None:
+            self.ended.append((now, report.ended == AN_ERROR))
+
+    def of(self, now: float) -> Minute:
+        """The minute up to now."""
+        since = now - WINDOW_S
+        while self.first_audio and self.first_audio[0][0] < since:
+            self.first_audio.popleft()
+        while self.errors and self.errors[0] < since:
+            self.errors.popleft()
+        while self.ended and self.ended[0][0] < since:
+            self.ended.popleft()
+        waits = sorted(value for _, value in self.first_audio)
+        return Minute(
+            ended=len(self.ended),
+            failed=sum(1 for _, failed in self.ended if failed),
+            errors=len(self.errors),
+            turns=len(waits),
+            first_audio_p95_s=_p95(waits) if len(waits) >= TURNS_FOR_A_P95 else None,
+        )
+
+
+class _Heard(asyncio.DatagramProtocol):
+    def __init__(self, minute: LastMinute) -> None:
+        self.minute = minute
+
+    @override
+    def datagram_received(self, data: bytes, addr: tuple[str | Any, int]) -> None:
+        del addr
+        self.minute.heard(data, asyncio.get_running_loop().time())
+
+
+# Private to the unit on the box (PrivateTmp=yes); the health port tells two workers of a machine
+# apart, as it must already, since each binds its own.
+def measures_path(settings: Settings) -> Path:
+    """Where a worker hears its jobs: a datagram socket its main process binds."""
+    name = f"pinecall-{settings.fleet}-{settings.worker_http_port}.measures"
+    return Path(tempfile.gettempdir()) / name
+
+
+# Bound before livekit starts its processes; a leftover of a worker that died is taken over.
+async def listening(path: Path, minute: LastMinute) -> asyncio.DatagramTransport:
+    """Bind the socket the worker's jobs tell it through."""
+    await asyncio.to_thread(path.unlink, missing_ok=True)
+    transport, _ = await asyncio.get_running_loop().create_datagram_endpoint(
+        lambda: _Heard(minute), local_addr=str(path), family=socket.AF_UNIX
+    )
+    return transport
+
+
+# Called on every batch a job sends: most carry none of the three, and send nothing.
+def reported(path: Path, entries: Sequence[BatchedEntry]) -> None:
+    """Tell the worker what a batch of its call's entries says: first audio, errors, an end."""
+    messages = [message for entry in entries if (message := _message_of(entry)) is not None]
+    if not messages:
+        return
+    with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sender:
+        sender.settimeout(0)
+        for message in messages:
+            try:
+                sender.sendto(message, str(path))
+            except OSError:
+                # A worker not listening (a test, or its main process gone) loses the measure only.
+                return
+
+
+# Nearest rank, of values already sorted.
+def _p95(values: Sequence[float]) -> float:
+    return values[max(math.ceil(0.95 * len(values)) - 1, 0)]
+
+
+def _message_of(entry: BatchedEntry) -> bytes | None:
+    if entry.type == "turn.agent":
+        measured = entry.data.get("metrics")
+        waited = measured.get("e2e_latency") if isinstance(measured, dict) else None
+        if isinstance(waited, int | float):
+            return JobReport(first_audio_s=waited).model_dump_json().encode()
+    if entry.type == AN_ERROR:
+        return JobReport(error=True).model_dump_json().encode()
+    if entry.type == "call.ended":
+        reason = entry.data.get("reason")
+        return JobReport(ended=reason if isinstance(reason, str) else "").model_dump_json().encode()
+    return None

pinecall/fleet/roster.py

@@ -0,0 +1,232 @@
+"""The roster of workers, one per fleet: who is up, what each holds, and whether a fleet is full."""
+
+import time
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from typing import Literal
+
+from pydantic import TypeAdapter
+
+from pinecall.process.shared import Assigned, Shared, newest
+from pinecall.process.signal import LocalSignal, Signal
+from pinecall.wire.rest.fleet import FleetTotals, HeartbeatRequest, HeartbeatResponse, WorkerStatus
+
+type WorkerState = Literal["gone", "cordoned", "draining", "accepting", "failing", "full"]
+
+# LiveKit's line, the same on both of its sides: livekit-server's agent.DefaultTargetLoad
+# (pkg/agent/config.go; a job is offered to a worker whose reported load is under it) and
+# livekit-agents' load_threshold (the worker declines at or over it). A worker gated on CPU
+# reports its share of the machine, which is that scale. A worker that counts its calls reports
+# 0.7 times its calls over its slots to LiveKit (fleet/heartbeat.Load), so LiveKit's full is its
+# last slot, and calls over slots to the gateway, full at 1.0: on 2026-10-02 it reported calls over
+# slots to both, and LiveKit stopped at 0.7 of them while the gateway still counted seats free.
+REFUSED_AT = 0.7
+EVERY_SLOT = 1.0
+OVERFLOW = "overflow"
+HEARTBEAT_S = 5.0
+# A worker silent this long is no longer capacity.
+STALE_AFTER_S = 30.0
+# A silent worker stays listed, as gone, this long.
+FORGOTTEN_AFTER_S = 3600.0
+# A worker up but bad: at least half the calls that ended in its last minute ended in an error,
+# judged on four calls at least, or its callers waited over five seconds for first audio at the
+# p95, over twenty turns at least. Set so they change nothing a healthy worker does.
+FAILED_SHARE = 0.5
+ENDED_TO_JUDGE = 4
+SLOW_FIRST_AUDIO_S = 5.0
+TURNS_TO_JUDGE = 20
+
+# Every gateway says the heartbeats it heard and the cordons it set, once a heartbeat; each answers
+# a heartbeat's `full` from totals at most this old, not by walking the fleet every time.
+ROSTER_CHANNEL = "roster"
+TOTALS_KEPT_S = 1.0
+CORDONED = "cordoned"
+
+
+@dataclass(frozen=True)
+class Heard:
+    """One gateway's share: the workers that beat on it lately, and the cordons it set."""
+
+    seats: tuple[WorkerStatus, ...] = ()
+    cordons: tuple[Assigned, ...] = ()
+
+
+# In memory, on every gateway: each keeps the heartbeats that reached it and hears the others', so
+# a worker beating on any gateway is counted on all of them. After a restart the next round of
+# heartbeats, five seconds, rebuilds it; a cordon set on one gateway stands on all, the newest
+# setting of a worker's cordon winning.
+class Roster:
+    """The workers of every fleet, by fleet and name, as their heartbeats say on any gateway."""
+
+    def __init__(self, signal: Signal | None = None) -> None:
+        """Nobody heard from yet."""
+        # Every gateway's workers, the newest heartbeat of each: what every read answers from.
+        self.seats: dict[tuple[str, str], WorkerStatus] = {}
+        self.heard_here: dict[tuple[str, str], WorkerStatus] = {}
+        self.cordons: dict[tuple[str, str], Assigned] = {}
+        self.cordoned: dict[tuple[str, str], str] = {}
+        self.shared = Shared(signal or LocalSignal(), ROSTER_CHANNEL, _HEARD, Heard(), self._merged)
+        self.shared.every_s = HEARTBEAT_S
+        self.shared.gathered = self._gathered
+        self._totals: dict[str, tuple[float, FleetTotals]] = {}
+
+    async def start(self) -> None:
+        """Hear the other gateways' workers, and say this one's every heartbeat."""
+        await self.shared.start()
+
+    async def close(self) -> None:
+        """Stop hearing and saying."""
+        await self.shared.close()
+
+    def report(self, beat: HeartbeatRequest, now: float) -> HeartbeatResponse:
+        """Keep the heartbeat and answer the worker's standing; a cordon outlives heartbeats."""
+        cordoned = (beat.fleet, beat.worker) in self.cordoned
+        seat = WorkerStatus(
+            fleet=beat.fleet,
+            worker=beat.worker,
+            agent_name=beat.agent_name,
+            active=beat.active,
+            max_jobs=beat.max_jobs,
+            load=beat.load,
+            draining=beat.draining,
+            cordoned=cordoned,
+            seen_at=now,
+            ended=beat.ended,
+            failed=beat.failed,
+            errors=beat.errors,
+            turns=beat.turns,
+            first_audio_p95_s=beat.first_audio_p95_s,
+        )
+        self.heard_here[(beat.fleet, beat.worker)] = seat
+        self.seats[(beat.fleet, beat.worker)] = seat
+        return HeartbeatResponse(cordoned=cordoned, full=self._kept_totals(beat.fleet, now).full)
+
+    def cordon(self, fleet: str, worker: str, *, on: bool = True) -> bool:
+        """Cordon a worker, or take the cordon back; False for a name nobody has."""
+        if (fleet, worker) not in self.seats:
+            return False
+        holder = CORDONED if on else None
+        self.cordons[(fleet, worker)] = Assigned(fleet, worker, holder, time.time())
+        self._merged()
+        self.shared.put(self._gathered())
+        return True
+
+    def of(self, fleet: str, now: float) -> list[WorkerStatus]:
+        """A fleet's workers by last heartbeat, forgetting the ones silent for an hour."""
+        for key, seat in list(self.seats.items()):
+            if now - seat.seen_at > FORGOTTEN_AFTER_S:
+                del self.seats[key]
+        return sorted(
+            (seat for seat in self.seats.values() if seat.fleet == fleet),
+            key=lambda seat: seat.seen_at,
+        )
+
+    def _kept_totals(self, fleet: str, now: float) -> FleetTotals:
+        kept = self._totals.get(fleet)
+        if kept is not None and 0 <= now - kept[0] < TOTALS_KEPT_S:
+            return kept[1]
+        totals = self.totals(fleet, now)
+        self._totals[fleet] = (now, totals)
+        return totals
+
+    def _gathered(self) -> Heard:
+        return Heard(seats=tuple(self.heard_here.values()), cordons=tuple(self.cordons.values()))
+
+    # Each worker's newest heartbeat, wherever it beat; each cordon's newest setting.
+    def _merged(self) -> None:
+        theirs = [heard.share for heard in self.shared.theirs.values()]
+        settings = (setting for share in theirs for setting in share.cordons)
+        self.cordoned = newest(self.cordons, settings)
+        merged = dict(self.heard_here)
+        for share in theirs:
+            for seat in share.seats:
+                known = merged.get((seat.fleet, seat.worker))
+                if known is None or known.seen_at < seat.seen_at:
+                    merged[(seat.fleet, seat.worker)] = seat
+        self.seats = {
+            key: seat.model_copy(update={"cordoned": key in self.cordoned})
+            for key, seat in merged.items()
+        }
+
+    # A fleet nobody heard from is not full: a gateway with no worker yet refuses nobody.
+    def totals(self, fleet: str, now: float) -> FleetTotals:
+        """A fleet's workers heard from lately, summed."""
+        up = [
+            seat for seat in self.seats.values() if seat.fleet == fleet and heard_lately(seat, now)
+        ]
+        accepting = len(accepting_of(up, now))
+        return FleetTotals(
+            fleet=fleet,
+            workers=len(up),
+            active=sum(seat.active for seat in up),
+            seats=sum(seat.max_jobs or 0 for seat in up),
+            free=sum(max(0, (seat.max_jobs or 0) - seat.active) for seat in up if seat.max_jobs),
+            accepting=accepting,
+            full=bool(up) and accepting == 0,
+        )
+
+
+_HEARD: TypeAdapter[Heard] = TypeAdapter(Heard)
+
+
+def heard_lately(seat: WorkerStatus, now: float) -> bool:
+    """Whether the worker's last heartbeat is recent enough to count."""
+    return now - seat.seen_at <= STALE_AFTER_S
+
+
+def failing(seat: WorkerStatus) -> bool:
+    """Whether the worker's last minute is past the line: its calls fail, or its callers wait."""
+    ended, failed = seat.ended or 0, seat.failed or 0
+    if ended >= ENDED_TO_JUDGE and failed / ended >= FAILED_SHARE:
+        return True
+    slow = seat.first_audio_p95_s
+    return (seat.turns or 0) >= TURNS_TO_JUDGE and slow is not None and slow > SLOW_FIRST_AUDIO_S
+
+
+# A failing worker is not counted while another of its fleet accepts and is not failing: the line
+# never empties a fleet, so a box of one worker per world keeps counting its one.
+def accepting_of(seats: Iterable[WorkerStatus], now: float) -> list[WorkerStatus]:
+    """The fleet's workers counted as accepting: up and under the line, and not failing."""
+    accepting = [seat for seat in seats if _accepting_now(seat, now)]
+    sound = [seat for seat in accepting if not failing(seat)]
+    return sound or accepting
+
+
+def worker_state(seat: WorkerStatus, fleet: Sequence[WorkerStatus], now: float) -> WorkerState:
+    """How the roster counts the worker among its fleet's: gone, cordoned, draining, and so on."""
+    if not heard_lately(seat, now):
+        return "gone"
+    if seat.cordoned:
+        return "cordoned"
+    if seat.draining:
+        return "draining"
+    if seat in accepting_of(fleet, now):
+        return "accepting"
+    return "failing" if _accepting_now(seat, now) else "full"
+
+
+# A worker registers with LiveKit under <fleet>/<worker>, the fleet's overflow under
+# <fleet>/overflow: a dispatch reaches exactly the workers registered under its name.
+def agent_name(fleet: str, worker: str) -> str:
+    """The name a worker of a fleet is registered under with LiveKit."""
+    return f"{fleet}/{worker}"
+
+
+def overflow_name(fleet: str) -> str:
+    """The name the fleet's overflow is registered under with LiveKit."""
+    return agent_name(fleet, OVERFLOW)
+
+
+def _refused_at(max_jobs: int | None) -> float:
+    """The load a heartbeat says full at: every slot of a worker that counts, 0.7 of a CPU."""
+    return REFUSED_AT if max_jobs is None else EVERY_SLOT
+
+
+def _accepting_now(seat: WorkerStatus, now: float) -> bool:
+    """Whether livekit would dispatch to it: up, not cordoned, not draining, under its line."""
+    return (
+        heard_lately(seat, now)
+        and not seat.cordoned
+        and not seat.draining
+        and seat.load < _refused_at(seat.max_jobs)
+    )

pinecall/fleet/worlds.py

@@ -0,0 +1,44 @@
+"""The fleet each world's calls are dispatched to, as the operator named them."""
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from pinecall.domain.names import PRODUCTION, SANDBOX, Env
+from pinecall.postgres.pool import Pool
+from pinecall.process import box_settings
+from pinecall.process.settings import A_FLEET_NAME
+
+FLEETS = "fleets"
+
+
+class Fleets(BaseModel):
+    """The fleet of workers each world's calls are dispatched to."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    production: str = Field(default="pinecall", pattern=A_FLEET_NAME)
+    sandbox: str = Field(default="pinecall-sandbox", pattern=A_FLEET_NAME)
+
+
+async def fleets(pool: Pool) -> Fleets:
+    """The fleet each world's calls go to."""
+    async with pool.connection() as connection:
+        value = await box_settings.read(connection, FLEETS)
+    return Fleets() if value is None else Fleets.model_validate(value)
+
+
+async def set_fleets(pool: Pool, named: Fleets) -> None:
+    """Write the fleet of each world, whole."""
+    async with pool.connection() as connection:
+        await box_settings.write(connection, FLEETS, named.model_dump(mode="json"))
+
+
+def world_of(named: Fleets, fleet: str) -> Env | None:
+    """The world whose calls go to the fleet; None when no world's do."""
+    if fleet == named.production:
+        return PRODUCTION
+    return SANDBOX if fleet == named.sandbox else None
+
+
+def fleet_of(named: Fleets, env: Env) -> str:
+    """The fleet a call of the world is dispatched to."""
+    return named.production if env == PRODUCTION else named.sandbox

pinecall/gateway/_call_setup.py

@@ -0,0 +1,53 @@
+"""What a call of an agent is set up with: its config tuned by the scope, the keys, a refusal."""
+
+from cryptography.fernet import MultiFernet
+
+from pinecall.domain.agent import AgentConfig, Versions
+from pinecall.domain.errors import QuotaExhausted
+from pinecall.domain.scope import Scope
+from pinecall.log.logs import Logs
+from pinecall.postgres.pool import Pool
+from pinecall.providers.catalog import Providers
+from pinecall.providers.credentials import Keyring
+from pinecall.providers.declared import apply_tuning
+from pinecall.tenancy import admission, scopes, vault
+from pinecall.tenancy.scopes import Picked
+from pinecall.wire.events import (
+    CreditsExhausted,
+)
+
+
+# A call's own id picks its version where the scope stands on a canary; no call is the rest.
+async def tuned(
+    pool: Pool,
+    declared: AgentConfig,
+    scope: Scope,
+    configured: Providers,
+    picked: Picked | None = None,
+) -> tuple[AgentConfig, Versions]:
+    """The declaration under the scope's settings, and the versions it was built from."""
+    existing = await scopes.current(pool, scope, declared.slug, picked)
+    config = apply_tuning(declared, existing.tuning, existing.lexicon, defaults=configured.defaults)
+    return config, existing.versions
+
+
+# Read per call, so a rotated key runs from the next call on.
+async def keys_of(pool: Pool, sealed: MultiFernet, scope: Scope) -> Keyring:
+    """What the org's calls in the world may run on."""
+    quotas = await admission.quotas_of(pool, scope.org, scope.env)
+    return Keyring(
+        own=await vault.credentials_of(pool, sealed, scope.org),
+        box=await vault.box_credentials(pool, sealed),
+        lends=quotas.lends,
+    )
+
+
+# The refusal is written on the agent's log as the numbers the quota ran out at.
+async def exhausted(logs: Logs, org: str, agent: str, refused: QuotaExhausted) -> None:
+    """credits.exhausted on the agent's log, for a refusal that names its quota."""
+    if refused.quota is None:
+        return
+    text = CreditsExhausted.model_validate(
+        {"org": org, "quota": refused.quota, "used": refused.used, "limit": refused.limit}
+    )
+    await logs.agent(agent).append("credits.exhausted", text.written())