pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/gateway/ending/stranded.py

@@ -0,0 +1,117 @@
+"""A call whose worker went away mid-call: its caller told once, a call back offered, drained."""
+
+import logging
+
+from livekit import api
+from livekit.protocol.webhook import WebhookEvent
+
+from pinecall.channels import rooms
+from pinecall.channels.offers import Offering
+from pinecall.channels.rooms import Dispatch
+from pinecall.domain.names import Env
+from pinecall.fleet import worlds
+from pinecall.gateway._served import Serving
+from pinecall.log import queries
+from pinecall.log.queries import CallScope
+from pinecall.wire.events import CallbackRequested, CallEnded
+
+logger = logging.getLogger(__name__)
+
+LEFT = "participant_left"
+
+# The reasons livekit gives when a participant's connection was lost rather than closed: a worker
+# killed, a machine gone, a job process that died. A worker that ends a call, drains or hands it on
+# leaves CLIENT_INITIATED; a room deleted says so. livekit-server pkg/rtc/types ToDisconnectReason.
+LOST = frozenset(
+    {
+        api.DisconnectReason.SIGNAL_CLOSE,
+        api.DisconnectReason.CONNECTION_TIMEOUT,
+        api.DisconnectReason.STATE_MISMATCH,
+        api.DisconnectReason.JOIN_FAILURE,
+        api.DisconnectReason.MEDIA_FAILURE,
+        api.DisconnectReason.AGENT_ERROR,
+    }
+)
+
+CALLBACK = "callback.requested"
+
+STRANDED = "call %s: its worker went away (%s); the caller is told once and the call ends drained"
+
+
+# The end is written first and only once: it is what tells a second delivery of the same event,
+# the worker's own late end and the told job's own leaving that there is nothing left to do. The
+# room is asked of the LiveKit of the world that sent the event.
+async def stranded(
+    serving: Serving, offering: Offering, world: Env, event: WebhookEvent
+) -> str | None:
+    """End the call whose agent the event says was lost, and send its fleet to tell the caller."""
+    server = offering.servers[world]
+    if not _an_agent_lost(event):
+        return None
+    call = event.room.name
+    kept = await queries.scope_of_call(serving.connections.pool, call)
+    if kept is None or kept.sealed or kept.scope is None:
+        return None
+    if not await rooms.left_alone(server, call):
+        return None
+    if not await _ended(serving, call, kept):
+        return None
+    await _call_back_offered(serving, call, kept.agent)
+    scope = kept.scope
+    fleet = worlds.fleet_of(await worlds.fleets(serving.connections.pool), scope.env)
+    dispatch = Dispatch(
+        agent=kept.agent,
+        org=scope.org,
+        env=scope.env,
+        holder=scope.holder or None,
+        worker_gone=True,
+        entries_written=kept.written,
+    )
+    await offering.offer(call, fleet, dispatch)
+    reason = api.DisconnectReason.Name(event.participant.disconnect_reason)
+    logger.warning(STRANDED, call, reason)
+    return call
+
+
+def _an_agent_lost(event: WebhookEvent) -> bool:
+    return (
+        event.event == LEFT
+        and event.participant.kind == api.ParticipantInfo.Kind.AGENT
+        and event.participant.disconnect_reason in LOST
+        and bool(event.room.name)
+    )
+
+
+# A log this process does not serve (it restarted since the call opened) is written and let go,
+# as the reaper does: the told job's own entries then find no call here, and the reaper seals it.
+async def _ended(serving: Serving, call: str, kept: CallScope) -> bool:
+    now = serving.logs.store.clock()
+    started = now if kept.started_at is None else kept.started_at
+    ended = CallEnded(
+        reason="drained",
+        ended_by="platform",
+        ended_at=now,
+        duration_s=max(now - started, 0.0),
+    )
+    served = serving.live.calls.get(call)
+    if served is not None:
+        return await served.log.append_first("call.ended", ended.written()) is not None
+    log = serving.logs.writing(call, kept.agent)
+    try:
+        return await log.append_first("call.ended", ended.written()) is not None
+    finally:
+        serving.logs.forget(call)
+
+
+# Only a phone has a number to call back: the caller's, or on an outbound call the one dialled.
+async def _call_back_offered(serving: Serving, call: str, agent: str) -> None:
+    facts = (await queries.facts_of_calls(serving.connections.pool, [call])).get(call)
+    if facts is None or facts.channel != "phone":
+        return
+    number = facts.from_number if facts.direction == "inbound" else facts.to_number
+    if not number:
+        return
+    wanted = CallbackRequested(
+        channel="phone", number=number, via="overflow", call=call, contact=None
+    )
+    await serving.logs.agent(agent).append(CALLBACK, wanted.written())

pinecall/log/_histogram.py

@@ -0,0 +1,74 @@
+"""A stage's seconds as a fixed histogram: counted in buckets, added across calls, read by rank."""
+
+import math
+from collections.abc import Iterable, Sequence
+
+# Each bucket is 10 % wider than the one below it, from 10 ms to past a minute: a rank read off a
+# bucket's geometric middle is within 5 % of the true value, and two histograms add bucket by
+# bucket, which exact percentiles cannot.
+SMALLEST_S = 0.01
+
+
+GROWTH = 1.1
+
+
+# Under 10 ms, then 92 buckets growing by GROWTH up to 63 s, then everything slower.
+BUCKETS = 94
+
+
+MEDIAN = 0.5
+
+
+P95 = 0.95
+
+
+# Bucket i (1 to BUCKETS - 2) holds [SMALLEST_S * GROWTH ** (i - 1), SMALLEST_S * GROWTH ** i).
+UPPER: tuple[float, ...] = tuple(SMALLEST_S * GROWTH**index for index in range(BUCKETS - 1))
+
+
+def empty() -> list[int]:
+    """A histogram that counted nothing."""
+    return [0] * BUCKETS
+
+
+def counted(values: Iterable[float]) -> list[int]:
+    """A histogram of these times; a negative one (livekit's -1 for none) is not counted."""
+    buckets = empty()
+    for seconds in values:
+        if seconds >= 0:
+            buckets[_bucket_of(seconds)] += 1
+    return buckets
+
+
+def added(first: Sequence[int], second: Sequence[int]) -> list[int]:
+    """Two histograms as one."""
+    return [mine + theirs for mine, theirs in zip(first, second, strict=True)]
+
+
+# Nearest rank: the k-th smallest time, k = ceil(q * n), read as its bucket's middle.
+def rank(buckets: Sequence[int], quantile: float) -> float | None:
+    """The time at this quantile, within 5 %; None for a histogram that counted nothing."""
+    total = sum(buckets)
+    if total == 0:
+        return None
+    wanted = max(1, math.ceil(quantile * total))
+    seen = 0
+    for index, count in enumerate(buckets):
+        seen += count
+        if seen >= wanted:
+            return _middle(index)
+    return _middle(BUCKETS - 1)
+
+
+def _middle(index: int) -> float:
+    if index == 0:
+        return SMALLEST_S / 2
+    if index == BUCKETS - 1:
+        return UPPER[-1]
+    return math.sqrt(UPPER[index - 1] * UPPER[index])
+
+
+def _bucket_of(seconds: float) -> int:
+    if seconds < SMALLEST_S:
+        return 0
+    return min(BUCKETS - 1, 1 + math.floor(math.log(seconds / SMALLEST_S, GROWTH)))

pinecall/log/_relay.py

@@ -0,0 +1,259 @@
+"""The relay: what any gateway wrote, delivered to this process's live readers in seq order."""
+
+import asyncio
+import json
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from uuid import uuid4
+
+from pydantic import BaseModel, ConfigDict
+
+from pinecall.domain.errors import NotAvailable
+from pinecall.log.store import Store, log_name
+from pinecall.postgres.pool import box_task
+from pinecall.process.signal import Listening, Signal
+from pinecall.wire.frames import Entry
+
+# Each log has a channel of its own, so a gateway hears only the logs something local reads;
+# an org's feed in a world has one, and the box's floor one, followed without a sequencer: a
+# feed spans calls and is live-only, each entry carrying its log and seq for a reader that
+# needs a call's order.
+LOG_PREFIX = "log:"
+FEED_PREFIX = "feed:"
+CHANNEL = "log:{name}"
+FEED_CHANNEL = "feed:{org}:{env}"
+BOX_CHANNEL = "box"
+
+# A durable entry over this many bytes travels as its address, and the hearer reads it back.
+STUB_OVER_BYTES = 64 * 1024
+
+# How long an entry that arrived ahead of its turn waits for the ones before it, before the
+# store is asked for them; and how often the store is asked for every followed log while the
+# signal is down, when durable entries still arrive and ephemerals do not.
+FILL_AFTER_S = 0.1
+POLL_S = 1.0
+
+# Both are told the channel: a log's (`log:{name}`), an org feed's, or the box's.
+type Delivered = Callable[[str, Entry], None]
+type Dropped = Callable[[str], None]
+
+
+class Stored(BaseModel):
+    """Where a big entry is: the hearer reads it back from the store."""
+
+    log: str
+    seq: int
+
+
+class Travelled(BaseModel):
+    """What a message on a channel carries: an entry whole, or the address of a big one."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    # The relay that wrote it: its own delivery was local, so it skips it when heard back.
+    sender: str
+    entry: Entry | None = None
+    stored: Stored | None = None
+
+
+@dataclass
+class Following:
+    """One channel this process follows: its readers, the last seq delivered, what came early."""
+
+    # The channel.
+    name: str
+    # A log is delivered in seq order; a feed as it comes.
+    ordered: bool = True
+    readers: int = 1
+    last: int = 0
+    held: dict[int, Entry] = field(default_factory=dict[int, Entry])
+    # Set once the subscription is confirmed, or once the store is polled instead.
+    ready: asyncio.Event = field(default_factory=asyncio.Event)
+    listening: Listening | None = None
+    task: asyncio.Task[None] | None = None
+    filling: asyncio.Task[None] | None = None
+    closed: bool = False
+
+
+# Stored before published, and every hearer resumes from the store: a message may be lost, or
+# arrive out of turn, and what a reader is given is still whole and in seq order.
+class Relay:
+    """Follows the logs something local reads, and hands each entry on once, in its turn."""
+
+    def __init__(
+        self, store: Store, signal: Signal, delivered: Delivered, dropped: Dropped
+    ) -> None:
+        """Hand each entry to `delivered`; tell `dropped` when a log's readers must resume."""
+        self.store = store
+        self.signal = signal
+        # Several relays may share one in-process signal: each skips only what it sent itself.
+        self.id = uuid4().hex
+        self._delivered = delivered
+        self._dropped = dropped
+        self._following: dict[str, Following] = {}
+
+    @property
+    def followed(self) -> int:
+        """How many logs this process follows now."""
+        return len(self._following)
+
+    async def follow(self, name: str, *, ordered: bool = True) -> None:
+        """Follow the channel for one more reader; returns once what is published next is heard."""
+        following = self._following.get(name)
+        if following is None:
+            following = self._following[name] = Following(name, ordered=ordered)
+            following.task = box_task(self._run(following))
+        else:
+            following.readers += 1
+        await following.ready.wait()
+
+    def unfollow(self, name: str) -> None:
+        """One reader left the log; the last one leaving lets the channel go."""
+        following = self._following.get(name)
+        if following is None:
+            return
+        following.readers -= 1
+        if following.readers <= 0:
+            self._let_go(following)
+
+    def local(self, channel: str, entry: Entry) -> None:
+        """An entry this process wrote: told to every gateway, and delivered here in its turn."""
+        self.signal.publish(channel, encoded(entry, self.id))
+        following = self._following.get(channel)
+        if following is None or not following.ordered or not following.ready.is_set():
+            self._delivered(channel, entry)
+        else:
+            self._sequenced(following, entry)
+
+    async def close(self) -> None:
+        """Let every log go and wait for their tasks."""
+        followings = list(self._following.values())
+        for following in followings:
+            self._let_go(following)
+        tasks = [following.task for following in followings if following.task is not None]
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def _run(self, following: Following) -> None:
+        try:
+            listening = await self.signal.subscribe(following.name)
+        except NotAvailable:
+            listening = None
+        if following.ordered:
+            following.last = await self.store.latest_seq(following.name.removeprefix(LOG_PREFIX))
+        following.listening = listening
+        following.ready.set()
+        if following.closed:
+            if listening is not None:
+                listening.close()
+            return
+        if listening is None:
+            await self._polled(following)
+            return
+        async for data in listening:
+            await self._heard(following, data)
+        self._over(following)
+
+    # The signal is down: a log's durable entries are read off the store each second (a feed
+    # spans calls and cannot be, so its readers hear this process alone), until it is back, when
+    # the readers are told to resume so the channel is followed again.
+    async def _polled(self, following: Following) -> None:
+        while not following.closed:
+            await asyncio.sleep(POLL_S)
+            if following.closed:
+                return
+            if following.ordered:
+                for entry in await self.store.since(
+                    following.name.removeprefix(LOG_PREFIX), after=following.last
+                ):
+                    if entry.seq > following.last:
+                        self._handed(following, entry)
+            if self.signal.up:
+                self._over(following)
+                return
+
+    async def _heard(self, following: Following, data: bytes) -> None:
+        travelled = Travelled.model_validate_json(data)
+        if travelled.sender == self.id:
+            return
+        if travelled.entry is not None:
+            self._taken(following, travelled.entry)
+        elif travelled.stored is not None:
+            seq = travelled.stored.seq
+            found = await self.store.since(travelled.stored.log, after=seq - 1, limit=1)
+            if found and found[0].seq == seq:
+                self._taken(following, found[0])
+
+    def _taken(self, following: Following, entry: Entry) -> None:
+        if following.ordered:
+            self._sequenced(following, entry)
+        elif not following.closed:
+            self._delivered(following.name, entry)
+
+    def _sequenced(self, following: Following, entry: Entry) -> None:
+        if following.closed or entry.seq <= following.last:
+            return
+        if entry.seq == following.last + 1:
+            self._handed(following, entry)
+            while (queued := following.held.pop(following.last + 1, None)) is not None:
+                self._handed(following, queued)
+            return
+        following.held[entry.seq] = entry
+        if following.filling is None:
+            following.filling = box_task(self._filled(following))
+
+    # What is missing before the held ones is read from the store: by the head's lock every
+    # durable entry below a seq given out is committed, so a seq not found is an ephemeral, lost.
+    async def _filled(self, following: Following) -> None:
+        await asyncio.sleep(FILL_AFTER_S)
+        try:
+            if following.closed or not following.held:
+                return
+            highest = max(following.held)
+            wanted = highest - following.last
+            name = following.name.removeprefix(LOG_PREFIX)
+            for entry in await self.store.since(name, after=following.last, limit=wanted):
+                if following.last < entry.seq <= highest:
+                    following.held.setdefault(entry.seq, entry)
+            for seq in sorted(seq for seq in following.held if seq <= highest):
+                entry = following.held.pop(seq)
+                if seq > following.last:
+                    self._handed(following, entry)
+        finally:
+            following.filling = None
+            if following.held and not following.closed:
+                following.filling = box_task(self._filled(following))
+
+    # The channel was lost: this following is over, so a reader that comes back follows a new one
+    # (subscribed again) rather than this one, which nobody listens on; its readers are dropped.
+    def _over(self, following: Following) -> None:
+        if following.closed:
+            return
+        following.closed = True
+        if self._following.get(following.name) is following:
+            del self._following[following.name]
+        self._dropped(following.name)
+
+    def _handed(self, following: Following, entry: Entry) -> None:
+        following.last = entry.seq
+        self._delivered(following.name, entry)
+
+    def _let_go(self, following: Following) -> None:
+        following.closed = True
+        self._following.pop(following.name, None)
+        if following.listening is not None:
+            following.listening.close()
+        if following.filling is not None:
+            following.filling.cancel()
+        if following.task is not None and following.task is not asyncio.current_task():
+            following.task.cancel()
+
+
+# A big entry's address names its own log, which a feed's channel does not.
+def encoded(entry: Entry, sender: str) -> bytes:
+    """The message an entry travels as: itself, or its address when it is big and durable."""
+    whole = f'{{"sender":{json.dumps(sender)},"entry":{entry.written_json()}}}'
+    if len(whole) > STUB_OVER_BYTES and not entry.ephemeral:
+        log = log_name(entry.call, entry.agent)
+        stub = {"sender": sender, "stored": {"log": log, "seq": entry.seq}}
+        return json.dumps(stub, separators=(",", ":")).encode()
+    return whole.encode()