pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/gateway/calls/known.py

@@ -0,0 +1,23 @@
+"""A call a gateway learns of from another: served here from what was kept when it opened."""
+
+import time
+
+from pinecall.gateway._served import Served, Serving
+from pinecall.gateway.calls.serving import first_seen
+from pinecall.log import openings, queries
+
+
+# None for a call nobody claimed, one sealed, or one opened by a release that kept no opening.
+async def known_here(serving: Serving, call: str) -> Served | None:
+    """The call as this gateway serves it, first seen here from its opening when need be."""
+    served = serving.live.calls.get(call)
+    if served is not None:
+        return served
+    pool = serving.connections.pool
+    kept = await queries.scope_of_call(pool, call)
+    if kept is None or kept.scope is None or kept.sealed:
+        return None
+    opening = await openings.opening_of(pool, call)
+    if opening is None:
+        return None
+    return first_seen(serving, opening.context, opening.config, kept, time.monotonic())

pinecall/gateway/calls/owners.py

@@ -0,0 +1,87 @@
+"""Who runs each written call and holds each WhatsApp thread, across the gateways of a box."""
+
+from dataclasses import dataclass
+
+from pydantic import TypeAdapter
+
+from pinecall.process.shared import Shared
+from pinecall.process.signal import Signal
+
+# Each gateway says the written calls it runs and the threads it holds here, merged by each.
+OWNED_CHANNEL = "owned"
+
+# Where a gateway hands a WhatsApp message on to the one holding its contact's thread.
+THREAD_CHANNEL = "thread:{gateway}"
+
+
+@dataclass(frozen=True)
+class Owned:
+    """One gateway's share: the written calls whose sessions run there, the threads it holds."""
+
+    texts: tuple[str, ...] = ()
+    threads: tuple[str, ...] = ()
+
+
+# A session and a thread live where they were opened: another gateway takes one up only once its
+# owner fell silent, when the owner is forgotten (process/shared.py).
+class Owners:
+    """This gateway's written calls and threads, and every other gateway's as it says them."""
+
+    def __init__(self, signal: Signal) -> None:
+        """Nothing run and nothing held yet."""
+        self.shared = Shared(signal, OWNED_CHANNEL, _OWNED, Owned(), self._merged)
+        self.texts: set[str] = set()
+        self.threads: set[str] = set()
+        self._texts_elsewhere: set[str] = set()
+        # Each thread another live gateway holds, and that gateway's name on the signal.
+        self.threads_elsewhere: dict[str, str] = {}
+
+    @property
+    def id(self) -> str:
+        """This gateway's name on the signal."""
+        return self.shared.id
+
+    async def start(self) -> None:
+        """Hear the other gateways, and say what runs here."""
+        await self.shared.start()
+
+    async def close(self) -> None:
+        """Stop hearing and saying."""
+        await self.shared.close()
+
+    def running(self, call: str, *, here: bool) -> None:
+        """A written call's session started here, or ended."""
+        if (call in self.texts) == here:
+            return
+        if here:
+            self.texts.add(call)
+        else:
+            self.texts.discard(call)
+        self._say()
+
+    def holding(self, thread: str, *, here: bool) -> None:
+        """A contact's thread opened here, or closed."""
+        if (thread in self.threads) == here:
+            return
+        if here:
+            self.threads.add(thread)
+        else:
+            self.threads.discard(thread)
+        self._say()
+
+    def text_elsewhere(self, call: str) -> bool:
+        """Whether a live gateway other than this one runs the written call."""
+        return call in self._texts_elsewhere
+
+    def _say(self) -> None:
+        self.shared.put(Owned(texts=tuple(sorted(self.texts)), threads=tuple(sorted(self.threads))))
+
+    def _merged(self) -> None:
+        heard = self.shared.theirs.items()
+        self._texts_elsewhere = {call for _, share in heard for call in share.share.texts}
+        self.threads_elsewhere = {
+            thread: gateway for gateway, share in heard for thread in share.share.threads
+        }
+
+
+_OWNED: TypeAdapter[Owned] = TypeAdapter(Owned)

pinecall/gateway/calls/pump.py

@@ -0,0 +1,33 @@
+"""A bound call's pump: what any gateway writes to it, sent down its app socket in seq order."""
+
+import logging
+from collections.abc import Awaitable, Callable, Sequence
+
+from pinecall.log.logs import Log
+from pinecall.wire.frames import Entry
+
+type Send = Callable[[Entry], Awaitable[None]]
+
+logger = logging.getLogger(__name__)
+
+# What a stream says of itself, never sent to an app.
+MARKERS = frozenset({"log.caught_up", "log.gap"})
+
+
+# The app runs the call's tools and holds its state: it is sent them as they were written, from
+# `after`; `then` goes down right after the first entry (the tools still waiting, after
+# call.attached). The stream follows the log through the relay, so an entry another gateway
+# wrote reaches the socket here, and it ends when the call does.
+async def pumped(log: Log, after: int, send: Send, then: Sequence[Entry]) -> None:
+    """Send the call's entries to its socket until the call ends or the socket stops taking them."""
+    waiting = list(then)
+    try:
+        async for entry in log.stream(after=after):
+            if entry.type in MARKERS:
+                continue
+            await send(log.opened(entry))
+            for tool_call in waiting:
+                await send(log.opened(tool_call))
+            waiting = []
+    except (OSError, RuntimeError):
+        logger.warning("an app socket stopped taking its call's entries", exc_info=True)

pinecall/gateway/calls/serving.py

@@ -0,0 +1,118 @@
+"""A call served here, as a door reaches it: its socket, set up, first seen, looked up, claimed."""
+
+import asyncio
+from dataclasses import replace
+from datetime import UTC, datetime
+from functools import partial
+from typing import Literal
+
+from pinecall.domain.agent import AgentConfig
+from pinecall.domain.call import CallContext
+from pinecall.domain.names import CHANNELS_WITH_A_NUMBER, JsonObject
+from pinecall.domain.scope import Scope
+from pinecall.gateway._served import Served, Serving
+from pinecall.gateway._sockets import Registration, SocketId, Sockets, orgs_own
+from pinecall.log import queries
+from pinecall.log.logs import Log, arrival_entry
+from pinecall.log.private import Privacy
+from pinecall.log.queries import CallScope
+from pinecall.retrieval import lookups
+from pinecall.retrieval.lookups import OnTheCall
+from pinecall.session.tools import ToolCalls
+from pinecall.tenancy import admission
+from pinecall.tenancy.codes import Codes
+from pinecall.wire.events import CallClaimed
+from pinecall.wire.rest.calls import LookupRequest
+
+
+# A call opened with a key goes to that key's scope; one that rang, to its phone owner, or the line.
+def serving_agent(
+    sockets: Sockets, scope: Scope, agent: str, app: SocketId | None, context: CallContext
+) -> Registration | None:
+    """The socket a new call of the agent reaches."""
+    if app is None and context.route.channel in CHANNELS_WITH_A_NUMBER:
+        return sockets.taking(orgs_own(scope), agent, context.caller)
+    return sockets.serving(scope, agent, app)
+
+
+def served_call(
+    serving: Serving,
+    owner: SocketId | None,
+    context: CallContext,
+    config: AgentConfig,
+    scope: Scope,
+) -> Served:
+    """Serve the call to its socket, or park it for the next one; before its first entry."""
+    log = serving.logs.writing(context.call, config.slug)
+    log.privacy = Privacy(serving.connections.vault, config)
+    served = Served(
+        agent=config.slug,
+        scope=scope,
+        app=owner,
+        log=log,
+        commands=asyncio.Queue(),
+        context=context,
+        config=config,
+        tools=ToolCalls(
+            config,
+            log,
+            partial(queries.tool_answered, serving.connections.pool, context.call),
+            partial(queries.tool_called, serving.connections.pool, context.call),
+        ),
+    )
+    serving.live.serve(served)
+    return serving.live.calls[context.call]
+
+
+# Parked and not counted: the socket holding its agent and the quota are its opener's.
+def first_seen(
+    serving: Serving, context: CallContext, config: AgentConfig, kept: CallScope, now: float
+) -> Served:
+    """Serve a call another gateway opened, from what was kept when it opened."""
+    serving.live.idle(now)
+    served = served_call(
+        serving, None, context, config, kept.scope or Scope(context.route.org, context.env)
+    )
+    served = replace(served, opened_here=False, opened_by=kept.opened_by)
+    serving.live.calls[served.call] = served
+    serving.live.seen[served.call] = now
+    return served
+
+
+async def opened(log: Log, context: CallContext, agent: str) -> None:
+    """call.ringing for an inbound call; an outbound one has its call.dialing already."""
+    if context.direction == "outbound":
+        return
+    kind, data = arrival_entry(context, context.route.number or agent)
+    await log.append(kind, data)
+
+
+# The quotas are read per lookup, so a plan changed mid-call applies from the next turn.
+async def looked_up(serving: Serving, served: Served, request: LookupRequest) -> JsonObject:
+    """Recall or search for a call served here, written on its log, answered as the model reads."""
+    pool = serving.connections.pool
+    quotas = await admission.quotas_of(pool, served.scope.org, served.scope.env)
+    on_the_call = OnTheCall(
+        scope=served.scope,
+        context=served.context,
+        config=served.config,
+        log=served.log,
+        now=now_of(serving),
+    )
+    return await lookups.lookup(pool, serving.embedder, on_the_call, request, quotas=quotas)
+
+
+async def claim_code(
+    codes: Codes, served: Served, code: str, *, via: Literal["keypad", "agent"]
+) -> bool:
+    """Tie the call to the page showing the code; False when no page waits on it."""
+    issued = await codes.claim(served.scope.env, served.agent, code, served.call)
+    if issued is None:
+        return False
+    await served.log.append("call.claimed", CallClaimed(code=code, via=via).written())
+    return True
+
+
+def now_of(serving: Serving) -> datetime:
+    """This moment by the store's clock, the one every entry of a call is stamped with."""
+    return datetime.fromtimestamp(serving.logs.store.clock(), UTC)

pinecall/gateway/calls/threads.py

@@ -0,0 +1,367 @@
+"""The WhatsApp threads this gateway keeps open, one per contact; their doors are api/threads.py."""
+
+import asyncio
+import logging
+from dataclasses import dataclass, field
+
+import httpx
+from pydantic import TypeAdapter
+
+from pinecall.channels import routes, whatsapp
+from pinecall.channels.whatsapp import IDLE_S, WINDOW_S, Inbound, Waiting
+from pinecall.domain.call import CallContext, Contact, Route, new_call_id, today_in
+from pinecall.domain.errors import NotAvailable, PinecallError, QuotaExhausted, UpstreamFailed
+from pinecall.domain.scope import Scope
+from pinecall.gateway._call_setup import exhausted
+from pinecall.gateway._served import Serving
+from pinecall.gateway._sockets import Registration, Sockets
+from pinecall.gateway._text_calls import open_text, resume_text, tokens_of
+from pinecall.gateway.calls.owners import THREAD_CHANNEL
+from pinecall.log import inbox
+from pinecall.postgres.pool import Pool, box_task
+from pinecall.session import text
+from pinecall.session.session import Session
+from pinecall.tenancy import admission
+from pinecall.wire.events import ErrorEvent
+from pinecall.wire.frames import Entry
+
+# One conversation: the number written to, and who wrote.
+type Door = tuple[str, str, str, str]
+
+
+logger = logging.getLogger(__name__)
+
+
+NOT_TEXT = "whatsapp: a %s to %s was dropped: only text is read"
+
+
+NOBODY_AT = "whatsapp: nobody answers %s: `pinecall numbers import %s --channel whatsapp` routes it"
+
+
+NOT_ANSWERED = "whatsapp: agent %s cannot answer at %s: %s"
+
+
+KEPT = "whatsapp: nobody holds agent %s, so a message to %s waits for somebody"
+
+
+EXPIRED = "whatsapp: a message to agent %s outlived Meta's window unanswered"
+
+
+AGAIN = "whatsapp: message %s delivered again, dropped: %s"
+
+
+TURN_FAILED = "whatsapp: a turn of call %s failed and was not answered"
+
+
+NOT_SENT = "whatsapp_not_sent"
+
+
+HANDED = "whatsapp: a message for %s was handed to the gateway holding its thread"
+
+NOT_HANDED = "whatsapp: a message handed on by another gateway was not read: %s"
+
+# How long the listener for messages handed on waits for the signal before listening again.
+HANDED_RETRY_S = 1.0
+
+_INBOUND: TypeAdapter[Inbound] = TypeAdapter(Inbound)
+
+# However the call ends, the conversation is over with it: a written call a supervisor ends
+# only says so, and its session is closed here.
+OVER = {"call.ended", "supervisor.ended"}
+
+
+@dataclass
+class OpenThread:
+    """One contact's open conversation: its session, the token it replies with, what is unread."""
+
+    session: Session
+    inbound: Inbound
+    http: httpx.AsyncClient
+    token: str
+    # A None is the call over: the desk ended it, or it was hung up some other way.
+    heard: asyncio.Queue[str | None] = field(default_factory=asyncio.Queue[str | None])
+    served: asyncio.Task[None] | None = None
+
+    # The model's turns, the agent's `say` and a supervisor's reach the contact on one path.
+    async def replied(self, entry: Entry) -> None:
+        """Send a durable turn.agent to the contact; Meta's refusal is an error on the call."""
+        if entry.type in OVER:
+            self.heard.put_nowait(None)
+            return
+        answer = entry.data.get("text")
+        if (
+            entry.type != "turn.agent"
+            or entry.ephemeral
+            or not isinstance(answer, str)
+            or not answer
+        ):
+            return
+        try:
+            await whatsapp.send_text(self.http, self.token, self.inbound, answer)
+        except (UpstreamFailed, httpx.HTTPError) as refused:
+            error = ErrorEvent(code=NOT_SENT, message=str(refused), recoverable=True)
+            # Queued, not awaited: the queue is the one appending the entry this tap runs on.
+            self.session.call.writing.write("error", error)
+
+
+class Threads:
+    """The open WhatsApp threads of this process, keyed by org, world, number and contact."""
+
+    def __init__(self, serving: Serving, sockets: Sockets) -> None:
+        """No conversation open and nothing waiting until `loaded` reads the logs."""
+        self.serving = serving
+        self.sockets = sockets
+        self.idle_s: float = IDLE_S
+        self.open: dict[Door, OpenThread] = {}
+        self.waiting: list[Waiting] = []
+        self.answering_now: set[asyncio.Task[None]] = set()
+        self.handed: asyncio.Task[None] | None = None
+
+    async def start(self) -> None:
+        """Take the messages other gateways hand on for the threads held here."""
+        self.handed = box_task(self._handed_here())
+
+    # Meta delivers a message again when it thinks it unanswered. The row is claimed before the
+    # message is read, so of two deliveries at once only one reads it; it is marked read once the
+    # message is queued or kept, and a reading that fails gives the claim back, so Meta's next
+    # delivery reads it again. A process that dies mid-reading leaves a claim its lease expires.
+    async def received(self, inbound: Inbound) -> bool:
+        """Read the message once, onto its contact's conversation; False while another reads it."""
+        route = await _route_of(self.serving.connections.pool, inbound)
+        if route is None:
+            return True
+        pool = self.serving.connections.pool
+        now = self.serving.logs.store.clock()
+        claim = await whatsapp.claimed(pool, route.org, inbound.message_id, now)
+        if claim != "new":
+            logger.info(AGAIN, inbound.message_id, claim)
+            return claim == "seen"
+        try:
+            await self._heard(route, inbound)
+        except Exception:
+            await whatsapp.released(pool, route.org, inbound.message_id)
+            raise
+        await whatsapp.read(pool, route.org, inbound.message_id, now)
+        return True
+
+    # Returns once the message is queued or kept: Meta sends a slow webhook again.
+    # A contact's thread is held by one gateway: a message landing on another is handed to it,
+    # and taken here only when nobody holds it, or its holder no longer listens.
+    async def _heard(self, route: Route, inbound: Inbound) -> None:
+        scope = Scope(route.org, route.env)
+        registration = self.sockets.taking(scope, route.agent, inbound.caller)
+        door = _door(registration.scope if registration else scope, inbound)
+        thread = self.open.get(door)
+        holder = self.serving.live.owners.threads_elsewhere.get("|".join(door))
+        if thread is None and holder is not None and await self._handed_to(holder, inbound):
+            logger.info(HANDED, inbound.number)
+            return
+        if thread is None and registration is None:
+            self.waiting.append(await whatsapp.kept(self.serving.logs, route, inbound))
+            logger.warning(KEPT, route.agent, inbound.number)
+            return
+        if thread is None and registration is not None:
+            thread = await self._opened(registration, route, inbound)
+        if thread is not None:
+            thread.heard.put_nowait(inbound.text)
+
+    # Off the app socket's own loop: opening a conversation may wait on the model.
+    def answering(self, registration: Registration) -> None:
+        """Answer the messages that waited for this agent, now that somebody holds it."""
+        if not any(waiter.agent == registration.slug for waiter in self.waiting):
+            return
+        task = asyncio.create_task(self._answered(registration))
+        self.answering_now.add(task)
+        task.add_done_callback(self.answering_now.discard)
+
+    # Oldest first, so one contact's messages reach their conversation in order.
+    async def _answered(self, registration: Registration) -> None:
+        now = self.serving.logs.store.clock()
+        for waiting in [waiter for waiter in self.waiting if waiter.agent == registration.slug]:
+            if waiting.env != registration.scope.env:
+                continue
+            self.waiting.remove(waiting)
+            if now - waiting.received_at > WINDOW_S:
+                logger.warning(EXPIRED, waiting.agent)
+                await whatsapp.taken(self.serving.logs, waiting, None)
+                continue
+            # Claimed and read when it was kept: it is heard, not received again.
+            route = await _route_of(self.serving.connections.pool, waiting.inbound)
+            if route is not None:
+                await self._heard(route, waiting.inbound)
+            opened = self.open.get(_door(registration.scope, waiting.inbound))
+            await whatsapp.taken(
+                self.serving.logs,
+                waiting,
+                None if opened is None else opened.session.call.context.call,
+            )
+
+    async def loaded(self) -> None:
+        """What was waiting when this process started, read back off the agents' logs."""
+        self.waiting = await whatsapp.waiting_in(self.serving.logs.store)
+
+    async def closed(self) -> None:
+        """Stop answering: the threads stay open in their logs for the next process."""
+        serving = [value.served for value in self.open.values() if value.served is not None]
+        serving += list(self.answering_now)
+        serving += [] if self.handed is None else [self.handed]
+        for task in serving:
+            task.cancel()
+        await asyncio.gather(*serving, return_exceptions=True)
+        self.open.clear()
+
+    async def _opened(
+        self, registration: Registration, route: Route, inbound: Inbound
+    ) -> OpenThread | None:
+        token = await whatsapp.meta_token_for(
+            self.serving.connections.pool,
+            self.serving.connections.vault,
+            route.org,
+            inbound.phone_number_id,
+        )
+        if token is None:
+            logger.warning(NOT_ANSWERED, route.agent, inbound.number, "no Meta token here")
+            return None
+        try:
+            taken = await self._taken_up(registration, inbound)
+            session = taken or await open_text(
+                self.serving, registration, self._context(registration, route, inbound)
+            )
+        except PinecallError as refused:
+            logger.warning(NOT_ANSWERED, route.agent, inbound.number, refused)
+            return None
+        thread = OpenThread(
+            session=session, inbound=inbound, http=self.serving.connections.http, token=token
+        )
+        # Before the session starts, so every turn the contact is owed reaches them.
+        served = self.serving.live.calls[session.call.context.call]
+        served.log.tapped(thread.replied)
+        if taken is None:
+            await session.start()
+        door = _door(registration.scope, inbound)
+        self.open[door] = thread
+        self.serving.live.owners.holding("|".join(door), here=True)
+        thread.served = asyncio.create_task(self._served(door, thread))
+        return thread
+
+    async def _handed_to(self, holder: str, inbound: Inbound) -> bool:
+        channel = THREAD_CHANNEL.format(gateway=holder)
+        try:
+            return (
+                await self.serving.live.signal.published(channel, _INBOUND.dump_json(inbound)) > 0
+            )
+        except NotAvailable:
+            return False
+
+    # Claimed and read by the gateway it landed on: here it is only heard.
+    async def _handed_here(self) -> None:
+        signal = self.serving.live.signal
+        channel = THREAD_CHANNEL.format(gateway=self.serving.live.owners.id)
+        while True:
+            try:
+                listening = await signal.subscribe(channel)
+            except NotAvailable:
+                await asyncio.sleep(HANDED_RETRY_S)
+                continue
+            try:
+                async for data in listening:
+                    await self._taken_on(_INBOUND.validate_json(data))
+            finally:
+                listening.close()
+            await asyncio.sleep(HANDED_RETRY_S)
+
+    async def _taken_on(self, inbound: Inbound) -> None:
+        route = await _route_of(self.serving.connections.pool, inbound)
+        if route is None:
+            return
+        try:
+            await self._heard(route, inbound)
+        except PinecallError as refused:
+            logger.warning(NOT_HANDED, refused)
+
+    # A conversation this process never saw is taken up from its log while it has idle time left.
+    async def _taken_up(self, registration: Registration, inbound: Inbound) -> Session | None:
+        newest = await inbox.calls_with(
+            self.serving.connections.pool,
+            registration.scope,
+            registration.slug,
+            inbound.caller,
+            limit=1,
+        )
+        if not newest:
+            return None
+        entries = await self.serving.logs.store.whole(newest[0])
+        if not entries or self.serving.logs.store.clock() - entries[-1].ts > self.idle_s:
+            return None
+        return await resume_text(
+            self.serving, registration, newest[0], self.serving.connections.settings.timezone
+        )
+
+    def _context(self, registration: Registration, route: Route, inbound: Inbound) -> CallContext:
+        answering = Route(
+            org=route.org,
+            agent=registration.slug,
+            channel="whatsapp",
+            number=route.number,
+            env=route.env,
+        )
+        return CallContext(
+            call=new_call_id(),
+            channel="whatsapp",
+            direction="inbound",
+            caller=inbound.caller,
+            route=answering,
+            today=today_in(self.serving.connections.settings.timezone),
+            contact=Contact(phone=inbound.caller, name=inbound.name),
+            holder=registration.scope.holder or None,
+        )
+
+    # The idle period starts once every message is answered: a slow turn is never cut off.
+    async def _served(self, door: Door, thread: OpenThread) -> None:
+        session = thread.session
+        scope = session.call.context.route
+        try:
+            while True:
+                try:
+                    answer = await asyncio.wait_for(thread.heard.get(), timeout=self.idle_s)
+                except TimeoutError:
+                    await text.end(session, "timeout", "platform")
+                    return
+                if answer is None:
+                    await session.close()
+                    return
+                try:
+                    await admission.admit_turn(
+                        self.serving.connections.pool,
+                        Scope(scope.org, scope.env),
+                        turns=session.call.turns,
+                        tokens=tokens_of(session.usage),
+                        at=self.serving.logs.store.clock(),
+                    )
+                except QuotaExhausted as refused:
+                    await exhausted(self.serving.logs, scope.org, session.call.config.slug, refused)
+                    await text.end(session, "timeout", "platform")
+                    return
+                try:
+                    await text.hears(session, answer)
+                except PinecallError:
+                    logger.warning(TURN_FAILED, session.call.context.call, exc_info=True)
+        finally:
+            self.open.pop(door, None)
+            self.serving.live.owners.holding("|".join(door), here=False)
+            self.serving.live.close(session.call.context.call)
+            self.serving.logs.forget(session.call.context.call)
+
+
+def _door(scope: Scope, inbound: Inbound) -> Door:
+    return (scope.org, scope.env, inbound.number, inbound.wa_id)
+
+
+async def _route_of(pool: Pool, inbound: Inbound) -> Route | None:
+    if inbound.kind != "text" or not inbound.text:
+        logger.warning(NOT_TEXT, inbound.kind or "message", inbound.number)
+        return None
+    route = await routes.at(pool, "whatsapp", inbound.number)
+    if route is None:
+        logger.warning(NOBODY_AT, inbound.number, inbound.number)
+    return route