pinecall-runtime 0.1.6__py3-none-any.whl

pinecall/runner/main.py

@@ -0,0 +1,306 @@
+"""The runner: a world's hosted apps kept running as the gateway wants them, one pod each."""
+
+import asyncio
+import hashlib
+import logging
+import signal
+import socket
+import time
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+
+import httpx
+import uvicorn
+from starlette.applications import Starlette
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Route
+
+from pinecall.domain.errors import GatewayRefused, SettingsRefused, UpstreamFailed
+from pinecall.domain.names import PRODUCTION, Env
+from pinecall.process.settings import Settings
+from pinecall.runner._kube import Cluster, Container, Engine, Launch, in_cluster
+from pinecall.runner._plan import (
+    Failed,
+    Revive,
+    Start,
+    Step,
+    Stop,
+    WentLive,
+    crashes_within,
+    label_of,
+    planned,
+)
+from pinecall.wire.rest.hosting import (
+    AppEnvironment,
+    AppLogs,
+    RunnerHeartbeatRequest,
+    RunnerHeartbeatResponse,
+    RunnerReport,
+    WantedApp,
+)
+
+logger = logging.getLogger(__name__)
+
+
+BEAT_S = 5.0
+
+
+# What a failure says of the process's own output: its last lines.
+LAST_LINES = 20
+
+
+# What a person asking for an app's logs is sent: its process's last lines.
+SENT_LINES = 300
+
+
+LONGEST_WHY = 2000
+
+
+START = ("./node_modules/.bin/pinecall", "start")
+
+
+NO_KEY = "PINECALL_RUNNER_KEY: a runner knocks its gateway with its world's runner key"
+
+
+NOT_THE_RELEASE = "release {release}'s sources are not the ones uploaded: sha256 differs"
+
+
+START_FAILED = "the cluster could not start the release:\n{output}"
+
+
+# A release is (org, name, release): what the gateway numbers and what a pod carries.
+type Release = tuple[str, str, int]
+
+
+@dataclass
+class Runner:
+    """A world's runner: what it holds between two beats, and the steps it has in hand."""
+
+    cluster: Cluster
+    gateway: httpx.AsyncClient
+    reports: list[RunnerReport] = field(default_factory=list[RunnerReport])
+    logs: list[AppLogs] = field(default_factory=list[AppLogs])
+    # When each host's process exited and was run again, so one that keeps exiting is given up.
+    crashes: dict[str, list[float]] = field(default_factory=dict[str, list[float]])
+    # One task per app with steps under way; an app with one is left alone this beat.
+    working: dict[str, asyncio.Task[None]] = field(default_factory=dict[str, asyncio.Task[None]])
+    # The releases a pod of this world may fetch, by their digest: read off the gateway, checked.
+    sources: dict[str, bytes] = field(default_factory=dict[str, bytes])
+    digests: dict[Release, str] = field(default_factory=dict[Release, str])
+
+    async def beat(self, name: str) -> RunnerHeartbeatResponse:
+        """Send what happened since the last beat; the gateway answers what should be running."""
+        reports, self.reports = self.reports, []
+        logs, self.logs = self.logs, []
+        body = RunnerHeartbeatRequest(runner=name, reports=reports, logs=logs)
+        try:
+            answer = await self.gateway.post("/v1/runner/heartbeat", json=body.written())
+        except httpx.HTTPError:
+            self.reports, self.logs = reports + self.reports, logs + self.logs
+            raise
+        if not answer.is_success:
+            self.reports, self.logs = reports + self.reports, logs + self.logs
+            raise GatewayRefused(answer.text, answered=answer.status_code)
+        return RunnerHeartbeatResponse.model_validate(answer.json())
+
+    async def reconcile(self, wanted: RunnerHeartbeatResponse, now: float) -> None:
+        """Make the pods what the gateway wants; the long steps go on past this beat."""
+        # Finished first: a step that ends after the listing would be planned again on a stale one.
+        self._finished()
+        containers = await self.cluster.pods(wanted.world)
+        for label, steps in planned(wanted.apps, containers, self.crashes, now).items():
+            if label not in self.working:
+                self.working[label] = asyncio.create_task(self._done(wanted.world, steps, now))
+        await self._logs_asked_for(wanted.apps, containers)
+        self._swept(wanted.apps, containers)
+
+    async def close(self) -> None:
+        """Leave: every step under way is cut short; the pods go on without the runner."""
+        for task in self.working.values():
+            task.cancel()
+        await asyncio.gather(*self.working.values(), return_exceptions=True)
+        self.working.clear()
+
+    def _finished(self) -> None:
+        for label, task in list(self.working.items()):
+            if task.done():
+                del self.working[label]
+                if not task.cancelled() and task.exception() is not None:
+                    logger.error("%s: its steps failed", label, exc_info=task.exception())
+
+    async def _done(self, world: Env, steps: Sequence[Step], now: float) -> None:
+        for step in steps:
+            match step:
+                case Start(app):
+                    await self._started(world, app, app.host, app.release)
+                case Revive(app, container):
+                    await self._revived(world, app, container, now)
+                case Stop(container):
+                    await self.cluster.stop(container.name)
+                    logger.info("stopped %s", container.name)
+                case WentLive(app):
+                    self._report(app, "live", "")
+                case Failed(app, why, container):
+                    output = await self.cluster.logs(container.name, LAST_LINES)
+                    self._report(app, "failed", why.format(output=_tail(output)))
+
+    async def _started(self, world: Env, app: WantedApp, host: str, release: int) -> None:
+        try:
+            digest = await self._fetched(app, release)
+            environment = await self._environment(app)
+        except UpstreamFailed as failure:
+            self._report(app, "failed", str(failure))
+            return
+        launch = Launch(
+            world=world,
+            app=label_of(app),
+            host=host,
+            release=release,
+            sha256=digest,
+            command=[*START, "--prod"] if world == PRODUCTION else list(START),
+        )
+        try:
+            await self.cluster.start(launch, environment)
+        except UpstreamFailed as failure:
+            self._report(app, "failed", START_FAILED.format(output=str(failure)))
+            return
+        logger.info("started %s release %d as %s", label_of(app), release, host)
+
+    # Its last lines are kept as the app's logs before the pod goes, so the exit is read.
+    async def _revived(self, world: Env, app: WantedApp, container: Container, now: float) -> None:
+        output = await self.cluster.logs(container.name, SENT_LINES)
+        self.logs.append(AppLogs(org=app.org, name=app.name, host=container.name, lines=output))
+        logger.warning(
+            "%s: %s exited, run again:\n%s", label_of(app), container.name, _tail(output)
+        )
+        self.crashes[container.name] = [
+            *crashes_within(self.crashes.get(container.name, []), now),
+            now,
+        ]
+        await self.cluster.stop(container.name, grace_s=0)
+        await self._started(world, app, container.name, container.release)
+
+    # The wanted release is checked against the digest the gateway kept when it was uploaded; an
+    # older one, revived, is the one the gateway still holds under its number.
+    async def _fetched(self, app: WantedApp, release: int) -> str:
+        known = self.digests.get((app.org, app.name, release))
+        if known is not None and known in self.sources:
+            return known
+        answer = await self.gateway.get(
+            f"/v1/runner/apps/{app.org}/{app.name}/releases/{release}/source"
+        )
+        if not answer.is_success:
+            raise UpstreamFailed(f"the release's sources: {answer.status_code} {answer.text}")
+        digest = hashlib.sha256(answer.content).hexdigest()
+        if release == app.release and digest != app.sha256:
+            raise UpstreamFailed(NOT_THE_RELEASE.format(release=release))
+        self.sources[digest] = answer.content
+        self.digests[(app.org, app.name, release)] = digest
+        return digest
+
+    async def _environment(self, app: WantedApp) -> Mapping[str, str]:
+        answer = await self.gateway.get(f"/v1/runner/apps/{app.org}/{app.name}/environment")
+        if not answer.is_success:
+            raise UpstreamFailed(f"the app's environment: {answer.status_code} {answer.text}")
+        return AppEnvironment.model_validate(answer.json()).environment
+
+    # The host asked for, else the one still serving while it installs.
+    async def _logs_asked_for(
+        self, apps: Sequence[WantedApp], containers: Sequence[Container]
+    ) -> None:
+        for app in apps:
+            ours = [each.name for each in containers if each.app == label_of(app)]
+            host = app.host if app.host in ours else next(iter(ours), None)
+            if not app.logs_wanted or host is None:
+                continue
+            output = await self.cluster.logs(host, SENT_LINES)
+            self.logs.append(AppLogs(org=app.org, name=app.name, host=host, lines=output))
+
+    # A release no pod runs and nobody wants is no longer anybody's to fetch.
+    def _swept(self, apps: Sequence[WantedApp], containers: Sequence[Container]) -> None:
+        in_use: set[Release] = {(app.org, app.name, app.release) for app in apps}
+        for each in containers:
+            org, _, name = each.app.partition("/")
+            in_use.add((org, name, each.release))
+        for release in [each for each in self.digests if each not in in_use]:
+            del self.digests[release]
+        kept = set(self.digests.values())
+        for digest in [each for each in self.sources if each not in kept]:
+            del self.sources[digest]
+
+    def _report(self, app: WantedApp, state: str, why: str) -> None:
+        if state == "failed":
+            logger.warning("%s release %d failed: %s", label_of(app), app.release, why)
+        self.reports.append(
+            RunnerReport.model_validate(
+                {
+                    "org": app.org,
+                    "name": app.name,
+                    "host": app.host,
+                    "state": state,
+                    "why": why[-LONGEST_WHY:],
+                }
+            )
+        )
+
+
+def sources_app(runner: Runner) -> Starlette:
+    """The one door a pod of this world knocks: its release's sources, by their digest."""
+
+    async def served(request: Request) -> Response:
+        found = runner.sources.get(str(request.path_params["sha256"]))
+        if found is None:
+            return Response(status_code=404)
+        return Response(found, media_type="application/gzip")
+
+    return Starlette(routes=[Route("/sources/{sha256}", served)])
+
+
+async def run(settings: Settings) -> int:
+    """Keep the world's hosted apps running until told to stop; the apps go on without it."""
+    if settings.runner_key is None:
+        raise SettingsRefused(NO_KEY)
+    stop = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    for each in (signal.SIGTERM, signal.SIGINT):
+        loop.add_signal_handler(each, stop.set)
+    engine = Engine(
+        image=settings.runner_image,
+        runtime_class=settings.runner_runtime_class,
+        namespace=settings.runner_namespace,
+        sources_url=settings.runner_sources_url,
+    )
+    headers = {"Authorization": f"Bearer {settings.runner_key}"}
+    async with (
+        httpx.AsyncClient(base_url=settings.gateway_url, headers=headers, timeout=30.0) as gateway,
+        in_cluster(engine).http as kube,
+    ):
+        runner = Runner(cluster=Cluster(http=kube, engine=engine), gateway=gateway)
+        server = uvicorn.Server(
+            uvicorn.Config(
+                sources_app(runner),
+                host=settings.runner_listen,
+                port=settings.runner_port,
+                log_level="warning",
+            )
+        )
+        serving = asyncio.create_task(server.serve())
+        name = socket.gethostname().split(".")[0]
+        while not stop.is_set():
+            try:
+                await runner.reconcile(await runner.beat(name), time.time())
+            except (UpstreamFailed, httpx.HTTPError):
+                logger.warning("runner: this beat did nothing", exc_info=True)
+            try:
+                await asyncio.wait_for(stop.wait(), BEAT_S)
+            except TimeoutError:
+                continue
+        await runner.close()
+        server.should_exit = True
+        await serving
+    return 0
+
+
+def _tail(output: str) -> str:
+    return "\n".join(output.strip().splitlines()[-LAST_LINES:])

pinecall/session/_agent.py

@@ -0,0 +1,141 @@
+"""The livekit Agent the session runs: the prompt, the ears' filter, and the hooks around a turn."""
+
+import asyncio
+import contextlib
+from collections.abc import AsyncIterable, AsyncIterator, Callable
+from contextlib import AbstractAsyncContextManager
+from typing import override
+
+from livekit import rtc
+from livekit.agents import (
+    llm,
+    stt,
+)
+from livekit.agents.types import TimedString
+from livekit.agents.voice import Agent, AgentSession, ModelSettings
+
+from pinecall.session import _prompt, tools
+from pinecall.session._livekit import transcript_of
+from pinecall.session._prompt import Blocks
+from pinecall.wire.events import ErrorEvent
+
+type Heard = AsyncIterator[stt.SpeechEvent]
+type Declared = list[llm.Tool | llm.Toolset]
+type Thought = AsyncIterator[llm.ChatChunk | str]
+type Words = AsyncIterator[str | TimedString]
+# What plays while the caller waits: the hold melody after its grace, or nothing.
+type Waiting = Callable[[], AbstractAsyncContextManager[None]]
+
+
+LLM_TIMEOUT = "llm_timeout"
+
+
+MODEL_SILENT = "the model said nothing within {seconds:g}s, so the caller was asked to say it again"
+
+
+# How long a turn waits for the model's first word when the agent sets no deadline: past it the
+# caller hears MODEL_LATE instead of a silence a model stuck for 20 s would leave them in.
+LLM_TIMEOUT_S = 12.0
+
+
+# What the caller hears when the model is past the deadline, by the agent's language; English
+# where the agent's language has no sentence here.
+MODEL_LATE = {
+    "es": "Disculpe, me demoré. ¿Me lo repite, por favor?",
+    "en": "Sorry, that took me too long. Could you say that again?",
+    "pt": "Desculpe, demorei. Pode repetir, por favor?",
+}
+
+
+class CallAgent(Agent):
+    """livekit's Agent reading the call's prompt blocks, lookups and tools."""
+
+    def __init__(
+        self,
+        live: AgentSession[None],
+        blocks: Blocks,
+        lookups: tools.Lookups,
+        declared: Declared,
+        waiting: Waiting,
+    ) -> None:
+        """The agent with the static blocks as its instructions and every tool of the call."""
+        super().__init__(instructions=blocks.instructions, tools=declared)  # pyright: ignore[reportUnknownMemberType]
+        self.live = live
+        self.call = lookups.call
+        self.blocks = blocks
+        self.lookups = lookups
+        self.waiting = waiting
+
+    # Runs before the request and livekit times it, so the lookups here are the ones already
+    # running (started on an interim) or quick ones. No speech handle exists yet.
+    @override
+    async def on_user_turn_completed(
+        self, turn_ctx: llm.ChatContext, new_message: llm.ChatMessage
+    ) -> None:
+        """Collect this turn's lookups and write the ones that did not run."""
+        for skipped in await self.lookups.turn_ended(new_message.text_content or ""):
+            self.call.writing.write("error", skipped)
+
+    # Lookups and dynamic blocks go into this request only, so the cached prefix (instructions
+    # and history) stays the same bytes from turn to turn. The wait for the first token plays the
+    # melody as a tool's does, and the agent's deadline cuts it, livekit's retries with it.
+    @override
+    async def llm_node(
+        self, chat_ctx: llm.ChatContext, tools: list[llm.Tool], model_settings: ModelSettings
+    ) -> Thought:
+        """The model run on the history, this turn's lookups, and the dynamic blocks."""
+        params = _prompt.request(chat_ctx, self.blocks, self.lookups.items)
+        if self.call.context.run is not None:
+            self.call.requests.append(_prompt.as_asked(params, tools))
+        deadline = self.call.config.llm_timeout_s or LLM_TIMEOUT_S
+        thought = Agent.default.llm_node(self, params, tools, model_settings)
+        waited = asyncio.timeout(deadline)
+        async with contextlib.aclosing(thought):
+            try:
+                async with self.waiting(), waited:
+                    first = await anext(thought, None)
+            except TimeoutError:
+                # A vendor's own TimeoutError is livekit's to handle, as it was before.
+                if not waited.expired():
+                    raise
+                why = MODEL_SILENT.format(seconds=deadline)
+                self.call.writing.write(
+                    "error", ErrorEvent(code=LLM_TIMEOUT, message=why, recoverable=True)
+                )
+                yield _late_in(self.call.config.language)
+                return
+            if isinstance(first, llm.ChatChunk | str):
+                yield first
+            async for chunk in thought:
+                if isinstance(chunk, llm.ChatChunk | str):
+                    yield chunk
+
+    # Runs on what was played, so an interrupted reply's transcript stops where its audio did; a
+    # written call plays everything. Aligned speech yields one timed string per word.
+    @override
+    async def transcription_node(
+        self, text: AsyncIterable[str | TimedString], model_settings: ModelSettings
+    ) -> Words:
+        """Each played piece of the reply, written as it plays, then passed on."""
+        async for delta in Agent.default.transcription_node(self, text, model_settings):
+            if str(delta):
+                self.call.writing.write("agent.transcript", transcript_of(self.live, delta))
+            yield delta
+
+    # Before the turn detector: the last point to drop a backchannel at no cost of a model.
+    @override
+    async def stt_node(
+        self, audio: AsyncIterable[rtc.AudioFrame], model_settings: ModelSettings
+    ) -> Heard:
+        """What the ears heard, but agreement said over the agent's voice."""
+        async for event in Agent.default.stt_node(self, audio, model_settings):
+            if not self._agreement(event):
+                yield event
+
+    def _agreement(self, event: stt.SpeechEvent) -> bool:
+        text = event.alternatives[0].text if event.alternatives else ""
+        return self.call.agent_speaking and self.call.turn_policy.is_a_backchannel(text)
+
+
+def _late_in(language: str | None) -> str:
+    return MODEL_LATE.get((language or "en")[:2].lower(), MODEL_LATE["en"])

pinecall/session/_hearing.py

@@ -0,0 +1,101 @@
+"""What the ears are told: the words that only agree, the pause before a barge-in, the keyterms."""
+
+import re
+from dataclasses import dataclass
+
+from pinecall.domain.agent import AgentConfig, Turn
+from pinecall.domain.names import JsonObject
+
+# livekit counts no words by default; one word is as often a cough or an echo.
+MIN_WORDS = 2
+
+
+# Silence before an interruption is judged false: livekit's 2 s sound like a dropped call.
+FALSE_INTERRUPTION_TIMEOUT_S = 1.0
+
+
+# A turn made of these alone is somebody agreeing, and does not take the floor. livekit's own
+# backchannel detector is hosted only. Per language: "vale" takes the floor in English.
+BACKCHANNELS: dict[str, frozenset[str]] = {
+    "es": frozenset(
+        {
+            "aha", "ajá", "ah", "ajam", "bien", "bueno", "claro", "dale", "eh", "em", "exacto",
+            "hm", "hmm", "mm", "mmm", "ok", "okay", "perfecto", "sí", "si", "vale", "ya", "uh",
+            "uhu", "uhum",
+        }
+    ),
+    "en": frozenset(
+        {
+            "aha", "ah", "hm", "hmm", "mm", "mmm", "mhm", "ok", "okay", "yeah", "yes", "yep",
+            "right", "sure", "uh", "uh-huh", "gotcha", "exactly",
+        }
+    ),
+    "pt": frozenset({"aham", "ah", "sim", "tá", "certo", "claro", "ok", "beleza", "hum", "uhum"}),
+    "ca": frozenset({"sí", "vale", "d'acord", "clar", "entesos", "ok", "mm", "aha", "eh"}),
+}  # fmt: skip
+
+
+_A_WORD = re.compile(r"[^\W_]+(?:['\u2019-][^\W_]+)*")
+
+
+# Keyterms are names, not sentences: longer text dilutes them. The cap is ours, since the ears'
+# vendors either document none or count the terms against the model's budget.
+LONGEST_TERM = 40
+
+
+MOST_WORDS = 4
+
+
+MOST_TERMS = 50
+
+
+@dataclass(frozen=True)
+class TurnPolicy:
+    """What the caller must say to take the floor, in the agent's language."""
+
+    backchannels: frozenset[str]
+    min_words: int = MIN_WORDS
+    false_interruption_s: float = FALSE_INTERRUPTION_TIMEOUT_S
+    # How long the caller speaks over the agent before it stops; None leaves livekit's own.
+    min_speech_s: float | None = None
+
+    def is_a_backchannel(self, text: str) -> bool:
+        """Whether every word said is somebody agreeing."""
+        words: list[str] = [str(found.group(0)).lower() for found in _A_WORD.finditer(text)]
+        return bool(words) and all(word in self.backchannels for word in words)
+
+
+def policy_for(language: str | None, turn: Turn | None = None) -> TurnPolicy:
+    """The policy of the agent's language (`es-ES` is `es`) and its turn knobs; else every list."""
+    tag = (language or "").split("-", 1)[0].lower()
+    words = BACKCHANNELS.get(tag) or frozenset[str]().union(*BACKCHANNELS.values())
+    declared = turn or Turn()
+    return TurnPolicy(
+        backchannels=words,
+        min_words=MIN_WORDS
+        if declared.min_interruption_words is None
+        else declared.min_interruption_words,
+        min_speech_s=None
+        if declared.min_interruption_ms is None
+        else declared.min_interruption_ms / 1000,
+    )
+
+
+# The declared words first, so the cap drops the names found in the state before them. One
+# level deep catches `patient = {name, phone}`; a term needs a letter, which leaves out numbers.
+def keyterms(config: AgentConfig, state: JsonObject) -> list[str]:
+    """The words the ears are told to expect: the agent's own, then names the state holds."""
+    found: list[object] = []
+    for value in state.values():
+        found += list(value.values()) if isinstance(value, dict) else [value]
+    names = [name for name in (_a_name(item) for item in found) if name]
+    return list(dict.fromkeys(term for term in (*config.hears, *names) if term))[:MOST_TERMS]
+
+
+def _a_name(value: object) -> str:
+    if not isinstance(value, str):
+        return ""
+    name = value.strip()
+    if not name or len(name) > LONGEST_TERM or len(name.split()) > MOST_WORDS:
+        return ""
+    return name if any(letter.isalpha() for letter in name) else ""