agent-learning-kit 0.1.0__py3-none-any.whl

fi/simulate/simulation/bridge/livekit.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+
+from livekit import rtc
+from livekit.api import AccessToken, VideoGrants
+
+from fi.simulate.simulation.bridge.audio import PCMResampler
+from fi.simulate.simulation.bridge.connector import ProviderConnector
+
+logger = logging.getLogger(__name__)
+ROOM_SAMPLE_RATE = 48000
+ROOM_CHANNELS = 1
+TRACK_TIMEOUT_SECONDS = 30.0
+WATCHDOG_TIMEOUT_SECONDS = 60.0
+PROVIDER_READY_BUFFER_FRAMES = 3000
+PROVIDER_AUDIO_TIMEOUT_SECONDS = 120.0
+PROVIDER_QUEUE_MAX_CHUNKS = 200
+
+
+class LiveKitAudioBridge:
+    def __init__(
+        self,
+        *,
+        url: str,
+        api_key: str,
+        api_secret: str,
+        room_name: str,
+        identity: str,
+        connector: ProviderConnector,
+    ) -> None:
+        self._url = url
+        self._api_key = api_key
+        self._api_secret = api_secret
+        self._room_name = room_name
+        self._identity = identity
+        self._connector = connector
+        self._room = rtc.Room()
+        self._track_future: asyncio.Future[rtc.RemoteAudioTrack] | None = None
+        self._room_disconnected: asyncio.Future[None] | None = None
+        self._audio_source: rtc.AudioSource | None = None
+        self._closed = False
+        self._close_lock = asyncio.Lock()
+        self._last_audio_at = time.monotonic()
+        self._last_provider_audio_at = time.monotonic()
+
+    @property
+    def call_id(self) -> str | None:
+        return self._connector.call_id
+
+    async def connect(self) -> None:
+        loop = asyncio.get_running_loop()
+        self._track_future = loop.create_future()
+        self._room_disconnected = loop.create_future()
+
+        @self._room.on("track_subscribed")
+        def _on_track(track, publication, participant) -> None:
+            if not isinstance(track, rtc.RemoteAudioTrack):
+                return
+            if self._track_future is None or self._track_future.done():
+                return
+            if publication.source != rtc.TrackSource.SOURCE_MICROPHONE:
+                return
+            if participant.kind == rtc.ParticipantKind.PARTICIPANT_KIND_SIP:
+                return
+            self._track_future.set_result(track)
+
+        @self._room.on("disconnected")
+        def _on_disconnected(*_args) -> None:
+            if self._room_disconnected and not self._room_disconnected.done():
+                self._room_disconnected.set_result(None)
+
+        token = (
+            AccessToken(self._api_key, self._api_secret)
+            .with_identity(self._identity)
+            .with_name("FutureAGI Web Bridge")
+            .with_kind("sip")
+            .with_grants(
+                VideoGrants(
+                    room_join=True,
+                    room=self._room_name,
+                    can_publish=True,
+                    can_subscribe=True,
+                )
+            )
+            .to_jwt()
+        )
+        try:
+            await self._room.connect(self._url, token)
+            self._latch_preexisting_track()
+            self._audio_source = rtc.AudioSource(ROOM_SAMPLE_RATE, ROOM_CHANNELS)
+            track = rtc.LocalAudioTrack.create_audio_track(
+                "bridge-audio", self._audio_source
+            )
+            await self._room.local_participant.publish_track(
+                track,
+                rtc.TrackPublishOptions(source=rtc.TrackSource.SOURCE_MICROPHONE),
+            )
+            await self._connector.connect()
+        except Exception:
+            await self.aclose()
+            raise
+
+    async def run(self) -> None:
+        tasks = {
+            asyncio.create_task(self._room_to_provider()),
+            asyncio.create_task(self._provider_to_room()),
+            asyncio.create_task(self._watchdog()),
+            asyncio.create_task(self._wait_for_room_disconnect()),
+        }
+        try:
+            done, pending = await asyncio.wait(
+                tasks, return_when=asyncio.FIRST_COMPLETED
+            )
+            for task in pending:
+                task.cancel()
+            await asyncio.gather(*pending, return_exceptions=True)
+            for task in done:
+                await task
+        finally:
+            await self.aclose()
+
+    async def aclose(self) -> None:
+        async with self._close_lock:
+            if self._closed:
+                return
+            self._closed = True
+            try:
+                await self._connector.disconnect()
+            finally:
+                await self._room.disconnect()
+
+    def _latch_preexisting_track(self) -> None:
+        if self._track_future is None or self._track_future.done():
+            return
+        for participant in self._room.remote_participants.values():
+            if participant.kind == rtc.ParticipantKind.PARTICIPANT_KIND_SIP:
+                continue
+            for publication in participant.track_publications.values():
+                if (
+                    publication.track
+                    and isinstance(publication.track, rtc.RemoteAudioTrack)
+                    and publication.source == rtc.TrackSource.SOURCE_MICROPHONE
+                ):
+                    self._track_future.set_result(publication.track)
+                    return
+
+    async def _room_to_provider(self) -> None:
+        if self._track_future is None:
+            raise RuntimeError("bridge_not_connected")
+        silence = asyncio.create_task(self._send_silence_until_track())
+        try:
+            track = await asyncio.wait_for(
+                self._track_future, timeout=TRACK_TIMEOUT_SECONDS
+            )
+        finally:
+            silence.cancel()
+            await asyncio.gather(silence, return_exceptions=True)
+        buffered_frames: list[tuple[bytes, int]] = []
+        async for event in rtc.AudioStream(track):
+            frame = event.frame
+            self._last_audio_at = time.monotonic()
+            frame_data = frame.data.tobytes()
+            if not self._connector.is_agent_ready:
+                if len(buffered_frames) < PROVIDER_READY_BUFFER_FRAMES:
+                    buffered_frames.append((frame_data, frame.sample_rate))
+                continue
+            for buffered_data, buffered_rate in buffered_frames:
+                await self._connector.send_audio(buffered_data, buffered_rate)
+            buffered_frames.clear()
+            await self._connector.send_audio(frame_data, frame.sample_rate)
+
+    async def _provider_to_room(self) -> None:
+        if self._audio_source is None:
+            raise RuntimeError("bridge_not_connected")
+        # The websocket reader must never block on room playback: a stalled
+        # ``capture_frame`` would stop close/hangup frames from being seen and
+        # keep a dead provider call alive. Live audio, so drop oldest when full.
+        queue: asyncio.Queue[tuple[bytes, int] | None] = asyncio.Queue(
+            maxsize=PROVIDER_QUEUE_MAX_CHUNKS
+        )
+
+        async def _pump() -> None:
+            try:
+                async for chunk in self._connector.recv_audio():
+                    now = time.monotonic()
+                    self._last_audio_at = now
+                    self._last_provider_audio_at = now
+                    if queue.full():
+                        queue.get_nowait()
+                    queue.put_nowait(chunk)
+            finally:
+                if queue.full():
+                    queue.get_nowait()
+                queue.put_nowait(None)
+
+        pump = asyncio.create_task(_pump())
+        resamplers: dict[int, PCMResampler] = {}
+        try:
+            while True:
+                chunk = await queue.get()
+                if chunk is None:
+                    break
+                pcm, sample_rate = chunk
+                if sample_rate != ROOM_SAMPLE_RATE:
+                    resampler = resamplers.setdefault(
+                        sample_rate,
+                        PCMResampler(
+                            from_rate=sample_rate,
+                            to_rate=ROOM_SAMPLE_RATE,
+                            channels=ROOM_CHANNELS,
+                        ),
+                    )
+                    pcm = resampler.convert(pcm)
+                await self._audio_source.capture_frame(
+                    rtc.AudioFrame(
+                        data=pcm,
+                        sample_rate=ROOM_SAMPLE_RATE,
+                        num_channels=ROOM_CHANNELS,
+                        samples_per_channel=len(pcm) // 2,
+                    )
+                )
+        finally:
+            pump.cancel()
+            await asyncio.gather(pump, return_exceptions=True)
+
+    async def _send_silence_until_track(self) -> None:
+        frame = b"\x00" * int(16000 * 0.02 * 2)
+        while self._track_future is not None and not self._track_future.done():
+            await self._connector.send_audio(frame, 16000)
+            await asyncio.sleep(0.02)
+
+    async def _watchdog(self) -> None:
+        # ``_last_audio_at`` refreshes on simulator silence frames too, so it is
+        # blind to a dead provider; track provider-received audio separately.
+        while True:
+            await asyncio.sleep(5.0)
+            now = time.monotonic()
+            if now - self._last_audio_at > WATCHDOG_TIMEOUT_SECONDS:
+                raise RuntimeError("bridge_audio_watchdog_timeout")
+            if (
+                now - self._last_provider_audio_at
+                > PROVIDER_AUDIO_TIMEOUT_SECONDS
+            ):
+                raise RuntimeError("provider_audio_timeout")
+
+    async def _wait_for_room_disconnect(self) -> None:
+        if self._room_disconnected is None:
+            raise RuntimeError("bridge_not_connected")
+        await self._room_disconnected

fi/simulate/simulation/bridge/retell.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from collections.abc import AsyncIterator
+
+import aiohttp
+from livekit import rtc
+
+from fi.simulate.agent.definition import RetellTargetConfig
+from fi.simulate.simulation.bridge.audio import PCMResampler
+from fi.simulate.simulation.bridge.connector import ConnectorConfig, ProviderConnector
+
+logger = logging.getLogger(__name__)
+RETELL_SAMPLE_RATE = 48000
+
+
+class RetellWebCallConnector(ProviderConnector):
+    def __init__(self, config: ConnectorConfig) -> None:
+        self._config = config
+        self._room: rtc.Room | None = None
+        self._audio_source: rtc.AudioSource | None = None
+        self._track_future: asyncio.Future[rtc.RemoteAudioTrack] | None = None
+        self._agent_disconnected = asyncio.Event()
+        self._agent_identity: str | None = None
+        self._agent_ready = asyncio.Event()
+        self._call_id: str | None = None
+        self._connected = False
+        self._resamplers: dict[int, PCMResampler] = {}
+
+    @classmethod
+    def from_target(cls, target: RetellTargetConfig) -> "RetellWebCallConnector":
+        api_key = os.environ.get(target.api_key_env, "").strip()
+        if not api_key:
+            raise ValueError("retell_webcall_config_missing: " + target.api_key_env)
+        return cls(
+            ConnectorConfig(
+                api_key=api_key,
+                assistant_id=target.agent_id,
+                api_url=str(target.api_url),
+                livekit_url=str(target.livekit_url),
+            )
+        )
+
+    @classmethod
+    def from_env(cls) -> "RetellWebCallConnector":
+        api_key = os.environ.get("RETELL_API_KEY", "").strip()
+        agent_id = os.environ.get("RETELL_AGENT_ID", "").strip()
+        missing = [
+            name
+            for name, value in (
+                ("RETELL_API_KEY", api_key),
+                ("RETELL_AGENT_ID", agent_id),
+            )
+            if not value
+        ]
+        if missing:
+            raise ValueError("retell_webcall_config_missing: " + ", ".join(missing))
+        return cls.from_target(
+            RetellTargetConfig(
+                agent_id=agent_id,
+                api_url=os.environ.get(
+                    "RETELL_API_URL",
+                    "https://api.retellai.com/v2/create-web-call",
+                ),
+                livekit_url=os.environ.get(
+                    "RETELL_LIVEKIT_URL",
+                    "wss://retell-ai-4ihahnq7.livekit.cloud",
+                ),
+            )
+        )
+
+    async def connect(self) -> None:
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                self._config.api_url,
+                headers={"Authorization": f"Bearer {self._config.api_key}"},
+                json={"agent_id": self._config.assistant_id},
+            ) as response:
+                if response.status != 201:
+                    raise RuntimeError(
+                        f"retell_webcall_create_failed:{response.status}"
+                    )
+                payload = await response.json()
+        access_token = (
+            payload.get("access_token") if isinstance(payload, dict) else None
+        )
+        call_id = payload.get("call_id") if isinstance(payload, dict) else None
+        if not isinstance(access_token, str) or not access_token.strip():
+            raise ValueError("retell_webcall_response_missing_access_token")
+        if not isinstance(call_id, str) or not call_id.strip():
+            raise ValueError("retell_webcall_response_missing_call_id")
+        self._call_id = call_id
+        self._room = rtc.Room()
+        self._track_future = asyncio.get_running_loop().create_future()
+
+        @self._room.on("track_subscribed")
+        def _on_track(track, _publication, participant) -> None:
+            if not isinstance(track, rtc.RemoteAudioTrack):
+                return
+            if self._track_future is None or self._track_future.done():
+                return
+            identity = getattr(participant, "identity", None)
+            if identity is not None:
+                self._agent_identity = str(identity)
+            self._track_future.set_result(track)
+            self._agent_ready.set()
+
+        @self._room.on("participant_disconnected")
+        def _on_participant_disconnected(participant) -> None:
+            identity = getattr(participant, "identity", None)
+            if self._agent_identity is None or str(identity) == self._agent_identity:
+                self._agent_disconnected.set()
+
+        @self._room.on("disconnected")
+        def _on_disconnected(*_args) -> None:
+            self._agent_disconnected.set()
+
+        try:
+            await self._room.connect(self._config.livekit_url, access_token)
+            self._audio_source = rtc.AudioSource(RETELL_SAMPLE_RATE, 1)
+            track = rtc.LocalAudioTrack.create_audio_track(
+                "bridge-audio", self._audio_source
+            )
+            await self._room.local_participant.publish_track(
+                track,
+                rtc.TrackPublishOptions(source=rtc.TrackSource.SOURCE_MICROPHONE),
+            )
+            self._connected = True
+            if self._track_future is None:
+                raise RuntimeError("retell_webcall_track_future_missing")
+            await asyncio.wait_for(asyncio.shield(self._track_future), timeout=30.0)
+            logger.info("retell_webcall_connected", extra={"call_id": call_id})
+        except asyncio.TimeoutError as exc:
+            await self.disconnect()
+            raise RuntimeError("retell_webcall_agent_track_timeout") from exc
+        except Exception:
+            await self.disconnect()
+            raise
+
+    async def send_audio(self, data: bytes, sample_rate: int) -> None:
+        if not self._audio_source or not self._connected:
+            return
+        if sample_rate != RETELL_SAMPLE_RATE:
+            resampler = self._resamplers.setdefault(
+                sample_rate,
+                PCMResampler(from_rate=sample_rate, to_rate=RETELL_SAMPLE_RATE),
+            )
+            data = resampler.convert(data)
+        await self._audio_source.capture_frame(
+            rtc.AudioFrame(
+                data=data,
+                sample_rate=RETELL_SAMPLE_RATE,
+                num_channels=1,
+                samples_per_channel=len(data) // 2,
+            )
+        )
+
+    async def recv_audio(self) -> AsyncIterator[tuple[bytes, int]]:
+        if self._track_future is None:
+            return
+        try:
+            track = await asyncio.wait_for(self._track_future, timeout=30.0)
+        except asyncio.TimeoutError as exc:
+            self._connected = False
+            raise RuntimeError("retell_webcall_agent_track_timeout") from exc
+        async for event in rtc.AudioStream(track):
+            frame = event.frame
+            yield frame.data.tobytes(), frame.sample_rate
+        self._connected = False
+
+    async def disconnect(self) -> None:
+        self._connected = False
+        if self._room:
+            await self._room.disconnect()
+
+    @property
+    def is_connected(self) -> bool:
+        return self._connected
+
+    @property
+    def is_agent_ready(self) -> bool:
+        return self._agent_ready.is_set()
+
+    @property
+    def call_id(self) -> str | None:
+        return self._call_id

fi/simulate/simulation/bridge/vapi.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+from collections.abc import AsyncIterator
+
+import aiohttp
+
+from fi.simulate.agent.definition import VapiTargetConfig
+from fi.simulate.simulation.bridge.audio import PCMResampler
+from fi.simulate.simulation.bridge.connector import ConnectorConfig, ProviderConnector
+
+logger = logging.getLogger(__name__)
+VAPI_SAMPLE_RATE = 16000
+
+
+class VapiWebSocketConnector(ProviderConnector):
+    def __init__(self, config: ConnectorConfig) -> None:
+        self._config = config
+        self._session: aiohttp.ClientSession | None = None
+        self._ws: aiohttp.ClientWebSocketResponse | None = None
+        self._call_id: str | None = None
+        self._connected = False
+        self._resamplers: dict[int, PCMResampler] = {}
+
+    @classmethod
+    def from_target(
+        cls,
+        target: VapiTargetConfig,
+        *,
+        first_message_mode: str | None = None,
+    ) -> "VapiWebSocketConnector":
+        api_key = os.environ.get(target.api_key_env, "").strip()
+        if not api_key:
+            raise ValueError("vapi_websocket_config_missing: " + target.api_key_env)
+        return cls(
+            ConnectorConfig(
+                api_key=api_key,
+                assistant_id=target.assistant_id,
+                api_url=f"{str(target.api_base_url).rstrip('/')}/call",
+                first_message_mode=first_message_mode,
+            )
+        )
+
+    @classmethod
+    def from_env(cls) -> "VapiWebSocketConnector":
+        api_key = os.environ.get("VAPI_API_KEY", "").strip()
+        assistant_id = os.environ.get("VAPI_ASSISTANT_ID", "").strip()
+        missing = [
+            name
+            for name, value in (
+                ("VAPI_API_KEY", api_key),
+                ("VAPI_ASSISTANT_ID", assistant_id),
+            )
+            if not value
+        ]
+        if missing:
+            raise ValueError("vapi_websocket_config_missing: " + ", ".join(missing))
+        return cls.from_target(
+            VapiTargetConfig(
+                assistant_id=assistant_id,
+                api_base_url=os.environ.get("VAPI_API_BASE_URL", "https://api.vapi.ai"),
+            )
+        )
+
+    async def connect(self) -> None:
+        self._session = aiohttp.ClientSession()
+        try:
+            payload = {
+                "assistantId": self._config.assistant_id,
+                "transport": {
+                    "provider": "vapi.websocket",
+                    "audioFormat": {
+                        "format": "pcm_s16le",
+                        "container": "raw",
+                        "sampleRate": VAPI_SAMPLE_RATE,
+                    },
+                },
+            }
+            if self._config.first_message_mode:
+                payload["assistantOverrides"] = {
+                    "firstMessageMode": self._config.first_message_mode,
+                }
+            async with self._session.post(
+                self._config.api_url,
+                headers={"Authorization": f"Bearer {self._config.api_key}"},
+                json=payload,
+            ) as response:
+                if response.status != 201:
+                    raise RuntimeError(
+                        f"vapi_websocket_call_create_failed:{response.status}"
+                    )
+                payload = await response.json()
+            call_id = payload.get("id") if isinstance(payload, dict) else None
+            transport = payload.get("transport") if isinstance(payload, dict) else None
+            websocket_url = (
+                transport.get("websocketCallUrl")
+                if isinstance(transport, dict)
+                else None
+            )
+            if not isinstance(call_id, str) or not call_id.strip():
+                raise ValueError("vapi_websocket_response_missing_call_id")
+            if not isinstance(websocket_url, str) or not websocket_url.startswith(
+                ("ws://", "wss://")
+            ):
+                raise ValueError("vapi_websocket_response_missing_url")
+            self._call_id = call_id
+            self._ws = await self._session.ws_connect(websocket_url)
+            self._connected = True
+            logger.info("vapi_websocket_connected", extra={"call_id": call_id})
+        except Exception:
+            await self.disconnect()
+            raise
+
+    async def send_audio(self, data: bytes, sample_rate: int) -> None:
+        if not self._ws or self._ws.closed:
+            return
+        if sample_rate != VAPI_SAMPLE_RATE:
+            resampler = self._resamplers.setdefault(
+                sample_rate,
+                PCMResampler(from_rate=sample_rate, to_rate=VAPI_SAMPLE_RATE),
+            )
+            data = resampler.convert(data)
+        await self._ws.send_bytes(data)
+
+    async def recv_audio(self) -> AsyncIterator[tuple[bytes, int]]:
+        if not self._ws or self._ws.closed:
+            return
+        async for message in self._ws:
+            if message.type == aiohttp.WSMsgType.BINARY:
+                yield message.data, VAPI_SAMPLE_RATE
+            elif message.type == aiohttp.WSMsgType.TEXT:
+                if self._is_call_end_event(message.data):
+                    logger.info(
+                        "vapi_websocket_call_ended",
+                        extra={"call_id": self._call_id},
+                    )
+                    break
+            elif message.type in {
+                aiohttp.WSMsgType.CLOSED,
+                aiohttp.WSMsgType.CLOSING,
+                aiohttp.WSMsgType.CLOSE,
+                aiohttp.WSMsgType.ERROR,
+            }:
+                break
+        self._connected = False
+
+    @staticmethod
+    def _is_call_end_event(payload: str) -> bool:
+        try:
+            data = json.loads(payload)
+        except ValueError:
+            return False
+        if not isinstance(data, dict):
+            return False
+        event_type = str(data.get("type") or "").lower()
+        if event_type in {"hangup", "call-ended", "end-of-call-report"}:
+            return True
+        return event_type == "status-update" and str(
+            data.get("status") or ""
+        ).lower() in {"ended", "ending"}
+
+    async def disconnect(self) -> None:
+        self._connected = False
+        if self._ws and not self._ws.closed:
+            await self._ws.close()
+        if self._session and not self._session.closed:
+            await self._session.close()
+
+    @property
+    def is_connected(self) -> bool:
+        return self._connected
+
+    @property
+    def call_id(self) -> str | None:
+        return self._call_id