livekit-plugins-chatfx 0.1.0__py3-none-any.whl

livekit/plugins/chatfx/__init__.py

@@ -0,0 +1,37 @@
+"""ChatFX Avatar for LiveKit Agents.
+
+A real-time talking face for your voice agent. The avatar joins your agent's LiveKit room as one more participant, takes
+your agent's voice and speaks it with its lips in time, shown as your agent.
+
+    from livekit.plugins import chatfx
+
+    avatar = chatfx.AvatarSession(avatar_id="chatfx-lois")  # your key in CHATFX_AVATAR_API_KEY
+    await avatar.start(session, room=ctx.room)
+
+Docs: https://chatfx.com/support/avatar-connect/
+"""
+
+from livekit.agents import Plugin
+
+from .avatar import AvatarSession, ChatFXException, warm
+from .log import logger
+from .version import __version__
+
+__all__ = ["AvatarSession", "ChatFXException", "warm", "__version__"]
+
+
+class ChatFXPlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(ChatFXPlugin())
+
+# Cleanup docs of unexported modules
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit/plugins/chatfx/avatar.py

@@ -0,0 +1,624 @@
+"""The ChatFX Avatar session: sends a talking face into your agent's LiveKit room.
+
+Everything about how the avatar behaves in a call lives here, so every customer gets it by adding two lines:
+
+1. A waiting picture, with no work on your page. The moment the call starts, your agent shows the avatar's picture as its own
+   video, with "Video starting..." on it, and it comes down when the live face is ready. If the face never comes, the
+   picture stays, without the words, with your agent's voice over it. Any page that shows the agent's video gets this.
+2. Face first. A waking GPU takes about 11 seconds, so start() holds your agent back for up to face_first_seconds (10 by
+   default) and the agent greets with its face.
+3. Voice if the face is late. After that wait your agent starts talking with its own voice, and the voice moves to the face
+   between replies when it arrives, so a sentence is never split between two voices.
+4. The call never depends on the face. If the face can't come, or leaves mid-call, your agent carries on by voice.
+5. It keeps trying while a GPU wakes or is busy, for up to start_timeout seconds, and stops at once on a refusal more tries
+   won't fix (a wrong key, no minutes left).
+6. Your page can follow along. The agent's participant attribute "chatfx-face" says "waking", "voice", "on" or "off", so
+   you can show a photo or a message while the face is on its way.
+7. No call runs up the meter by accident. It ends after 2 minutes with nobody talking, 2 minutes after the visitor leaves
+   (long enough for a page reload), or at 1 hour. Each of these is a setting.
+8. Your agent can't go deaf. It learns the avatar has finished a reply from a message the face sends; if that's late, the
+   plugin ends the reply itself once the sound has run out, so the next question isn't ignored.
+9. Optional head start: warm(), called from your own server the moment a visitor is about to start a call.
+
+How it connects: a LiveKit pass (access token) is made here with your own LiveKit key and secret, for the participant
+"chatfx-avatar", to join this one room only, shown as your agent (lk.publish_on_behalf), valid for 5 hours. Your LiveKit key
+and secret never leave your server. POST https://api.chatfx.com/v1/sessions starts the avatar; your agent's voice goes to it
+as a LiveKit data stream; a check-in every 20 seconds keeps it going; aclose() (run when the job shuts down) ends it, so
+billing stops.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import os
+import time
+from datetime import timedelta
+from typing import Any, Literal
+
+import aiohttp
+
+from livekit import api, rtc
+from livekit.agents import AgentSession, get_job_context, utils
+from livekit.agents.voice.avatar import AvatarSession as _AvatarSession, DataStreamAudioOutput
+from livekit.agents.voice.room_io import ATTRIBUTE_PUBLISH_ON_BEHALF
+
+from .log import logger
+
+DEFAULT_API_URL = "https://api.chatfx.com"
+AVATAR_IDENTITY = "chatfx-avatar"
+AVATAR_NAME = "ChatFX"
+KEEPALIVE_SECONDS = 20
+# Our API accepts passes that expire within 6 hours.
+PASS_TTL = timedelta(hours=5)
+# Where the face is, for your page: the attribute on your agent's participant.
+FACE_ATTRIBUTE = "chatfx-face"
+FaceStatus = Literal["waking", "voice", "on", "off"]
+# How long the GPU has, once it said yes, to put the picture in the room.
+PICTURE_SECONDS = 8.0
+# The API can hold a start while a GPU wakes (up to 45 seconds).
+START_REQUEST_SECONDS = 60
+REQUEST_SECONDS = 15
+RETRY_SECONDS = 3.0
+# How often the call is checked for the idle rules and stuck speech.
+WATCH_SECONDS = 1.0
+# Past the end of its own sound, the avatar isn't talking, whatever the face has reported.
+SOUND_GRACE_SECONDS = 5.0
+# The waiting picture is the same frame over and over, so a few a second is plenty.
+STILL_FPS = 5
+STILL_TRACK = "chatfx-waiting"
+STILL_LONGEST_SIDE = 1280
+
+
+class ChatFXException(Exception):
+    """The ChatFX Avatar API said no. code is its error code (for example invalid_api_key or out_of_minutes)."""
+
+    def __init__(self, message: str, *, code: str = "error", status: int = 0) -> None:
+        super().__init__(message)
+        self.code = code
+        self.status = status
+
+
+def _settings(api_key: str | None, api_url: str | None) -> tuple[str, str]:
+    key = api_key or os.environ.get("CHATFX_AVATAR_API_KEY")
+    if not key:
+        raise ChatFXException("Set CHATFX_AVATAR_API_KEY, or pass api_key. Create a key on the API keys page.", code="missing_api_key")
+    return key, (api_url or os.environ.get("CHATFX_AVATAR_API_URL") or DEFAULT_API_URL).rstrip("/")
+
+
+async def warm(*, api_key: str | None = None, api_url: str | None = None) -> bool:
+    """Starts waking a GPU ahead of a call, so the face is ready sooner. Optional, free, and safe to call often.
+
+    Call it from your own server the moment a visitor is about to start a call (never from a web page, where your key would
+    show), and again every 20 seconds while they get ready. True when the API took the request.
+    """
+    key, url = _settings(api_key, api_url)
+    try:
+        async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=REQUEST_SECONDS)) as http:
+            async with http.post(url + "/v1/warm", headers={"Authorization": f"Bearer {key}"}) as response:
+                await response.read()
+                return response.status == 200
+    except (aiohttp.ClientError, asyncio.TimeoutError):
+        logger.debug("chatfx head start didn't go through", exc_info=True)
+        return False
+
+
+def _still_frame(picture: bytes, text: str | None) -> rtc.VideoFrame:
+    """The avatar's picture as a video frame, with the words in a soft band near the bottom."""
+    from PIL import Image, ImageDraw, ImageFont
+
+    image = Image.open(io.BytesIO(picture)).convert("RGBA")
+    image.thumbnail((STILL_LONGEST_SIDE, STILL_LONGEST_SIDE))
+    # Video encoders need even sizes.
+    image = image.crop((0, 0, image.width - image.width % 2, image.height - image.height % 2))
+    if text:
+        size = max(14, round(min(image.width, image.height) * 0.05))
+        font = ImageFont.load_default(size=size)
+        band = Image.new("RGBA", image.size, (0, 0, 0, 0))
+        draw = ImageDraw.Draw(band)
+        left, top, right, bottom = draw.textbbox((0, 0), text, font=font)
+        x, y = (image.width - (right - left)) // 2, image.height - size * 3
+        pad = size // 2
+        draw.rounded_rectangle((x - pad * 2, y - pad + top, x + right - left + pad * 2, y + bottom + pad), radius=size, fill=(12, 10, 40, 170))
+        draw.text((x - left, y), text, font=font, fill=(255, 255, 255, 255))
+        image = Image.alpha_composite(image, band)
+    return rtc.VideoFrame(image.width, image.height, rtc.VideoBufferType.RGBA, image.tobytes())
+
+
+class _WaitingPicture:
+    """The avatar's picture, published as the agent's own video until the live face is ready."""
+
+    def __init__(self, room: rtc.Room, picture: bytes, text: str | None) -> None:
+        self._room = room
+        self._with_words = _still_frame(picture, text)
+        self._plain = _still_frame(picture, None) if text else self._with_words
+        self._frame = self._with_words
+        self._source = rtc.VideoSource(self._frame.width, self._frame.height)
+        self._publication: Any = None
+        self._task: asyncio.Task[None] | None = None
+
+    async def show(self, words: bool = True) -> None:
+        self.words(words)
+        track = rtc.LocalVideoTrack.create_video_track(STILL_TRACK, self._source)
+        self._publication = await self._room.local_participant.publish_track(
+            track, rtc.TrackPublishOptions(source=rtc.TrackSource.SOURCE_CAMERA, simulcast=False)
+        )
+        self._task = asyncio.create_task(self._keep_sending())
+
+    def words(self, on: bool) -> None:
+        self._frame = self._with_words if on else self._plain
+
+    async def _keep_sending(self) -> None:
+        while True:
+            self._source.capture_frame(self._frame)
+            await asyncio.sleep(1 / STILL_FPS)
+
+    async def hide(self) -> None:
+        if self._task is not None:
+            await utils.aio.cancel_and_wait(self._task)
+            self._task = None
+        publication, self._publication = self._publication, None
+        if publication is not None:
+            try:
+                await self._room.local_participant.unpublish_track(publication.sid)
+            except Exception:
+                logger.debug("couldn't take down the waiting picture", exc_info=True)
+
+
+class _FaceVoice(DataStreamAudioOutput):
+    """The agent's voice going to the face, counting the sound sent so the plugin knows when a reply really ends."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.sound_ends_at = 0.0
+
+    async def capture_frame(self, frame: rtc.AudioFrame) -> None:
+        self.sound_ends_at = max(self.sound_ends_at, time.monotonic()) + frame.duration
+        await super().capture_frame(frame)
+
+
+class AvatarSession(_AvatarSession):
+    """A ChatFX avatar for one call.
+
+    Args:
+        avatar_id: the avatar's id from the dashboard (for example "chatfx-lois").
+        api_key: your ChatFX Avatar API key. Defaults to the CHATFX_AVATAR_API_KEY setting.
+        api_url: defaults to https://api.chatfx.com (or the CHATFX_AVATAR_API_URL setting).
+        avatar_participant_identity: the avatar's identity in the room. Defaults to "chatfx-avatar".
+        avatar_participant_name: the avatar's name in the room. Defaults to "ChatFX".
+        face_first_seconds: how long start() waits for the face before your agent starts by voice. Defaults to 10.
+        start_timeout: seconds to keep trying while a GPU wakes up or is busy. Defaults to 300.
+        quiet_seconds: end the call after this long with nobody talking. Defaults to 120. None turns it off.
+        visitor_gone_seconds: end the call this long after the last visitor leaves. Defaults to 120. None turns it off.
+        longest_call_seconds: end the call at this length. Defaults to 3600. None turns it off.
+        waiting_picture: show the avatar's picture as your agent's video until the live face is ready. Defaults to True.
+        waiting_text: the words on it while the face is on its way. Defaults to "Video starting...". None: no words.
+        end_call_when_idle: also close the LiveKit room when one of those rules ends the call, which stops your own meters
+            too. Defaults to True. False: only the avatar leaves, and your agent carries on by voice.
+    """
+
+    def __init__(
+        self,
+        *,
+        avatar_id: str,
+        api_key: str | None = None,
+        api_url: str | None = None,
+        avatar_participant_identity: str = AVATAR_IDENTITY,
+        avatar_participant_name: str = AVATAR_NAME,
+        face_first_seconds: float = 10.0,
+        start_timeout: float = 300.0,
+        quiet_seconds: float | None = 120.0,
+        visitor_gone_seconds: float | None = 120.0,
+        longest_call_seconds: float | None = 3600.0,
+        end_call_when_idle: bool = True,
+        waiting_picture: bool = True,
+        waiting_text: str | None = "Video starting...",
+    ) -> None:
+        super().__init__()
+        if not avatar_id:
+            raise ChatFXException("avatar_id is required. Copy one from the Avatars page.", code="invalid_request")
+        self._avatar_id = avatar_id
+        self._api_key, self._api_url = _settings(api_key, api_url)
+        self._identity = avatar_participant_identity
+        self._name = avatar_participant_name
+        self._face_first_seconds = face_first_seconds
+        self._start_timeout = start_timeout
+        self._session_id: str | None = None
+        self._keepalive_task: asyncio.Task[None] | None = None
+        self._face_task: asyncio.Task[None] | None = None
+        self._own_http: aiohttp.ClientSession | None = None
+        self._face_on = asyncio.Event()
+        self._face_status: FaceStatus | None = None
+        self._failure: ChatFXException | None = None
+        self._quiet_seconds = quiet_seconds
+        self._visitor_gone_seconds = visitor_gone_seconds
+        self._longest_call_seconds = longest_call_seconds
+        self._end_call_when_idle = end_call_when_idle
+        self._waiting_picture = waiting_picture
+        self._waiting_text = waiting_text
+        self._still: _WaitingPicture | None = None
+        self._still_task: asyncio.Task[None] | None = None
+        self._early_wake: asyncio.Task[bool] | None = None
+        self._watch_task: asyncio.Task[None] | None = None
+        self._face_voice: _FaceVoice | None = None
+        self._voice_output: Any = None
+        self._on_face = False
+        self._closing = False
+        self._began = 0.0
+        self._last_talk = 0.0
+        self._visitor_left_at: float | None = None
+        # Starts the GPU waking now, while the rest of the call gets ready.
+        try:
+            self._early_wake = asyncio.get_running_loop().create_task(warm(api_key=self._api_key, api_url=self._api_url))
+        except RuntimeError:
+            pass
+
+    @property
+    def avatar_identity(self) -> str:
+        return self._identity
+
+    @property
+    def provider(self) -> str:
+        return "chatfx"
+
+    @property
+    def session_id(self) -> str | None:
+        """The ChatFX session's id once the avatar has started."""
+        return self._session_id
+
+    @property
+    def face_status(self) -> FaceStatus | None:
+        """Where the face is: "waking", "voice", "on" or "off"."""
+        return self._face_status
+
+    @property
+    def failure(self) -> ChatFXException | None:
+        """Why there's no face, when the API refused (a wrong key, no minutes left). The call carries on by voice."""
+        return self._failure
+
+    async def start(
+        self,
+        agent_session: AgentSession,
+        room: rtc.Room,
+        *,
+        livekit_url: str | None = None,
+        livekit_api_key: str | None = None,
+        livekit_api_secret: str | None = None,
+    ) -> None:
+        """Sends the avatar into the room, waiting up to face_first_seconds for it. Call before session.start().
+
+        Returns with the face on, or with the face still on its way (your agent starts by voice and the voice moves over
+        when the face arrives), or with no face (your agent carries on by voice; see failure and the log).
+        """
+        url = livekit_url or os.environ.get("LIVEKIT_URL")
+        key = livekit_api_key or os.environ.get("LIVEKIT_API_KEY")
+        secret = livekit_api_secret or os.environ.get("LIVEKIT_API_SECRET")
+        if not url or not key or not secret:
+            raise ChatFXException("Set LIVEKIT_URL, LIVEKIT_API_KEY and LIVEKIT_API_SECRET, or pass them to start().", code="invalid_request")
+
+        await super().start(agent_session, room)
+        # The pass names the room, and the status goes on the agent's participant: both need the room joined.
+        job_ctx = get_job_context(required=False)
+        if job_ctx is not None and not room.isconnected():
+            await job_ctx.connect()
+        room.on("participant_disconnected", self._on_participant_left)
+        room.on("participant_connected", self._on_participant_joined)
+        agent_session.on("user_input_transcribed", self._on_heard)
+        agent_session.on("agent_state_changed", self._on_agent_state)
+        self._began = self._last_talk = time.monotonic()
+        self._watch_task = asyncio.create_task(self._watch_call())
+
+        await self._say_face("waking")
+        if self._waiting_picture:
+            self._still_task = asyncio.create_task(self._show_waiting_picture())
+        self._face_task = asyncio.create_task(self._bring_face(url, self._pass(room, key, secret)))
+        face_on = asyncio.create_task(self._face_on.wait())
+        try:
+            await asyncio.wait({face_on, self._face_task}, timeout=self._face_first_seconds, return_when=asyncio.FIRST_COMPLETED)
+        finally:
+            face_on.cancel()
+        if not self._face_on.is_set() and not self._face_task.done():
+            await self._say_face("voice")
+
+    async def aclose(self) -> None:
+        """Ends the ChatFX session (billing stops) and leaves the room. Runs by itself when the job shuts down."""
+        if self._closing:
+            return
+        self._closing = True
+        for task in (self._face_task, self._watch_task, self._still_task, self._early_wake):
+            # The watcher itself closes the call when an idle rule is met.
+            if task is not None and task is not asyncio.current_task():
+                await utils.aio.cancel_and_wait(task)
+        self._face_task = self._watch_task = self._still_task = self._early_wake = None
+        if self._still is not None:
+            await self._still.hide()
+            self._still = None
+        if self._room is not None:
+            self._room.off("participant_disconnected", self._on_participant_left)
+            self._room.off("participant_connected", self._on_participant_joined)
+        if self._agent_session is not None:
+            self._agent_session.off("user_input_transcribed", self._on_heard)
+            self._agent_session.off("agent_state_changed", self._on_agent_state)
+        await self._end_session()
+        if self._own_http is not None:
+            await self._own_http.close()
+            self._own_http = None
+        await super().aclose()
+
+    # The pass that lets the avatar join this one room as your agent.
+    def _pass(self, room: rtc.Room, key: str, secret: str) -> str:
+        agent_identity = _agent_identity(room)
+        return (
+            api.AccessToken(api_key=key, api_secret=secret)
+            .with_kind("agent")
+            .with_identity(self._identity)
+            .with_name(self._name)
+            .with_grants(api.VideoGrants(room_join=True, room=room.name))
+            .with_attributes({ATTRIBUTE_PUBLISH_ON_BEHALF: agent_identity})
+            .with_ttl(PASS_TTL)
+            .to_jwt()
+        )
+
+    async def _bring_face(self, livekit_url: str, token: str) -> None:
+        body = {"avatar_id": self._avatar_id, "livekit_url": livekit_url, "livekit_token": token}
+        deadline = time.monotonic() + self._start_timeout
+        while self._in_call() and time.monotonic() < deadline:
+            wait = RETRY_SECONDS
+            try:
+                status, data, wait = await self._call("POST", "/v1/sessions", body, seconds=START_REQUEST_SECONDS)
+            except (aiohttp.ClientError, asyncio.TimeoutError):
+                logger.debug("chatfx avatar api not reachable yet", exc_info=True)
+                status, data = 0, {}
+            if status == 201 and data.get("id"):
+                self._session_id = str(data["id"])
+                self._keepalive_task = asyncio.create_task(self._keep_alive())
+                if await self._wait_for_picture():
+                    logger.info("chatfx avatar on", extra={"session_id": self._session_id, "avatar_id": self._avatar_id})
+                    self._face_on.set()
+                    await self._move_voice_to_face()
+                    await self._say_face("on")
+                    await self._hide_waiting_picture()
+                    return
+                logger.warning("chatfx gpu took the call but sent no picture; trying again")
+                # Ended before asking again, so the try that failed stops counting.
+                await self._end_session()
+            elif status != 0 and status != 503 and status < 500 and not (status == 429 and data.get("error", {}).get("code") == "rate_limited"):
+                error = data.get("error") or {}
+                self._failure = ChatFXException(error.get("message") or f"ChatFX Avatar answered {status}.", code=error.get("code", "error"), status=status)
+                logger.error("chatfx avatar refused: %s (%s); the agent carries on by voice", self._failure, self._failure.code)
+                await self._say_face("off")
+                return
+            await asyncio.sleep(wait)
+        logger.warning("no chatfx face on this call; the agent carries on by voice")
+        await self._say_face("off")
+
+    async def _show_waiting_picture(self, words: bool = True) -> None:
+        try:
+            if self._still is None:
+                async with self._http().get(
+                    f"{self._api_url}/v1/avatars/{self._avatar_id}/picture",
+                    headers={"Authorization": f"Bearer {self._api_key}"},
+                    timeout=aiohttp.ClientTimeout(total=REQUEST_SECONDS),
+                ) as response:
+                    if response.status != 200:
+                        logger.debug("no waiting picture: the api answered %s", response.status)
+                        return
+                    picture = await response.read()
+                if self._face_on.is_set() or self._room is None:
+                    return
+                self._still = _WaitingPicture(self._room, picture, self._waiting_text)
+            await self._still.show(words)
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            logger.debug("couldn't show the waiting picture", exc_info=True)
+
+    async def _hide_waiting_picture(self) -> None:
+        if self._still_task is not None:
+            await utils.aio.cancel_and_wait(self._still_task)
+            self._still_task = None
+        if self._still is not None:
+            await self._still.hide()
+
+    def _in_call(self) -> bool:
+        return not self._closing and self._room is not None and self._room.isconnected()
+
+    async def _wait_for_picture(self) -> bool:
+        ends = time.monotonic() + PICTURE_SECONDS
+        while time.monotonic() < ends and self._in_call():
+            assert self._room is not None
+            avatar = self._room.remote_participants.get(self._identity)
+            if avatar and any(p.kind == rtc.TrackKind.KIND_VIDEO for p in avatar.track_publications.values()):
+                return True
+            await asyncio.sleep(0.1)
+        return False
+
+    # The voice moves between replies, so a sentence is never split between two voices.
+    async def _move_voice_to_face(self) -> None:
+        session = self._agent_session
+        if session is None:
+            return
+        if session.agent_state == "initializing":
+            # Not started yet: move it the moment it starts, before its first words. It has to have started first, so
+            # that LiveKit makes the agent's own voice output, which is what the call falls back on if the face leaves.
+            started = asyncio.Event()
+
+            def on_state(event: Any) -> None:
+                if event.new_state != "initializing":
+                    self._voice_to_face(session)
+                    started.set()
+
+            session.on("agent_state_changed", on_state)
+            try:
+                while not started.is_set() and self._in_call():
+                    try:
+                        await asyncio.wait_for(started.wait(), 0.5)
+                    except asyncio.TimeoutError:
+                        pass
+            finally:
+                session.off("agent_state_changed", on_state)
+            return
+        while session.agent_state == "speaking" and self._in_call():
+            await asyncio.sleep(0.1)
+        if self._in_call():
+            self._voice_to_face(session)
+
+    def _voice_to_face(self, session: AgentSession) -> None:
+        if self._on_face or self._room is None:
+            return
+        self._voice_output = session.output.audio
+        self._face_voice = _FaceVoice(
+            self._room,
+            destination_identity=self._identity,
+            wait_remote_track=rtc.TrackKind.KIND_VIDEO,
+        )
+        session.output.audio = self._face_voice
+        self._on_face = True
+
+    def _on_participant_joined(self, participant: rtc.RemoteParticipant) -> None:
+        if _is_visitor(participant, self._identity):
+            self._visitor_left_at = None
+
+    def _on_heard(self, event: Any) -> None:
+        if event.is_final:
+            self._last_talk = time.monotonic()
+
+    def _on_agent_state(self, event: Any) -> None:
+        if "speaking" in (event.old_state, event.new_state):
+            self._last_talk = time.monotonic()
+        # Cut off by the visitor, the rest of the sound will never play.
+        if event.old_state == "speaking" and self._face_voice is not None:
+            self._face_voice.sound_ends_at = 0.0
+
+    async def _watch_call(self) -> None:
+        while not self._closing:
+            await asyncio.sleep(WATCH_SECONDS)
+            session = self._agent_session
+            if session is None:
+                return
+            now = time.monotonic()
+            voice = self._face_voice
+            if self._on_face and voice is not None and session.agent_state == "speaking" and 0 < voice.sound_ends_at < now - SOUND_GRACE_SECONDS:
+                logger.warning("the chatfx avatar didn't report the end of a reply; ending it so the agent can listen")
+                voice.sound_ends_at = 0.0
+                session.interrupt(force=True)
+            reason = self._idle_reason(session, now)
+            if reason:
+                await self._end_idle_call(reason)
+                return
+
+    def _idle_reason(self, session: AgentSession, now: float) -> str | None:
+        if self._longest_call_seconds is not None and now - self._began >= self._longest_call_seconds:
+            return "it reached its longest length"
+        if self._visitor_gone_seconds is not None and self._visitor_left_at is not None and now - self._visitor_left_at >= self._visitor_gone_seconds:
+            return "the visitor left"
+        if self._quiet_seconds is not None and session.agent_state != "speaking" and now - self._last_talk >= self._quiet_seconds:
+            return "nobody was talking"
+        return None
+
+    async def _end_idle_call(self, reason: str) -> None:
+        logger.info("ending the chatfx avatar call: %s", reason)
+        job_ctx = get_job_context(required=False)
+        session = self._agent_session
+        if not self._end_call_when_idle and session is not None and self._on_face and self._voice_output is not None:
+            session.output.audio = self._voice_output
+            self._on_face = False
+        await self._say_face("off")
+        await self.aclose()
+        if self._end_call_when_idle and job_ctx is not None:
+            job_ctx.delete_room()
+
+    # The face left mid-call (a GPU problem, or the plan's minutes ran out): the call carries on by voice.
+    def _on_participant_left(self, participant: rtc.RemoteParticipant) -> None:
+        if _is_visitor(participant, self._identity):
+            if self._room is not None and not any(_is_visitor(p, self._identity) for p in self._room.remote_participants.values()):
+                self._visitor_left_at = time.monotonic()
+            return
+        if participant.identity != self._identity or self._closing or not self._face_on.is_set():
+            return
+        logger.warning("the chatfx avatar left the call; the agent carries on by voice")
+        session = self._agent_session
+        if session is not None and self._on_face and self._voice_output is not None:
+            session.output.audio = self._voice_output
+        self._on_face = False
+        asyncio.create_task(self._say_face("off"))
+        # Back to the picture, without the words, with the agent's voice over it.
+        if self._waiting_picture:
+            self._still_task = asyncio.create_task(self._show_waiting_picture(words=False))
+
+    async def _say_face(self, status: FaceStatus) -> None:
+        self._face_status = status
+        # No face is coming: the picture stays, without "Video starting".
+        if status == "off" and self._still is not None:
+            self._still.words(False)
+        try:
+            assert self._room is not None
+            await self._room.local_participant.set_attributes({FACE_ATTRIBUTE: status})
+        except Exception:
+            logger.debug("couldn't tell the page where the face is", exc_info=True)
+
+    async def _keep_alive(self) -> None:
+        while self._session_id:
+            await asyncio.sleep(KEEPALIVE_SECONDS)
+            session_id = self._session_id
+            if not session_id:
+                return
+            try:
+                status, data, _ = await self._call("POST", f"/v1/sessions/{session_id}/keepalive")
+            except (aiohttp.ClientError, asyncio.TimeoutError):
+                logger.warning("chatfx check-in didn't go through; trying again", exc_info=True)
+                continue
+            if status != 200:
+                # Ended (409), gone (404) or out of minutes (402): nothing left to keep alive.
+                logger.warning("chatfx avatar session ended", extra={"status": status, "error": data.get("error")})
+                self._session_id = None
+                return
+
+    async def _end_session(self) -> None:
+        if self._keepalive_task is not None:
+            await utils.aio.cancel_and_wait(self._keepalive_task)
+            self._keepalive_task = None
+        session_id, self._session_id = self._session_id, None
+        if session_id:
+            try:
+                await self._call("POST", f"/v1/sessions/{session_id}/end")
+            except Exception:
+                logger.warning("could not end the chatfx session; it ends by itself a minute after its last check-in", exc_info=True)
+
+    async def _call(self, method: str, path: str, body: dict[str, Any] | None = None, seconds: float = REQUEST_SECONDS) -> tuple[int, dict[str, Any], float]:
+        async with self._http().request(
+            method,
+            self._api_url + path,
+            json=body,
+            headers={"Authorization": f"Bearer {self._api_key}"},
+            timeout=aiohttp.ClientTimeout(total=seconds),
+        ) as response:
+            try:
+                data = await response.json(content_type=None)
+            except Exception:
+                data = {}
+            try:
+                retry_after = float(response.headers.get("Retry-After", str(RETRY_SECONDS)))
+            except ValueError:
+                retry_after = RETRY_SECONDS
+            return response.status, data if isinstance(data, dict) else {}, max(0.5, min(retry_after, 30.0))
+
+    def _http(self) -> aiohttp.ClientSession:
+        try:
+            return utils.http_context.http_session()
+        except RuntimeError:
+            # Outside a LiveKit job: our own connection pool, closed by aclose().
+            if self._own_http is None or self._own_http.closed:
+                self._own_http = aiohttp.ClientSession()
+            return self._own_http
+
+
+def _is_visitor(participant: Any, avatar_identity: str) -> bool:
+    return participant.identity != avatar_identity and participant.kind != rtc.ParticipantKind.PARTICIPANT_KIND_AGENT
+
+
+def _agent_identity(room: rtc.Room) -> str:
+    job_ctx = get_job_context(required=False)
+    if job_ctx is not None:
+        return job_ctx.local_participant_identity
+    return room.local_participant.identity

livekit/plugins/chatfx/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.chatfx")

livekit/plugins/chatfx/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

livekit_plugins_chatfx-0.1.0.dist-info/METADATA

@@ -0,0 +1,132 @@
+Metadata-Version: 2.5
+Name: livekit-plugins-chatfx
+Version: 0.1.0
+Summary: ChatFX Avatar for LiveKit Agents: a real-time talking face for your voice agent
+Project-URL: Homepage, https://chatfx.com/avatar/
+Project-URL: Documentation, https://chatfx.com/support/avatar-connect/
+Author-email: "ChatRep, Inc." <service@chatfx.com>
+License-Expression: Apache-2.0
+Keywords: agents,avatar,chatfx,livekit,realtime,voice
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: aiohttp>=3.9
+Requires-Dist: livekit-agents>=1.2
+Requires-Dist: pillow>=10.1
+Description-Content-Type: text/markdown
+
+# ChatFX Avatar for LiveKit Agents
+
+A real-time talking face for your LiveKit voice agent. The avatar joins your agent's LiveKit Cloud room as one more
+participant, takes your agent's voice and speaks it with its lips in time, shown as your agent. You keep your own
+listening, AI and voice; ChatFX adds the face, billed by the minute.
+
+## Install
+
+```bash
+pip install livekit-plugins-chatfx
+```
+
+Requires Python 3.10+ and `livekit-agents` 1.2 or newer.
+
+## Add your key
+
+Create a key under **API keys** at [avatar.chatfx.com](https://avatar.chatfx.com). Then add it to your voice agent's secrets
+at LiveKit, named exactly `CHATFX_AVATAR_API_KEY`. In your agent's folder:
+
+```bash
+lk agent update-secrets --secrets "CHATFX_AVATAR_API_KEY=cfx_..."
+```
+
+Running your agent on your own server instead? Add the key to that server's environment variables, with the same name.
+
+Your agent also needs `LIVEKIT_URL`, `LIVEKIT_API_KEY` and `LIVEKIT_API_SECRET`, which it already has. They never leave your
+server.
+
+## Use it
+
+In your agent's entrypoint, after you make the `AgentSession` and before `session.start()`:
+
+```python
+from livekit.plugins import chatfx
+
+avatar = chatfx.AvatarSession(avatar_id="chatfx-lois")  # copy the id from the Avatars page
+await avatar.start(session, room=ctx.room)
+
+await session.start(agent=MyAgent(), room=ctx.room)
+```
+
+That's all the code. When the job ends, the session ends too, so billing stops. To end it earlier, call
+`await avatar.aclose()`.
+
+## What it does for you
+
+- **Shows a waiting picture, with nothing to add to your page.** The moment a call starts, your agent shows the avatar's picture
+  as its own video, with "Video starting..." on it, and the live face takes over when it's ready. Any page that shows the
+  agent's video gets this.
+- **Greets with the face.** `start()` waits up to 10 seconds for the face, so your agent's first words come with it.
+- **Starts by voice, over the picture, if the face is late.** The voice moves to the face between replies, so a sentence is never split.
+- **Never depends on the face.** If the face can't come, or leaves mid-call, your agent carries on by voice.
+- **Keeps trying** while a GPU wakes or is busy, for up to 5 minutes, and stops at once on a refusal more tries won't fix.
+- **Stops forgotten calls**, so they don't use your minutes: after 2 minutes with nobody talking, 2 minutes after the visitor
+  leaves (long enough for a page reload), or at 1 hour.
+- **Can't go deaf.** If the face's "finished talking" message is late, the plugin ends the reply once the sound has run out,
+  so your agent hears the next question.
+
+## Options
+
+```python
+chatfx.AvatarSession(
+    avatar_id="chatfx-lois",
+    api_key=None,                           # defaults to CHATFX_AVATAR_API_KEY
+    api_url=None,                           # defaults to https://api.chatfx.com
+    avatar_participant_identity="chatfx-avatar",
+    avatar_participant_name="ChatFX",
+    face_first_seconds=10.0,                # how long start() waits for the face
+    start_timeout=300.0,                    # how long to keep trying while a GPU wakes
+    quiet_seconds=120.0,                    # end the call after this long with nobody talking (None: never)
+    visitor_gone_seconds=120.0,             # end the call this long after the last visitor leaves (None: never)
+    longest_call_seconds=3600.0,            # end the call at this length (None: never)
+    end_call_when_idle=True,                # False: only the avatar leaves, and your agent carries on by voice
+    waiting_picture=True,                   # show the avatar's picture until the live face is ready
+    waiting_text="Video starting...",        # the words on it, in your language (None: no words)
+)
+```
+
+`avatar.start()` also takes `livekit_url`, `livekit_api_key` and `livekit_api_secret` if you'd rather not use the settings.
+
+We recommend `"interruption": {"min_words": 1}` in your session's turn handling. The avatar's sound can't pause and carry on,
+so a false alarm would otherwise cut a reply off for good.
+
+## Optional: make the face appear sooner
+
+The plugin starts waking the GPU as soon as a call begins, which takes about 11 seconds. To start even earlier, call `warm()` from your website's server when a visitor
+is about to start a call, and again every 20 seconds while they get ready. It's free. Your website's server then needs the key
+in its environment variables too. Never put the key in a web page.
+
+```python
+await chatfx.warm()
+```
+
+Or without Python:
+
+```bash
+curl -X POST https://api.chatfx.com/v1/warm -H "Authorization: Bearer $CHATFX_AVATAR_API_KEY"
+```
+
+## If you build your own call screen
+
+Your agent's participant attribute `chatfx-face` says where the face is: `waking`, `voice`, `on` or `off`. In the agent,
+`avatar.face_status` says the same. You don't need either to get the waiting picture.
+
+## When there's no face
+
+A refusal doesn't stop the call. The log says why, and `avatar.failure` is a `chatfx.ChatFXException` with the API's `code`
+(for example `invalid_api_key`, `avatar_not_found`, `premium_avatar`, `out_of_minutes`) and a plain message. A missing key or
+missing LiveKit setting raises right away, so you find it while setting up.
+
+## Docs
+
+[Connecting your agent](https://chatfx.com/support/avatar-connect/) · [Building avatars](https://chatfx.com/support/avatar-builder/) · [Minutes and plans](https://chatfx.com/support/avatar-billing/)

livekit_plugins_chatfx-0.1.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+livekit/plugins/chatfx/__init__.py,sha256=6MymqvylS1XFooKMXYnUmgBHXrgFknPGEvbRQ351wIU,1036
+livekit/plugins/chatfx/avatar.py,sha256=kSKvt0Ysn9exBMBywykG1-xUDwoZvtlSWSeJndMKfjU,30930
+livekit/plugins/chatfx/log.py,sha256=rtpkiLFJoPR2ieFPOIpWhpqgYHN525DmVkPxefLilR8,69
+livekit/plugins/chatfx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/chatfx/version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
+livekit_plugins_chatfx-0.1.0.dist-info/METADATA,sha256=r9lOa136glsGOCO1GyOA4M7TPmoCdcVx9hN5KwQnZtA,6193
+livekit_plugins_chatfx-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+livekit_plugins_chatfx-0.1.0.dist-info/RECORD,,

livekit_plugins_chatfx-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any