livekit-plugins-talkingavatar 1.8.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- livekit_plugins_talkingavatar-1.8.11/PKG-INFO +71 -0
- livekit_plugins_talkingavatar-1.8.11/README.md +48 -0
- livekit_plugins_talkingavatar-1.8.11/livekit/plugins/talkingavatar/__init__.py +48 -0
- livekit_plugins_talkingavatar-1.8.11/livekit/plugins/talkingavatar/avatar.py +394 -0
- livekit_plugins_talkingavatar-1.8.11/livekit/plugins/talkingavatar/log.py +3 -0
- livekit_plugins_talkingavatar-1.8.11/livekit/plugins/talkingavatar/py.typed +0 -0
- livekit_plugins_talkingavatar-1.8.11/livekit/plugins/talkingavatar/version.py +15 -0
- livekit_plugins_talkingavatar-1.8.11/pyproject.toml +43 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: livekit-plugins-talkingavatar
|
|
3
|
+
Version: 1.8.11
|
|
4
|
+
Summary: Agent Framework plugin for Talking Avatar realistic avatars
|
|
5
|
+
Project-URL: Documentation, https://docs.talking-avatar.dev/realistic-avatars/livekit
|
|
6
|
+
Project-URL: Website, https://talking-avatar.dev
|
|
7
|
+
Project-URL: Source, https://github.com/Tushar1357/agents/tree/feat/talkingavatar-plugin/livekit-plugins/livekit-plugins-talkingavatar
|
|
8
|
+
Project-URL: Upstream, https://github.com/livekit/agents/pull/8491
|
|
9
|
+
Author-email: Talking Avatar <support@talking-avatar.dev>
|
|
10
|
+
License-Expression: Apache-2.0
|
|
11
|
+
Keywords: ai,audio,livekit,realtime,video,voice,webrtc
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
18
|
+
Classifier: Topic :: Multimedia :: Video
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: >=3.10.0
|
|
21
|
+
Requires-Dist: livekit-agents>=1.8.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# Talking Avatar plugin for LiveKit Agents
|
|
25
|
+
|
|
26
|
+
Support for [Talking Avatar](https://talking-avatar.dev) realistic avatars: a photorealistic avatar that joins the
|
|
27
|
+
room and speaks your agent's voice, with lip sync, from any LLM, TTS or realtime model.
|
|
28
|
+
|
|
29
|
+
See the [Talking Avatar LiveKit docs](https://docs.talking-avatar.dev/realistic-avatars/livekit) for more information.
|
|
30
|
+
|
|
31
|
+
## Installation
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install livekit-plugins-talkingavatar
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Pre-requisites
|
|
38
|
+
|
|
39
|
+
You'll need an API key from the [Talking Avatar dashboard](https://app.talking-avatar.dev). It can be set as an
|
|
40
|
+
environment variable: `TALKING_AVATAR_API_KEY`. The plugin also reads `LIVEKIT_URL`, `LIVEKIT_API_KEY` and
|
|
41
|
+
`LIVEKIT_API_SECRET` to mint the avatar's token.
|
|
42
|
+
|
|
43
|
+
## Usage
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
from livekit.agents import Agent, AgentSession, JobContext
|
|
47
|
+
from livekit.plugins import talkingavatar
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
async def entrypoint(ctx: JobContext):
|
|
51
|
+
session = AgentSession(
|
|
52
|
+
# stt=..., llm=..., tts=... or a realtime model
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
avatar = talkingavatar.AvatarSession(avatar_id="ava4") # any avatar ID from your dashboard
|
|
56
|
+
await avatar.start(session, room=ctx.room)
|
|
57
|
+
|
|
58
|
+
await session.start(agent=Agent(instructions="You are a friendly assistant."), room=ctx.room)
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The avatar joins the room as its own participant and publishes, on behalf of your agent:
|
|
62
|
+
|
|
63
|
+
| Track | Name | Content |
|
|
64
|
+
|---|---|---|
|
|
65
|
+
| Video | `avatar` | The avatar, 832 × 468 at 25 fps |
|
|
66
|
+
| Audio | `avatar-audio` | Your agent's voice, in sync with the lips |
|
|
67
|
+
|
|
68
|
+
Your agent no longer publishes its own audio. Interruptions stop the avatar mid-sentence, and the agent's
|
|
69
|
+
transcript is cut at what the user actually heard.
|
|
70
|
+
|
|
71
|
+
Billing is $0.02 per minute, per second, from the moment the avatar is ready until the session ends.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Talking Avatar plugin for LiveKit Agents
|
|
2
|
+
|
|
3
|
+
Support for [Talking Avatar](https://talking-avatar.dev) realistic avatars: a photorealistic avatar that joins the
|
|
4
|
+
room and speaks your agent's voice, with lip sync, from any LLM, TTS or realtime model.
|
|
5
|
+
|
|
6
|
+
See the [Talking Avatar LiveKit docs](https://docs.talking-avatar.dev/realistic-avatars/livekit) for more information.
|
|
7
|
+
|
|
8
|
+
## Installation
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
pip install livekit-plugins-talkingavatar
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Pre-requisites
|
|
15
|
+
|
|
16
|
+
You'll need an API key from the [Talking Avatar dashboard](https://app.talking-avatar.dev). It can be set as an
|
|
17
|
+
environment variable: `TALKING_AVATAR_API_KEY`. The plugin also reads `LIVEKIT_URL`, `LIVEKIT_API_KEY` and
|
|
18
|
+
`LIVEKIT_API_SECRET` to mint the avatar's token.
|
|
19
|
+
|
|
20
|
+
## Usage
|
|
21
|
+
|
|
22
|
+
```python
|
|
23
|
+
from livekit.agents import Agent, AgentSession, JobContext
|
|
24
|
+
from livekit.plugins import talkingavatar
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
async def entrypoint(ctx: JobContext):
|
|
28
|
+
session = AgentSession(
|
|
29
|
+
# stt=..., llm=..., tts=... or a realtime model
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
avatar = talkingavatar.AvatarSession(avatar_id="ava4") # any avatar ID from your dashboard
|
|
33
|
+
await avatar.start(session, room=ctx.room)
|
|
34
|
+
|
|
35
|
+
await session.start(agent=Agent(instructions="You are a friendly assistant."), room=ctx.room)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
The avatar joins the room as its own participant and publishes, on behalf of your agent:
|
|
39
|
+
|
|
40
|
+
| Track | Name | Content |
|
|
41
|
+
|---|---|---|
|
|
42
|
+
| Video | `avatar` | The avatar, 832 × 468 at 25 fps |
|
|
43
|
+
| Audio | `avatar-audio` | Your agent's voice, in sync with the lips |
|
|
44
|
+
|
|
45
|
+
Your agent no longer publishes its own audio. Interruptions stop the avatar mid-sentence, and the agent's
|
|
46
|
+
transcript is cut at what the user actually heard.
|
|
47
|
+
|
|
48
|
+
Billing is $0.02 per minute, per second, from the moment the avatar is ready until the session ends.
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# Copyright 2026 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Talking Avatar plugin for LiveKit Agents
|
|
16
|
+
|
|
17
|
+
See https://docs.livekit.io/agents/models/avatar/plugins/talkingavatar/ for more information.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from .avatar import AvatarSession, TalkingAvatarException
|
|
21
|
+
from .version import __version__
|
|
22
|
+
|
|
23
|
+
__all__ = [
|
|
24
|
+
"TalkingAvatarException",
|
|
25
|
+
"AvatarSession",
|
|
26
|
+
"__version__",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
from livekit.agents import Plugin
|
|
30
|
+
|
|
31
|
+
from .log import logger
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class TalkingAvatarPlugin(Plugin):
|
|
35
|
+
def __init__(self) -> None:
|
|
36
|
+
super().__init__(__name__, __version__, __package__, logger)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
Plugin.register_plugin(TalkingAvatarPlugin())
|
|
40
|
+
|
|
41
|
+
# Cleanup docs of unexported modules
|
|
42
|
+
_module = dir()
|
|
43
|
+
NOT_IN_ALL = [m for m in _module if m not in __all__]
|
|
44
|
+
|
|
45
|
+
__pdoc__ = {}
|
|
46
|
+
|
|
47
|
+
for n in NOT_IN_ALL:
|
|
48
|
+
__pdoc__[n] = False
|
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import contextlib
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import struct
|
|
8
|
+
import time
|
|
9
|
+
from collections import deque
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import aiohttp
|
|
14
|
+
|
|
15
|
+
from livekit import api, rtc
|
|
16
|
+
from livekit.agents import (
|
|
17
|
+
DEFAULT_API_CONNECT_OPTIONS,
|
|
18
|
+
NOT_GIVEN,
|
|
19
|
+
AgentSession,
|
|
20
|
+
APIConnectionError,
|
|
21
|
+
APIConnectOptions,
|
|
22
|
+
NotGivenOr,
|
|
23
|
+
get_job_context,
|
|
24
|
+
utils,
|
|
25
|
+
)
|
|
26
|
+
from livekit.agents.voice.avatar import AvatarSession as BaseAvatarSession
|
|
27
|
+
from livekit.agents.voice.io import AudioOutput, AudioOutputCapabilities
|
|
28
|
+
from livekit.agents.voice.room_io import ATTRIBUTE_PUBLISH_ON_BEHALF
|
|
29
|
+
|
|
30
|
+
from .log import logger
|
|
31
|
+
|
|
32
|
+
DEFAULT_API_URL = "wss://api.talking-avatar.dev"
|
|
33
|
+
DEFAULT_AVATAR_ID = "ava4"
|
|
34
|
+
SAMPLE_RATE = 24000
|
|
35
|
+
AUDIO_TRACK_NAME = "avatar-audio"
|
|
36
|
+
# the avatar joins the room before it answers "ready", which takes a few seconds
|
|
37
|
+
READY_TIMEOUT = 20.0
|
|
38
|
+
# the avatar reports whole milliseconds per run of audio
|
|
39
|
+
_PLAYED_TOLERANCE = 0.05
|
|
40
|
+
# each audio message: play time (0: queue it after what is queued), sample rate, then PCM16 mono
|
|
41
|
+
_AUDIO_HEADER = struct.Struct("<dI")
|
|
42
|
+
|
|
43
|
+
_AVATAR_AGENT_IDENTITY = "talkingavatar-avatar-agent"
|
|
44
|
+
_AVATAR_AGENT_NAME = "talkingavatar-avatar-agent"
|
|
45
|
+
|
|
46
|
+
_CLOSE_REASONS = {
|
|
47
|
+
1011: "the avatar's stream was lost",
|
|
48
|
+
4001: "invalid or missing API key",
|
|
49
|
+
4002: "out of credits, or the monthly spending limit is reached",
|
|
50
|
+
4003: "the session reached its 3-hour limit",
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class TalkingAvatarException(Exception):
|
|
55
|
+
"""Exception for Talking Avatar errors"""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class AvatarSession(BaseAvatarSession):
|
|
59
|
+
"""A Talking Avatar session: a photorealistic avatar that joins the room and speaks the agent's voice.
|
|
60
|
+
|
|
61
|
+
The agent's audio goes to the avatar over the Talking Avatar realtime API. The avatar publishes its video
|
|
62
|
+
and the voice, in sync with the lips, on behalf of the agent.
|
|
63
|
+
"""
|
|
64
|
+
|
|
65
|
+
def __init__(
|
|
66
|
+
self,
|
|
67
|
+
*,
|
|
68
|
+
avatar_id: NotGivenOr[str] = NOT_GIVEN,
|
|
69
|
+
api_key: NotGivenOr[str] = NOT_GIVEN,
|
|
70
|
+
api_url: NotGivenOr[str] = NOT_GIVEN,
|
|
71
|
+
avatar_participant_identity: NotGivenOr[str] = NOT_GIVEN,
|
|
72
|
+
avatar_participant_name: NotGivenOr[str] = NOT_GIVEN,
|
|
73
|
+
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
|
|
74
|
+
) -> None:
|
|
75
|
+
"""
|
|
76
|
+
Args:
|
|
77
|
+
avatar_id: An avatar ID from the Talking Avatar dashboard, such as ``ava4``. Defaults to the
|
|
78
|
+
``TALKING_AVATAR_AVATAR_ID`` environment variable, then ``ava4``.
|
|
79
|
+
api_key: Your ``tav_`` API key. Defaults to the ``TALKING_AVATAR_API_KEY`` environment variable.
|
|
80
|
+
api_url: The realtime API's URL. Defaults to ``TALKING_AVATAR_API_URL``, then
|
|
81
|
+
``wss://api.talking-avatar.dev``.
|
|
82
|
+
avatar_participant_identity: The avatar participant's identity in the room.
|
|
83
|
+
avatar_participant_name: The avatar participant's name in the room.
|
|
84
|
+
conn_options: Retries and timeouts for connecting to the API.
|
|
85
|
+
"""
|
|
86
|
+
super().__init__()
|
|
87
|
+
self._avatar_id = avatar_id or os.getenv("TALKING_AVATAR_AVATAR_ID") or DEFAULT_AVATAR_ID
|
|
88
|
+
self._api_url = api_url or os.getenv("TALKING_AVATAR_API_URL") or DEFAULT_API_URL
|
|
89
|
+
self._api_key = api_key or os.getenv("TALKING_AVATAR_API_KEY")
|
|
90
|
+
if not self._api_key:
|
|
91
|
+
raise TalkingAvatarException(
|
|
92
|
+
"The api_key must be set either by passing api_key to the client or "
|
|
93
|
+
"by setting the TALKING_AVATAR_API_KEY environment variable"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
self._avatar_participant_identity = avatar_participant_identity or _AVATAR_AGENT_IDENTITY
|
|
97
|
+
self._avatar_participant_name = avatar_participant_name or _AVATAR_AGENT_NAME
|
|
98
|
+
self._conn_options = conn_options
|
|
99
|
+
|
|
100
|
+
self._ws: aiohttp.ClientWebSocketResponse | None = None
|
|
101
|
+
self._own_http_session: aiohttp.ClientSession | None = None
|
|
102
|
+
self._audio_output: _AvatarAudioOutput | None = None
|
|
103
|
+
self._send_ch = utils.aio.Chan[bytes | str]()
|
|
104
|
+
self._tasks: list[asyncio.Task[Any]] = []
|
|
105
|
+
self._close_task: asyncio.Task[None] | None = None
|
|
106
|
+
self._closing = False
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def avatar_identity(self) -> str:
|
|
110
|
+
return self._avatar_participant_identity
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def provider(self) -> str:
|
|
114
|
+
return "talkingavatar"
|
|
115
|
+
|
|
116
|
+
async def start(
|
|
117
|
+
self,
|
|
118
|
+
agent_session: AgentSession,
|
|
119
|
+
room: rtc.Room,
|
|
120
|
+
*,
|
|
121
|
+
livekit_url: NotGivenOr[str] = NOT_GIVEN,
|
|
122
|
+
livekit_api_key: NotGivenOr[str] = NOT_GIVEN,
|
|
123
|
+
livekit_api_secret: NotGivenOr[str] = NOT_GIVEN,
|
|
124
|
+
) -> None:
|
|
125
|
+
await super().start(agent_session, room)
|
|
126
|
+
|
|
127
|
+
livekit_url = livekit_url or (os.getenv("LIVEKIT_URL") or NOT_GIVEN)
|
|
128
|
+
livekit_api_key = livekit_api_key or (os.getenv("LIVEKIT_API_KEY") or NOT_GIVEN)
|
|
129
|
+
livekit_api_secret = livekit_api_secret or (os.getenv("LIVEKIT_API_SECRET") or NOT_GIVEN)
|
|
130
|
+
if not livekit_url or not livekit_api_key or not livekit_api_secret:
|
|
131
|
+
raise TalkingAvatarException(
|
|
132
|
+
"livekit_url, livekit_api_key, and livekit_api_secret must be set "
|
|
133
|
+
"by arguments or environment variables"
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
job_ctx = get_job_context(required=False)
|
|
137
|
+
if job_ctx is not None:
|
|
138
|
+
local_participant_identity = job_ctx.local_participant_identity
|
|
139
|
+
elif room.isconnected():
|
|
140
|
+
local_participant_identity = room.local_participant.identity
|
|
141
|
+
else:
|
|
142
|
+
raise TalkingAvatarException("connect to the room before starting the avatar")
|
|
143
|
+
|
|
144
|
+
livekit_token = (
|
|
145
|
+
api.AccessToken(api_key=livekit_api_key, api_secret=livekit_api_secret)
|
|
146
|
+
.with_kind("agent")
|
|
147
|
+
.with_identity(self._avatar_participant_identity)
|
|
148
|
+
.with_name(self._avatar_participant_name)
|
|
149
|
+
# publish only: the agent's audio reaches the avatar through the Talking Avatar API
|
|
150
|
+
.with_grants(api.VideoGrants(room_join=True, room=room.name, can_subscribe=False))
|
|
151
|
+
# allow the avatar to publish audio and video on behalf of your local agent
|
|
152
|
+
.with_attributes({ATTRIBUTE_PUBLISH_ON_BEHALF: local_participant_identity})
|
|
153
|
+
.to_jwt()
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
logger.debug("starting avatar session")
|
|
157
|
+
self._ws = await self._connect(
|
|
158
|
+
{
|
|
159
|
+
"type": "start",
|
|
160
|
+
"apiKey": self._api_key,
|
|
161
|
+
"avatar": self._avatar_id,
|
|
162
|
+
"livekitUrl": livekit_url,
|
|
163
|
+
"token": livekit_token,
|
|
164
|
+
"audio": {"publish": True, "trackName": AUDIO_TRACK_NAME},
|
|
165
|
+
"sampleRate": SAMPLE_RATE,
|
|
166
|
+
}
|
|
167
|
+
)
|
|
168
|
+
logger.info("Talking Avatar session started", extra={"avatar_id": self._avatar_id})
|
|
169
|
+
|
|
170
|
+
self._audio_output = _AvatarAudioOutput(self._send_ch)
|
|
171
|
+
agent_session.output.replace_audio_tail(self._audio_output)
|
|
172
|
+
agent_session.on("close", self._on_agent_session_close)
|
|
173
|
+
self._tasks = [
|
|
174
|
+
asyncio.create_task(self._send_task(), name="talkingavatar_send"),
|
|
175
|
+
asyncio.create_task(self._recv_task(), name="talkingavatar_recv"),
|
|
176
|
+
]
|
|
177
|
+
|
|
178
|
+
async def aclose(self) -> None:
|
|
179
|
+
"""Ends the avatar session: the avatar leaves the room and billing stops."""
|
|
180
|
+
self._closing = True
|
|
181
|
+
self._send_ch.close()
|
|
182
|
+
if self._ws is not None:
|
|
183
|
+
await self._ws.close() # code 1000
|
|
184
|
+
await utils.aio.cancel_and_wait(*self._tasks)
|
|
185
|
+
if self._own_http_session is not None:
|
|
186
|
+
await self._own_http_session.close()
|
|
187
|
+
self._own_http_session = None
|
|
188
|
+
await super().aclose()
|
|
189
|
+
|
|
190
|
+
def _on_agent_session_close(self, _: Any) -> None:
|
|
191
|
+
if not self._closing and self._close_task is None:
|
|
192
|
+
self._close_task = asyncio.create_task(self.aclose())
|
|
193
|
+
|
|
194
|
+
def _http_session(self) -> aiohttp.ClientSession:
|
|
195
|
+
try:
|
|
196
|
+
return utils.http_context.http_session()
|
|
197
|
+
except RuntimeError: # outside a job (scripts, tests)
|
|
198
|
+
if self._own_http_session is None:
|
|
199
|
+
self._own_http_session = aiohttp.ClientSession()
|
|
200
|
+
return self._own_http_session
|
|
201
|
+
|
|
202
|
+
async def _connect(self, start_msg: dict[str, Any]) -> aiohttp.ClientWebSocketResponse:
|
|
203
|
+
for attempt in range(self._conn_options.max_retry + 1):
|
|
204
|
+
try:
|
|
205
|
+
ws = await asyncio.wait_for(
|
|
206
|
+
self._http_session().ws_connect(self._api_url), self._conn_options.timeout
|
|
207
|
+
)
|
|
208
|
+
try:
|
|
209
|
+
await ws.send_str(json.dumps(start_msg))
|
|
210
|
+
await asyncio.wait_for(_wait_ready(ws), READY_TIMEOUT)
|
|
211
|
+
except BaseException:
|
|
212
|
+
await ws.close()
|
|
213
|
+
raise
|
|
214
|
+
return ws
|
|
215
|
+
except (aiohttp.ClientError, asyncio.TimeoutError) as e:
|
|
216
|
+
logger.warning(
|
|
217
|
+
"failed to start the Talking Avatar session",
|
|
218
|
+
extra={"attempt": attempt + 1, "error": str(e) or type(e).__name__},
|
|
219
|
+
)
|
|
220
|
+
if attempt >= self._conn_options.max_retry:
|
|
221
|
+
raise APIConnectionError(
|
|
222
|
+
"Failed to start the Talking Avatar session after all retries"
|
|
223
|
+
) from e
|
|
224
|
+
await asyncio.sleep(self._conn_options.retry_interval)
|
|
225
|
+
|
|
226
|
+
raise APIConnectionError("Failed to start the Talking Avatar session after all retries")
|
|
227
|
+
|
|
228
|
+
async def _send_task(self) -> None:
|
|
229
|
+
assert self._ws is not None
|
|
230
|
+
with contextlib.suppress(
|
|
231
|
+
ConnectionError, aiohttp.ClientError
|
|
232
|
+
): # closed: _recv_task reports it
|
|
233
|
+
async for data in self._send_ch:
|
|
234
|
+
if isinstance(data, bytes):
|
|
235
|
+
await self._ws.send_bytes(data)
|
|
236
|
+
else:
|
|
237
|
+
await self._ws.send_str(data)
|
|
238
|
+
|
|
239
|
+
async def _recv_task(self) -> None:
|
|
240
|
+
assert self._ws is not None and self._audio_output is not None
|
|
241
|
+
try:
|
|
242
|
+
async for msg in self._ws:
|
|
243
|
+
if msg.type != aiohttp.WSMsgType.TEXT:
|
|
244
|
+
continue
|
|
245
|
+
event = json.loads(msg.data)
|
|
246
|
+
if event.get("type") == "error":
|
|
247
|
+
logger.error(f"Talking Avatar error: {event.get('message')}")
|
|
248
|
+
else:
|
|
249
|
+
self._audio_output._on_event(event)
|
|
250
|
+
finally:
|
|
251
|
+
self._audio_output._on_closed()
|
|
252
|
+
if not self._closing:
|
|
253
|
+
code = self._ws.close_code
|
|
254
|
+
logger.error(
|
|
255
|
+
f"Talking Avatar session ended unexpectedly: {_CLOSE_REASONS.get(code or 0, code)}",
|
|
256
|
+
extra={"code": code},
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
async def _wait_ready(ws: aiohttp.ClientWebSocketResponse) -> None:
|
|
261
|
+
error: str | None = None
|
|
262
|
+
async for msg in ws:
|
|
263
|
+
if msg.type != aiohttp.WSMsgType.TEXT:
|
|
264
|
+
continue
|
|
265
|
+
event = json.loads(msg.data)
|
|
266
|
+
if event.get("type") == "ready":
|
|
267
|
+
return
|
|
268
|
+
if event.get("type") == "error":
|
|
269
|
+
error = str(event.get("message"))
|
|
270
|
+
|
|
271
|
+
code = ws.close_code
|
|
272
|
+
if error or code in (4001, 4002):
|
|
273
|
+
# refused (API key, credits, unknown avatar, LiveKit join failed...): a retry fails the same way
|
|
274
|
+
raise TalkingAvatarException(error or _CLOSE_REASONS[code or 0])
|
|
275
|
+
raise aiohttp.ClientError(f"connection closed before the avatar was ready ({code})")
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
@dataclass
|
|
279
|
+
class _Segment:
|
|
280
|
+
pushed: float = 0.0 # seconds of audio sent
|
|
281
|
+
heard: float = 0.0 # seconds the avatar reported as played
|
|
282
|
+
flushed: bool = False
|
|
283
|
+
started: bool = False
|
|
284
|
+
clearing: bool = False
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
class _AvatarAudioOutput(AudioOutput):
|
|
288
|
+
"""Sends the agent's audio to the avatar, which plays it in the room in sync with the lips.
|
|
289
|
+
|
|
290
|
+
The avatar reports playback per run of back-to-back audio: ``speech_started``, then ``speech_ended`` with
|
|
291
|
+
the milliseconds played once its queue runs dry (also mid-segment, when the TTS falls behind), or ``cleared``
|
|
292
|
+
after an interruption. Those milliseconds are credited to segments in order, and a segment finishes once
|
|
293
|
+
it is flushed and fully heard.
|
|
294
|
+
"""
|
|
295
|
+
|
|
296
|
+
def __init__(self, send_ch: utils.aio.Chan[bytes | str]) -> None:
|
|
297
|
+
super().__init__(
|
|
298
|
+
label="TalkingAvatar",
|
|
299
|
+
capabilities=AudioOutputCapabilities(pause=False),
|
|
300
|
+
sample_rate=SAMPLE_RATE, # the agent session resamples to it
|
|
301
|
+
)
|
|
302
|
+
self._send_ch = send_ch
|
|
303
|
+
self._segments: deque[_Segment] = deque() # captured and not finished, oldest first
|
|
304
|
+
self._current: _Segment | None = None # being captured
|
|
305
|
+
self._clears: deque[list[_Segment]] = deque() # per clear sent, the segments it cuts
|
|
306
|
+
self._closed = False
|
|
307
|
+
|
|
308
|
+
async def capture_frame(self, frame: rtc.AudioFrame) -> None:
|
|
309
|
+
await super().capture_frame(frame)
|
|
310
|
+
if self._current is None:
|
|
311
|
+
self._current = _Segment()
|
|
312
|
+
self._segments.append(self._current)
|
|
313
|
+
if self._closed or not frame.samples_per_channel:
|
|
314
|
+
return
|
|
315
|
+
self._current.pushed += frame.duration
|
|
316
|
+
self._send(_AUDIO_HEADER.pack(0, frame.sample_rate) + frame.data.tobytes())
|
|
317
|
+
|
|
318
|
+
def flush(self) -> None:
|
|
319
|
+
super().flush()
|
|
320
|
+
if self._current is None:
|
|
321
|
+
return
|
|
322
|
+
self._current.flushed = True
|
|
323
|
+
self._current = None
|
|
324
|
+
if self._closed:
|
|
325
|
+
self._finish_all()
|
|
326
|
+
else:
|
|
327
|
+
self._finish_heard()
|
|
328
|
+
|
|
329
|
+
def clear_buffer(self) -> None:
|
|
330
|
+
self._current = None
|
|
331
|
+
if self._closed:
|
|
332
|
+
self._finish_all()
|
|
333
|
+
return
|
|
334
|
+
cut = [s for s in self._segments if not s.clearing]
|
|
335
|
+
if not cut: # nothing playing or queued (the session also clears after it has)
|
|
336
|
+
return
|
|
337
|
+
for s in cut:
|
|
338
|
+
s.clearing = True
|
|
339
|
+
self._clears.append(cut)
|
|
340
|
+
self._send(json.dumps({"type": "clear"}))
|
|
341
|
+
|
|
342
|
+
def _send(self, data: bytes | str) -> None:
|
|
343
|
+
with contextlib.suppress(utils.aio.ChanClosed):
|
|
344
|
+
self._send_ch.send_nowait(data)
|
|
345
|
+
|
|
346
|
+
def _on_event(self, event: dict[str, Any]) -> None:
|
|
347
|
+
kind = event.get("type")
|
|
348
|
+
if kind == "speech_started":
|
|
349
|
+
if self._segments and not self._segments[0].started:
|
|
350
|
+
self._segments[0].started = True
|
|
351
|
+
self.on_playback_started(created_at=time.time())
|
|
352
|
+
elif kind == "speech_ended":
|
|
353
|
+
self._credit(list(self._segments), event.get("playedMs", 0) / 1000)
|
|
354
|
+
self._finish_heard()
|
|
355
|
+
elif kind == "cleared":
|
|
356
|
+
cut = self._clears.popleft() if self._clears else []
|
|
357
|
+
self._credit(cut, event.get("playedMs", 0) / 1000)
|
|
358
|
+
for seg in cut:
|
|
359
|
+
self._finish(seg, interrupted=True)
|
|
360
|
+
|
|
361
|
+
def _on_closed(self) -> None:
|
|
362
|
+
self._closed = True
|
|
363
|
+
self._finish_all()
|
|
364
|
+
|
|
365
|
+
@staticmethod
|
|
366
|
+
def _credit(segments: list[_Segment], played: float) -> None:
|
|
367
|
+
for seg in segments:
|
|
368
|
+
take = min(played, seg.pushed - seg.heard)
|
|
369
|
+
if take > 0:
|
|
370
|
+
seg.heard += take
|
|
371
|
+
played -= take
|
|
372
|
+
|
|
373
|
+
def _finish_heard(self) -> None:
|
|
374
|
+
while self._segments:
|
|
375
|
+
seg = self._segments[0]
|
|
376
|
+
if seg.heard > 0 and not seg.started:
|
|
377
|
+
# it played back to back with the segment before it, in the same run, so the
|
|
378
|
+
# avatar sent no speech_started of its own: it started `heard` seconds before now
|
|
379
|
+
seg.started = True
|
|
380
|
+
self.on_playback_started(created_at=time.time() - seg.heard)
|
|
381
|
+
if seg.clearing or not seg.flushed or seg.heard < seg.pushed - _PLAYED_TOLERANCE:
|
|
382
|
+
return
|
|
383
|
+
self._finish(seg, interrupted=False)
|
|
384
|
+
|
|
385
|
+
def _finish_all(self) -> None:
|
|
386
|
+
self._clears.clear()
|
|
387
|
+
for seg in list(self._segments):
|
|
388
|
+
if seg is not self._current:
|
|
389
|
+
self._finish(seg, interrupted=True)
|
|
390
|
+
|
|
391
|
+
def _finish(self, seg: _Segment, *, interrupted: bool) -> None:
|
|
392
|
+
self._segments.remove(seg)
|
|
393
|
+
position = seg.pushed if not interrupted else min(seg.heard, seg.pushed)
|
|
394
|
+
self.on_playback_finished(playback_position=position, interrupted=interrupted)
|
|
File without changes
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Copyright 2026 LiveKit, Inc.
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
__version__ = "1.8.11"
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "livekit-plugins-talkingavatar"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Agent Framework plugin for Talking Avatar realistic avatars"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "Apache-2.0"
|
|
11
|
+
requires-python = ">=3.10.0"
|
|
12
|
+
authors = [{ name = "Talking Avatar", email = "support@talking-avatar.dev" }]
|
|
13
|
+
keywords = ["voice", "ai", "realtime", "audio", "video", "livekit", "webrtc"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"License :: OSI Approved :: Apache Software License",
|
|
17
|
+
"Topic :: Multimedia :: Sound/Audio",
|
|
18
|
+
"Topic :: Multimedia :: Video",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.10",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
]
|
|
24
|
+
dependencies = ["livekit-agents>=1.8.11"]
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Documentation = "https://docs.talking-avatar.dev/realistic-avatars/livekit"
|
|
28
|
+
Website = "https://talking-avatar.dev"
|
|
29
|
+
Source = "https://github.com/Tushar1357/agents/tree/feat/talkingavatar-plugin/livekit-plugins/livekit-plugins-talkingavatar"
|
|
30
|
+
Upstream = "https://github.com/livekit/agents/pull/8491"
|
|
31
|
+
|
|
32
|
+
[tool.hatch.version]
|
|
33
|
+
path = "livekit/plugins/talkingavatar/version.py"
|
|
34
|
+
|
|
35
|
+
[tool.hatch.build.targets.wheel]
|
|
36
|
+
packages = ["livekit"]
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.targets.sdist]
|
|
39
|
+
include = ["/livekit"]
|
|
40
|
+
|
|
41
|
+
[tool.uv]
|
|
42
|
+
exclude-newer = "7 days"
|
|
43
|
+
exclude-newer-package = { livekit-agents = "0 days" }
|