livekit-plugins-avarole 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- livekit/plugins/avarole/__init__.py +31 -0
- livekit/plugins/avarole/avatar.py +179 -0
- livekit/plugins/avarole/client.py +536 -0
- livekit/plugins/avarole/log.py +3 -0
- livekit/plugins/avarole/py.typed +0 -0
- livekit/plugins/avarole/version.py +1 -0
- livekit_plugins_avarole-0.1.0.dist-info/METADATA +161 -0
- livekit_plugins_avarole-0.1.0.dist-info/RECORD +10 -0
- livekit_plugins_avarole-0.1.0.dist-info/WHEEL +4 -0
- livekit_plugins_avarole-0.1.0.dist-info/licenses/LICENSE +204 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Avarole plugin for LiveKit Agents.
|
|
2
|
+
|
|
3
|
+
from livekit.plugins import avarole
|
|
4
|
+
|
|
5
|
+
session = avarole.AvatarSession(agent_id="...", api_key="...")
|
|
6
|
+
await session.start(ctx.room) # the avatar joins your room and talks to whoever is in it
|
|
7
|
+
|
|
8
|
+
``AvaroleClient`` is the same call without LiveKit, for any app that can supply 16 kHz microphone audio.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from livekit.agents import Plugin
|
|
12
|
+
|
|
13
|
+
from .avatar import AvatarSession
|
|
14
|
+
from .client import Action, AvaroleClient, AvaroleError, LiveKitJoin, ScreenInfo
|
|
15
|
+
from .log import logger
|
|
16
|
+
from .version import __version__
|
|
17
|
+
|
|
18
|
+
__all__ = ["Action", "AvaroleClient", "AvaroleError", "AvatarSession", "LiveKitJoin", "ScreenInfo", "__version__"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AvarolePlugin(Plugin):
|
|
22
|
+
def __init__(self) -> None:
|
|
23
|
+
super().__init__(__name__, __version__, __package__, logger)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
Plugin.register_plugin(AvarolePlugin())
|
|
27
|
+
|
|
28
|
+
# Hide the internals from `help()` and autocompletion.
|
|
29
|
+
__pdoc__ = {"AvarolePlugin": False}
|
|
30
|
+
for _n in [n for n in dir() if not n.startswith("_") and n not in __all__]:
|
|
31
|
+
__pdoc__[_n] = False
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""An Avarole avatar inside your LiveKit room.
|
|
2
|
+
|
|
3
|
+
Avarole runs the whole conversation (hearing, thinking, speaking, its knowledge base and tools) and renders the avatar
|
|
4
|
+
into its own LiveKit room. ``AvatarSession`` is the bridge between that room and yours:
|
|
5
|
+
|
|
6
|
+
people in your room --(their microphone, 16 kHz PCM)--> Avarole
|
|
7
|
+
Avarole's avatar --(video + voice)-----------------> published into your room as one participant
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
from typing import Callable, Optional
|
|
14
|
+
|
|
15
|
+
from livekit import rtc
|
|
16
|
+
|
|
17
|
+
from .client import SAMPLE_RATE, Action, AvaroleClient, ScreenInfo
|
|
18
|
+
from .log import logger
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AvatarSession:
|
|
22
|
+
"""Bring an Avarole avatar into a LiveKit room.
|
|
23
|
+
|
|
24
|
+
session = AvatarSession(agent_id="...", api_key="...")
|
|
25
|
+
await session.start(ctx.room)
|
|
26
|
+
|
|
27
|
+
The avatar greets people, listens to every person in the room (``listen_to`` narrows who), speaks, shows captions through
|
|
28
|
+
``session.client.on("caption", ...)`` and can call the ``actions`` you give it. ``await session.aclose()`` ends
|
|
29
|
+
the call; it also ends when the room closes or the plan's session time runs out.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
agent_id: str,
|
|
35
|
+
api_key: Optional[str] = None,
|
|
36
|
+
*,
|
|
37
|
+
actions: Optional[list] = None,
|
|
38
|
+
client: Optional[AvaroleClient] = None,
|
|
39
|
+
listen_to: Optional[Callable[[rtc.RemoteParticipant], bool]] = None,
|
|
40
|
+
**client_kwargs,
|
|
41
|
+
) -> None:
|
|
42
|
+
self.client = client or AvaroleClient(agent_id, api_key, actions=actions, **client_kwargs)
|
|
43
|
+
# By default Avarole hears every person in the room, and no other agent (so two agents never talk to each other).
|
|
44
|
+
self._listen_to = listen_to or (lambda p: p.kind != rtc.ParticipantKind.PARTICIPANT_KIND_AGENT)
|
|
45
|
+
self._room: Optional[rtc.Room] = None
|
|
46
|
+
self._avatar_room: Optional[rtc.Room] = None
|
|
47
|
+
self._tasks: set = set()
|
|
48
|
+
self._audio_source: Optional[rtc.AudioSource] = None
|
|
49
|
+
self._video_source: Optional[rtc.VideoSource] = None
|
|
50
|
+
self._heard: set = set()
|
|
51
|
+
self._closed = False
|
|
52
|
+
self._first_video = asyncio.Event()
|
|
53
|
+
|
|
54
|
+
# ------------------------------------------------------------- lifecycle
|
|
55
|
+
|
|
56
|
+
async def start(self, room: rtc.Room) -> None:
|
|
57
|
+
"""Start the call and put the avatar in ``room`` (a connected ``rtc.Room``, e.g. ``ctx.room``)."""
|
|
58
|
+
self._room = room
|
|
59
|
+
self._closed = False
|
|
60
|
+
self.client.on("expired", lambda _m: self._spawn(self.aclose()))
|
|
61
|
+
self.client.on("end", lambda _v: self._spawn(self.aclose()))
|
|
62
|
+
|
|
63
|
+
await self.client.start()
|
|
64
|
+
join = await self.client.livekit_join()
|
|
65
|
+
|
|
66
|
+
room.on("track_subscribed", self._on_user_track)
|
|
67
|
+
for participant in room.remote_participants.values():
|
|
68
|
+
for pub in participant.track_publications.values():
|
|
69
|
+
if pub.track is not None:
|
|
70
|
+
self._on_user_track(pub.track, pub, participant)
|
|
71
|
+
|
|
72
|
+
self._avatar_room = rtc.Room()
|
|
73
|
+
self._avatar_room.on("track_subscribed", self._on_avatar_track)
|
|
74
|
+
await self._avatar_room.connect(join.url, join.token, options=rtc.RoomOptions(auto_subscribe=True))
|
|
75
|
+
# The track may already be live when we join: pick it up too.
|
|
76
|
+
for participant in self._avatar_room.remote_participants.values():
|
|
77
|
+
for pub in participant.track_publications.values():
|
|
78
|
+
if pub.track is not None:
|
|
79
|
+
self._on_avatar_track(pub.track, pub, participant)
|
|
80
|
+
room.on("disconnected", lambda *_: self._spawn(self.aclose()))
|
|
81
|
+
|
|
82
|
+
async def aclose(self) -> None:
|
|
83
|
+
"""End the call and take the avatar out of the room. Safe to call twice."""
|
|
84
|
+
if self._closed:
|
|
85
|
+
return
|
|
86
|
+
self._closed = True
|
|
87
|
+
await self.client.stop()
|
|
88
|
+
for task in list(self._tasks):
|
|
89
|
+
if task is not asyncio.current_task():
|
|
90
|
+
task.cancel()
|
|
91
|
+
if self._avatar_room is not None:
|
|
92
|
+
try:
|
|
93
|
+
await self._avatar_room.disconnect()
|
|
94
|
+
except Exception:
|
|
95
|
+
logger.debug("avarole: avatar room disconnect failed", exc_info=True)
|
|
96
|
+
self._avatar_room = None
|
|
97
|
+
self._audio_source = self._video_source = None
|
|
98
|
+
|
|
99
|
+
# -------------------------------------------------- what you can ask for
|
|
100
|
+
|
|
101
|
+
def register_actions(self, actions: list[Action]) -> dict:
|
|
102
|
+
"""Offer actions the avatar may call. Same as ``client.register_actions``."""
|
|
103
|
+
return self.client.register_actions(actions)
|
|
104
|
+
|
|
105
|
+
def set_screen(self, info: Optional[ScreenInfo]) -> None:
|
|
106
|
+
"""Tell the avatar what the user is looking at. Same as ``client.set_screen``."""
|
|
107
|
+
self.client.set_screen(info)
|
|
108
|
+
|
|
109
|
+
# ----------------------------------------------------- user -> Avarole
|
|
110
|
+
|
|
111
|
+
def _on_user_track(
|
|
112
|
+
self, track: rtc.Track, pub: rtc.RemoteTrackPublication, participant: rtc.RemoteParticipant
|
|
113
|
+
) -> None:
|
|
114
|
+
if track.kind != rtc.TrackKind.KIND_AUDIO or not self._listen_to(participant):
|
|
115
|
+
return
|
|
116
|
+
if (participant.identity, track.sid) in self._heard:
|
|
117
|
+
return
|
|
118
|
+
self._heard.add((participant.identity, track.sid))
|
|
119
|
+
self._spawn(self._forward_user_audio(track))
|
|
120
|
+
|
|
121
|
+
async def _forward_user_audio(self, track: rtc.Track) -> None:
|
|
122
|
+
stream = rtc.AudioStream(track, sample_rate=SAMPLE_RATE, num_channels=1)
|
|
123
|
+
try:
|
|
124
|
+
async for ev in stream:
|
|
125
|
+
await self.client.send_audio(ev.frame.data.tobytes())
|
|
126
|
+
finally:
|
|
127
|
+
await stream.aclose()
|
|
128
|
+
|
|
129
|
+
# --------------------------------------------------- Avarole -> your room
|
|
130
|
+
|
|
131
|
+
def _on_avatar_track(
|
|
132
|
+
self, track: rtc.Track, pub: rtc.RemoteTrackPublication, participant: rtc.RemoteParticipant
|
|
133
|
+
) -> None:
|
|
134
|
+
if track.kind == rtc.TrackKind.KIND_AUDIO:
|
|
135
|
+
self._spawn(self._republish_audio(track))
|
|
136
|
+
elif track.kind == rtc.TrackKind.KIND_VIDEO:
|
|
137
|
+
self._spawn(self._republish_video(track))
|
|
138
|
+
|
|
139
|
+
async def _publish(self, track: rtc.LocalTrack, source: rtc.TrackSource) -> None:
|
|
140
|
+
if self._room is not None and self._room.isconnected():
|
|
141
|
+
await self._room.local_participant.publish_track(track, rtc.TrackPublishOptions(source=source))
|
|
142
|
+
|
|
143
|
+
async def _republish_audio(self, track: rtc.Track) -> None:
|
|
144
|
+
stream = rtc.AudioStream(track)
|
|
145
|
+
try:
|
|
146
|
+
async for ev in stream:
|
|
147
|
+
frame = ev.frame
|
|
148
|
+
if self._audio_source is None:
|
|
149
|
+
self._audio_source = rtc.AudioSource(frame.sample_rate, frame.num_channels)
|
|
150
|
+
await self._publish(
|
|
151
|
+
rtc.LocalAudioTrack.create_audio_track("avarole-voice", self._audio_source),
|
|
152
|
+
rtc.TrackSource.SOURCE_MICROPHONE,
|
|
153
|
+
)
|
|
154
|
+
await self._audio_source.capture_frame(frame)
|
|
155
|
+
finally:
|
|
156
|
+
await stream.aclose()
|
|
157
|
+
|
|
158
|
+
async def _republish_video(self, track: rtc.Track) -> None:
|
|
159
|
+
stream = rtc.VideoStream(track)
|
|
160
|
+
try:
|
|
161
|
+
async for ev in stream:
|
|
162
|
+
frame = ev.frame
|
|
163
|
+
if self._video_source is None:
|
|
164
|
+
self._video_source = rtc.VideoSource(frame.width, frame.height)
|
|
165
|
+
await self._publish(
|
|
166
|
+
rtc.LocalVideoTrack.create_video_track("avarole-avatar", self._video_source),
|
|
167
|
+
rtc.TrackSource.SOURCE_CAMERA,
|
|
168
|
+
)
|
|
169
|
+
self._video_source.capture_frame(frame)
|
|
170
|
+
if not self._first_video.is_set():
|
|
171
|
+
self._first_video.set()
|
|
172
|
+
await self.client.viewer_ready() # Avarole greets once the avatar can be seen
|
|
173
|
+
finally:
|
|
174
|
+
await stream.aclose()
|
|
175
|
+
|
|
176
|
+
def _spawn(self, coro) -> None:
|
|
177
|
+
task = asyncio.ensure_future(coro)
|
|
178
|
+
self._tasks.add(task)
|
|
179
|
+
task.add_done_callback(self._tasks.discard)
|
|
@@ -0,0 +1,536 @@
|
|
|
1
|
+
"""The Avarole call, with no LiveKit in it: microphone audio in, captions and actions out.
|
|
2
|
+
|
|
3
|
+
microphone -> 60 ms PCM frames -> websocket /stream (Avarole hears the user)
|
|
4
|
+
websocket events (captions, status, limits, actions) (what the assistant says and asks your app to do)
|
|
5
|
+
LiveKit room named after the session (the avatar's video and voice, see ``AvatarSession``)
|
|
6
|
+
|
|
7
|
+
This is the same protocol the Avarole web widget and the React Native SDK use.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import json
|
|
14
|
+
import re
|
|
15
|
+
import secrets
|
|
16
|
+
import time
|
|
17
|
+
from collections.abc import Awaitable
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from typing import Any, Callable, Optional, Union
|
|
20
|
+
from urllib.parse import quote, urlencode
|
|
21
|
+
|
|
22
|
+
import aiohttp
|
|
23
|
+
|
|
24
|
+
from .log import logger
|
|
25
|
+
|
|
26
|
+
DEFAULT_APP_URL = "https://app.avarole.com"
|
|
27
|
+
DEFAULT_SERVICE_URL = "https://ai.avarole.com:8080"
|
|
28
|
+
FRAME_BYTES = 1920 # 60 ms of 16 kHz mono 16-bit audio: what the call service reads
|
|
29
|
+
SAMPLE_RATE = 16000
|
|
30
|
+
MAX_ACTIONS = 30
|
|
31
|
+
MAX_RECONNECTS = 3
|
|
32
|
+
ACTION_TIMEOUT_S = 25.0
|
|
33
|
+
MAX_INFLIGHT_ACTIONS = 3
|
|
34
|
+
MAX_ACTIONS_PER_MINUTE = 30
|
|
35
|
+
MAX_RESULT_CHARS = 2500
|
|
36
|
+
_ACTION_NAME = re.compile(r"^[a-z][a-z0-9_]{0,39}$")
|
|
37
|
+
|
|
38
|
+
EVENTS = (
|
|
39
|
+
"status",
|
|
40
|
+
"session",
|
|
41
|
+
"agent_status",
|
|
42
|
+
"transcript",
|
|
43
|
+
"caption",
|
|
44
|
+
"emotion",
|
|
45
|
+
"interrupt",
|
|
46
|
+
"limit",
|
|
47
|
+
"ending",
|
|
48
|
+
"expired",
|
|
49
|
+
"error",
|
|
50
|
+
"end",
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class AvaroleError(Exception):
|
|
55
|
+
"""The call could not be made or was refused."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@dataclass
|
|
59
|
+
class LiveKitJoin:
|
|
60
|
+
url: str
|
|
61
|
+
token: str
|
|
62
|
+
room: str
|
|
63
|
+
session_id: str
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class ScreenInfo:
|
|
68
|
+
"""Where the user is in your app. Labels only: never put what they typed here."""
|
|
69
|
+
|
|
70
|
+
name: str
|
|
71
|
+
summary: str = ""
|
|
72
|
+
fields: list = field(default_factory=list) # [{"id", "label", "type", "filled"}]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class Action:
|
|
77
|
+
"""Something your app can do that the assistant may call (it sees it as ``app_<name>``).
|
|
78
|
+
|
|
79
|
+
``confirm=True`` asks the user first (via the ``confirm`` callback of the client); without one it is refused.
|
|
80
|
+
``handler`` receives the arguments the assistant chose, so validate them like any user input.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
name: str
|
|
84
|
+
description: str
|
|
85
|
+
handler: Callable[[dict], Union[Any, Awaitable[Any]]]
|
|
86
|
+
params: dict = field(default_factory=lambda: {"type": "object", "properties": {}})
|
|
87
|
+
confirm: bool = False
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _result(ok: bool, status: str, **extra: Any) -> dict:
|
|
91
|
+
return {"ok": ok, "status": status, **extra}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class AvaroleClient:
|
|
95
|
+
"""One call with an Avarole assistant.
|
|
96
|
+
|
|
97
|
+
``agent_id`` and ``api_key`` come from the Avarole dashboard (Integrations). Use it as an async context manager,
|
|
98
|
+
or call ``start()`` / ``stop()`` yourself::
|
|
99
|
+
|
|
100
|
+
async with AvaroleClient(agent_id="...", api_key="...") as call:
|
|
101
|
+
call.on("caption", print)
|
|
102
|
+
await call.send_audio(pcm_16khz_mono)
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
def __init__(
|
|
106
|
+
self,
|
|
107
|
+
agent_id: str,
|
|
108
|
+
api_key: Optional[str] = None,
|
|
109
|
+
*,
|
|
110
|
+
app_url: str = DEFAULT_APP_URL,
|
|
111
|
+
service_url: str = DEFAULT_SERVICE_URL,
|
|
112
|
+
user_id: Optional[str] = None,
|
|
113
|
+
contact: Optional[dict] = None,
|
|
114
|
+
confirm: Optional[Callable[[str, str], Union[bool, Awaitable[bool]]]] = None,
|
|
115
|
+
actions: Optional[list] = None,
|
|
116
|
+
http_session: Optional[aiohttp.ClientSession] = None,
|
|
117
|
+
) -> None:
|
|
118
|
+
if not agent_id:
|
|
119
|
+
raise ValueError("agent_id is required")
|
|
120
|
+
self.agent_id = agent_id
|
|
121
|
+
self.api_key = api_key
|
|
122
|
+
self.app_url = app_url.rstrip("/")
|
|
123
|
+
self.service_url = service_url.rstrip("/")
|
|
124
|
+
self.user_id = user_id
|
|
125
|
+
self.contact = contact or {}
|
|
126
|
+
self._confirm = confirm
|
|
127
|
+
self.status = "idle" # idle | connecting | live | reconnecting | ended | error
|
|
128
|
+
self.session_id = ""
|
|
129
|
+
self._http = http_session
|
|
130
|
+
self._own_http = http_session is None
|
|
131
|
+
self._ws: Optional[aiohttp.ClientWebSocketResponse] = None
|
|
132
|
+
self._reader: Optional[asyncio.Task] = None
|
|
133
|
+
self._tasks: set = set()
|
|
134
|
+
self._listeners: dict = {}
|
|
135
|
+
self._actions: dict = {}
|
|
136
|
+
self._screen: Optional[dict] = None
|
|
137
|
+
self._buf = bytearray()
|
|
138
|
+
self._muted = False
|
|
139
|
+
self._intentional = True
|
|
140
|
+
self._attempts = 0
|
|
141
|
+
self._inflight = 0
|
|
142
|
+
self._recent: list = []
|
|
143
|
+
self._live = asyncio.Event()
|
|
144
|
+
if actions:
|
|
145
|
+
self.register_actions(actions)
|
|
146
|
+
|
|
147
|
+
# ---------------------------------------------------------------- events
|
|
148
|
+
|
|
149
|
+
def on(self, event: str, fn: Callable[[Any], Any]) -> Callable[[], None]:
|
|
150
|
+
"""Listen for one of ``EVENTS``. Returns a function that stops listening.
|
|
151
|
+
|
|
152
|
+
``caption`` is what the assistant is saying, ``transcript`` what the user said, ``agent_status`` what it is
|
|
153
|
+
doing ("looking that up"), ``limit``/``ending``/``expired`` the session clock, ``interrupt`` a barge-in."""
|
|
154
|
+
if event not in EVENTS:
|
|
155
|
+
raise ValueError("unknown event %r, expected one of %s" % (event, ", ".join(EVENTS)))
|
|
156
|
+
self._listeners.setdefault(event, []).append(fn)
|
|
157
|
+
return lambda: self._listeners.get(event, []).remove(fn) if fn in self._listeners.get(event, []) else None
|
|
158
|
+
|
|
159
|
+
def _emit(self, event: str, value: Any = None) -> None:
|
|
160
|
+
for fn in list(self._listeners.get(event, [])):
|
|
161
|
+
try:
|
|
162
|
+
out = fn(value)
|
|
163
|
+
if asyncio.iscoroutine(out):
|
|
164
|
+
self._spawn(out)
|
|
165
|
+
except Exception: # a listener must never break the call
|
|
166
|
+
logger.exception("avarole: a %s listener raised", event)
|
|
167
|
+
|
|
168
|
+
def _set_status(self, status: str) -> None:
|
|
169
|
+
if self.status != status:
|
|
170
|
+
self.status = status
|
|
171
|
+
self._emit("status", status)
|
|
172
|
+
|
|
173
|
+
def _spawn(self, coro: Awaitable) -> None:
|
|
174
|
+
task = asyncio.ensure_future(coro)
|
|
175
|
+
self._tasks.add(task)
|
|
176
|
+
task.add_done_callback(self._tasks.discard)
|
|
177
|
+
|
|
178
|
+
# ------------------------------------------------------------------ call
|
|
179
|
+
|
|
180
|
+
async def __aenter__(self) -> AvaroleClient:
|
|
181
|
+
await self.start()
|
|
182
|
+
return self
|
|
183
|
+
|
|
184
|
+
async def __aexit__(self, *exc: object) -> None:
|
|
185
|
+
await self.stop()
|
|
186
|
+
|
|
187
|
+
def _http_session(self) -> aiohttp.ClientSession:
|
|
188
|
+
if self._http is None:
|
|
189
|
+
self._http = aiohttp.ClientSession()
|
|
190
|
+
return self._http
|
|
191
|
+
|
|
192
|
+
def _ws_url(self) -> str:
|
|
193
|
+
q = {"session_id": self.session_id, "agent_id": self.agent_id, "source": "sdk"}
|
|
194
|
+
if self.api_key:
|
|
195
|
+
q["api_key"] = self.api_key
|
|
196
|
+
if self.user_id:
|
|
197
|
+
q["user_id"] = self.user_id
|
|
198
|
+
if self.contact.get("email"):
|
|
199
|
+
q["contact_email"] = self.contact["email"]
|
|
200
|
+
if self.contact.get("phone"):
|
|
201
|
+
q["contact_phone"] = self.contact["phone"]
|
|
202
|
+
return self.service_url.replace("http", "ws", 1) + "/stream?" + urlencode(q)
|
|
203
|
+
|
|
204
|
+
async def start(self, timeout: float = 20.0) -> str:
|
|
205
|
+
"""Open the call and return its session id once Avarole has accepted it."""
|
|
206
|
+
if self.status in ("connecting", "live", "reconnecting"):
|
|
207
|
+
return self.session_id
|
|
208
|
+
self.session_id = secrets.token_hex(12)
|
|
209
|
+
self._intentional = False
|
|
210
|
+
self._attempts = 0
|
|
211
|
+
self._buf.clear()
|
|
212
|
+
self._live.clear()
|
|
213
|
+
self._set_status("connecting")
|
|
214
|
+
try:
|
|
215
|
+
await self._connect()
|
|
216
|
+
await asyncio.wait_for(self._live.wait(), timeout)
|
|
217
|
+
except Exception as exc:
|
|
218
|
+
await self.stop()
|
|
219
|
+
self._set_status("error")
|
|
220
|
+
raise AvaroleError("could not start the call: %s" % (exc or type(exc).__name__)) from exc
|
|
221
|
+
return self.session_id
|
|
222
|
+
|
|
223
|
+
async def _connect(self) -> None:
|
|
224
|
+
self._ws = await self._http_session().ws_connect(self._ws_url(), heartbeat=20, max_msg_size=0)
|
|
225
|
+
self._reader = asyncio.ensure_future(self._read(self._ws))
|
|
226
|
+
|
|
227
|
+
async def _read(self, ws: aiohttp.ClientWebSocketResponse) -> None:
|
|
228
|
+
deliberate = False
|
|
229
|
+
try:
|
|
230
|
+
async for msg in ws:
|
|
231
|
+
if msg.type == aiohttp.WSMsgType.TEXT and len(msg.data) <= 256 * 1024:
|
|
232
|
+
try:
|
|
233
|
+
await self._on_message(json.loads(msg.data))
|
|
234
|
+
except ValueError:
|
|
235
|
+
continue
|
|
236
|
+
elif msg.type in (aiohttp.WSMsgType.CLOSE, aiohttp.WSMsgType.CLOSING, aiohttp.WSMsgType.CLOSED):
|
|
237
|
+
break
|
|
238
|
+
deliberate = ws.close_code in (1000, 1001)
|
|
239
|
+
except Exception:
|
|
240
|
+
logger.debug("avarole: websocket read ended", exc_info=True)
|
|
241
|
+
if self._intentional or ws is not self._ws:
|
|
242
|
+
return
|
|
243
|
+
if not deliberate and self._attempts < MAX_RECONNECTS and self._live.is_set():
|
|
244
|
+
self._attempts += 1
|
|
245
|
+
self._set_status("reconnecting")
|
|
246
|
+
await asyncio.sleep(min(1.0 * self._attempts, 5.0))
|
|
247
|
+
if not self._intentional:
|
|
248
|
+
try:
|
|
249
|
+
await self._connect()
|
|
250
|
+
return
|
|
251
|
+
except Exception:
|
|
252
|
+
logger.warning("avarole: reconnect failed", exc_info=True)
|
|
253
|
+
self._set_status("ended" if deliberate else "error")
|
|
254
|
+
if not deliberate:
|
|
255
|
+
self._emit("error", "connection lost")
|
|
256
|
+
self._finish()
|
|
257
|
+
|
|
258
|
+
async def _on_message(self, m: dict) -> None:
|
|
259
|
+
kind = m.get("type")
|
|
260
|
+
if kind == "session":
|
|
261
|
+
self.session_id = str(m.get("sessionId") or self.session_id)
|
|
262
|
+
self._attempts = 0
|
|
263
|
+
self._set_status("live")
|
|
264
|
+
self._live.set()
|
|
265
|
+
self._emit("session", self.session_id)
|
|
266
|
+
await self._push_actions()
|
|
267
|
+
elif kind == "status":
|
|
268
|
+
self._emit("agent_status", str(m.get("msg") or ""))
|
|
269
|
+
elif kind == "transcript":
|
|
270
|
+
self._emit("transcript", str(m.get("text") or ""))
|
|
271
|
+
elif kind == "response_chunk":
|
|
272
|
+
self._emit("caption", str(m.get("text") or ""))
|
|
273
|
+
elif kind == "emotion":
|
|
274
|
+
self._emit("emotion", str(m.get("emotion") or ""))
|
|
275
|
+
elif kind == "interrupt":
|
|
276
|
+
self._emit("interrupt")
|
|
277
|
+
elif kind == "session_limit":
|
|
278
|
+
self._emit("limit", float(m.get("seconds") or 0))
|
|
279
|
+
elif kind == "session_ending":
|
|
280
|
+
self._emit("ending", float(m.get("seconds") or 0))
|
|
281
|
+
elif kind == "session_expired":
|
|
282
|
+
self._emit("expired", float(m.get("minutes") or 0))
|
|
283
|
+
self._spawn(self.stop())
|
|
284
|
+
elif kind == "error":
|
|
285
|
+
self._emit("error", str(m.get("msg") or "error"))
|
|
286
|
+
elif kind == "host_action":
|
|
287
|
+
self._spawn(self._run_action(m))
|
|
288
|
+
elif kind == "nav_action":
|
|
289
|
+
# Page tools (clicking, scrolling) only exist in a browser. Answer so the assistant does not wait.
|
|
290
|
+
self._spawn(
|
|
291
|
+
self._post(
|
|
292
|
+
"/nav/result",
|
|
293
|
+
{
|
|
294
|
+
"id": m.get("id"),
|
|
295
|
+
"tool": m.get("tool"),
|
|
296
|
+
"result": _result(
|
|
297
|
+
False,
|
|
298
|
+
"not_supported",
|
|
299
|
+
reason="This is not a web page, there is nothing to click or scroll.",
|
|
300
|
+
),
|
|
301
|
+
},
|
|
302
|
+
)
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
async def send_audio(self, pcm: bytes) -> None:
|
|
306
|
+
"""Send microphone audio: 16 kHz, mono, 16-bit little-endian PCM, any chunk size."""
|
|
307
|
+
if self._muted or self._ws is None or self._ws.closed or self.status not in ("live", "connecting"):
|
|
308
|
+
return
|
|
309
|
+
self._buf.extend(pcm)
|
|
310
|
+
while len(self._buf) >= FRAME_BYTES:
|
|
311
|
+
frame = bytes(self._buf[:FRAME_BYTES])
|
|
312
|
+
del self._buf[:FRAME_BYTES]
|
|
313
|
+
try:
|
|
314
|
+
await self._ws.send_bytes(frame)
|
|
315
|
+
except Exception:
|
|
316
|
+
return # the reader notices the drop and reconnects or ends the call
|
|
317
|
+
|
|
318
|
+
def mute(self, on: bool = True) -> None:
|
|
319
|
+
"""Stop sending the microphone; the call stays up."""
|
|
320
|
+
self._muted = on
|
|
321
|
+
if on:
|
|
322
|
+
self._buf.clear()
|
|
323
|
+
|
|
324
|
+
@property
|
|
325
|
+
def muted(self) -> bool:
|
|
326
|
+
return self._muted
|
|
327
|
+
|
|
328
|
+
async def stop(self) -> None:
|
|
329
|
+
"""End the call. Safe to call twice."""
|
|
330
|
+
self._intentional = True
|
|
331
|
+
ws, self._ws = self._ws, None
|
|
332
|
+
if ws is not None and not ws.closed:
|
|
333
|
+
try:
|
|
334
|
+
await ws.close(code=1000)
|
|
335
|
+
except Exception:
|
|
336
|
+
pass
|
|
337
|
+
if self._reader is not None and self._reader is not asyncio.current_task():
|
|
338
|
+
self._reader.cancel()
|
|
339
|
+
self._reader = None
|
|
340
|
+
was_live = self.status not in ("idle", "ended", "error")
|
|
341
|
+
if was_live:
|
|
342
|
+
self._set_status("ended")
|
|
343
|
+
self._finish()
|
|
344
|
+
for task in list(self._tasks):
|
|
345
|
+
if task is not asyncio.current_task():
|
|
346
|
+
task.cancel()
|
|
347
|
+
if self._own_http and self._http is not None:
|
|
348
|
+
await self._http.close()
|
|
349
|
+
self._http = None
|
|
350
|
+
|
|
351
|
+
def _finish(self) -> None:
|
|
352
|
+
self._buf.clear()
|
|
353
|
+
self._emit("end")
|
|
354
|
+
|
|
355
|
+
# --------------------------------------------------------------- avatar
|
|
356
|
+
|
|
357
|
+
async def livekit_join(self) -> LiveKitJoin:
|
|
358
|
+
"""What is needed to see and hear the avatar: the LiveKit server, a token for this call's room."""
|
|
359
|
+
url = "%s/livekit/token?room=%s&session_id=%s" % (
|
|
360
|
+
self.service_url,
|
|
361
|
+
quote(self.session_id),
|
|
362
|
+
quote(self.session_id),
|
|
363
|
+
)
|
|
364
|
+
async with self._http_session().get(url) as r:
|
|
365
|
+
if r.status != 200:
|
|
366
|
+
raise AvaroleError("avatar token refused (%s)" % r.status)
|
|
367
|
+
d = await r.json()
|
|
368
|
+
if not d.get("token") or not d.get("url"):
|
|
369
|
+
raise AvaroleError("avatar token response was incomplete")
|
|
370
|
+
return LiveKitJoin(url=d["url"], token=d["token"], room=self.session_id, session_id=self.session_id)
|
|
371
|
+
|
|
372
|
+
async def viewer_ready(self) -> None:
|
|
373
|
+
"""Tell Avarole the avatar's first video frame is on screen, so it greets someone who can see it."""
|
|
374
|
+
try:
|
|
375
|
+
async with self._http_session().post(
|
|
376
|
+
"%s/livekit/viewer-ready?session_id=%s" % (self.service_url, quote(self.session_id))
|
|
377
|
+
):
|
|
378
|
+
pass
|
|
379
|
+
except Exception:
|
|
380
|
+
logger.debug("avarole: viewer-ready not delivered", exc_info=True)
|
|
381
|
+
|
|
382
|
+
# -------------------------------------------------------------- actions
|
|
383
|
+
|
|
384
|
+
def register_actions(self, actions: list) -> dict:
|
|
385
|
+
"""Offer actions the assistant may call. Returns ``{"added": [...], "rejected": [...]}``."""
|
|
386
|
+
added, rejected = [], []
|
|
387
|
+
for a in actions or []:
|
|
388
|
+
ok = (
|
|
389
|
+
isinstance(a, Action)
|
|
390
|
+
and _ACTION_NAME.match(a.name or "")
|
|
391
|
+
and callable(a.handler)
|
|
392
|
+
and (a.name in self._actions or len(self._actions) < MAX_ACTIONS)
|
|
393
|
+
)
|
|
394
|
+
if not ok:
|
|
395
|
+
rejected.append(str(getattr(a, "name", a)))
|
|
396
|
+
continue
|
|
397
|
+
self._actions[a.name] = a
|
|
398
|
+
added.append(a.name)
|
|
399
|
+
if self.status == "live":
|
|
400
|
+
self._spawn(self._push_actions())
|
|
401
|
+
return {"added": added, "rejected": rejected}
|
|
402
|
+
|
|
403
|
+
def set_screen(self, info: Optional[ScreenInfo]) -> None:
|
|
404
|
+
"""Tell the assistant where the user is. Labels only, never what they typed."""
|
|
405
|
+
self._screen = (
|
|
406
|
+
None
|
|
407
|
+
if info is None
|
|
408
|
+
else {
|
|
409
|
+
"name": str(info.name or "")[:80],
|
|
410
|
+
"summary": str(info.summary or "")[:300],
|
|
411
|
+
"fields": [
|
|
412
|
+
{
|
|
413
|
+
"id": str(f.get("id"))[:40],
|
|
414
|
+
"label": str(f.get("label"))[:60],
|
|
415
|
+
"type": str(f.get("type") or "")[:20],
|
|
416
|
+
"filled": bool(f.get("filled")),
|
|
417
|
+
}
|
|
418
|
+
for f in (info.fields or [])[:40]
|
|
419
|
+
],
|
|
420
|
+
}
|
|
421
|
+
)
|
|
422
|
+
if self.status == "live":
|
|
423
|
+
self._spawn(self._push_screen())
|
|
424
|
+
|
|
425
|
+
async def _push_actions(self) -> None:
|
|
426
|
+
if self._actions:
|
|
427
|
+
await self._post(
|
|
428
|
+
"/host/actions",
|
|
429
|
+
{
|
|
430
|
+
"actions": [
|
|
431
|
+
{
|
|
432
|
+
"name": a.name,
|
|
433
|
+
"description": str(a.description or "")[:300],
|
|
434
|
+
"params": a.params,
|
|
435
|
+
"confirm": bool(a.confirm),
|
|
436
|
+
}
|
|
437
|
+
for a in self._actions.values()
|
|
438
|
+
]
|
|
439
|
+
},
|
|
440
|
+
)
|
|
441
|
+
await self._push_screen()
|
|
442
|
+
|
|
443
|
+
async def _push_screen(self) -> None:
|
|
444
|
+
if self._screen:
|
|
445
|
+
await self._post("/host/context", {"screen": self._screen})
|
|
446
|
+
|
|
447
|
+
async def _post(self, path: str, body: dict) -> None:
|
|
448
|
+
try:
|
|
449
|
+
async with self._http_session().post(self.service_url + path, json={"session_id": self.session_id, **body}):
|
|
450
|
+
pass
|
|
451
|
+
except Exception:
|
|
452
|
+
logger.debug("avarole: POST %s failed", path, exc_info=True)
|
|
453
|
+
|
|
454
|
+
async def _run_action(self, m: dict) -> None:
|
|
455
|
+
aid, name = m.get("id"), str(m.get("name") or "")
|
|
456
|
+
|
|
457
|
+
async def reply(result: dict) -> None:
|
|
458
|
+
await self._post("/host/result", {"id": aid, "name": name, "result": result})
|
|
459
|
+
|
|
460
|
+
now = time.monotonic()
|
|
461
|
+
self._recent = [t for t in self._recent if now - t < 60]
|
|
462
|
+
if self._inflight >= MAX_INFLIGHT_ACTIONS:
|
|
463
|
+
return await reply(_result(False, "busy"))
|
|
464
|
+
if len(self._recent) >= MAX_ACTIONS_PER_MINUTE:
|
|
465
|
+
return await reply(_result(False, "rate_limited", reason="Too many actions in a minute."))
|
|
466
|
+
action = self._actions.get(name)
|
|
467
|
+
if action is None:
|
|
468
|
+
return await reply(_result(False, "unknown_action", reason=name))
|
|
469
|
+
self._recent.append(now)
|
|
470
|
+
self._inflight += 1
|
|
471
|
+
try:
|
|
472
|
+
if action.confirm:
|
|
473
|
+
yes = False
|
|
474
|
+
if self._confirm is not None:
|
|
475
|
+
yes = self._confirm(
|
|
476
|
+
"Allow this action?", "The assistant wants to: %s." % (action.description or action.name)
|
|
477
|
+
)
|
|
478
|
+
if asyncio.iscoroutine(yes):
|
|
479
|
+
yes = await yes
|
|
480
|
+
if not yes:
|
|
481
|
+
return await reply(
|
|
482
|
+
_result(
|
|
483
|
+
False,
|
|
484
|
+
"declined",
|
|
485
|
+
reason=(
|
|
486
|
+
"The user chose not to allow it."
|
|
487
|
+
if self._confirm
|
|
488
|
+
else "There is no way to ask the user, so it was not run."
|
|
489
|
+
),
|
|
490
|
+
)
|
|
491
|
+
)
|
|
492
|
+
args = m.get("args") if isinstance(m.get("args"), dict) else {}
|
|
493
|
+
try:
|
|
494
|
+
out = action.handler(args)
|
|
495
|
+
if asyncio.iscoroutine(out):
|
|
496
|
+
out = await asyncio.wait_for(out, ACTION_TIMEOUT_S)
|
|
497
|
+
result = (
|
|
498
|
+
out if isinstance(out, dict) and isinstance(out.get("ok"), bool) else _result(True, "ok", data=out)
|
|
499
|
+
)
|
|
500
|
+
except asyncio.TimeoutError:
|
|
501
|
+
result = _result(False, "timeout", reason="The app took too long to respond.")
|
|
502
|
+
except Exception as exc:
|
|
503
|
+
result = _result(False, "error", reason=str(exc)[:120])
|
|
504
|
+
finally:
|
|
505
|
+
self._inflight -= 1
|
|
506
|
+
if len(json.dumps(result, default=str)) > MAX_RESULT_CHARS:
|
|
507
|
+
result = _result(
|
|
508
|
+
result["ok"],
|
|
509
|
+
result.get("status", "ok"),
|
|
510
|
+
reason="The result was too large and was cut.",
|
|
511
|
+
data=json.dumps(result.get("data"), default=str)[: MAX_RESULT_CHARS - 200],
|
|
512
|
+
)
|
|
513
|
+
await reply(result)
|
|
514
|
+
|
|
515
|
+
# ----------------------------------------------------------- the account
|
|
516
|
+
|
|
517
|
+
async def agent_config(self) -> dict:
|
|
518
|
+
"""The assistant's public settings (name, avatar, language...) as configured in the dashboard."""
|
|
519
|
+
async with self._http_session().get(
|
|
520
|
+
"%s/assistant/agents/%s/config" % (self.app_url, quote(self.agent_id))
|
|
521
|
+
) as r:
|
|
522
|
+
if r.status != 200:
|
|
523
|
+
raise AvaroleError("could not read the agent's settings (%s)" % r.status)
|
|
524
|
+
return await r.json()
|
|
525
|
+
|
|
526
|
+
async def sessions(self, limit: int = 50, offset: int = 0) -> list:
|
|
527
|
+
"""Past calls on your account, newest first. Needs the API key."""
|
|
528
|
+
if not self.api_key:
|
|
529
|
+
raise AvaroleError("api_key is required to list sessions")
|
|
530
|
+
async with self._http_session().get(
|
|
531
|
+
"%s/assistant/sessions?limit=%d&offset=%d" % (self.app_url, max(1, min(limit, 200)), max(0, offset)),
|
|
532
|
+
headers={"X-API-Key": self.api_key},
|
|
533
|
+
) as r:
|
|
534
|
+
if r.status != 200:
|
|
535
|
+
raise AvaroleError("could not list sessions (%s)" % r.status)
|
|
536
|
+
return await r.json()
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: livekit-plugins-avarole
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Avarole plugin for LiveKit Agents: a live avatar that talks, listens and acts inside your LiveKit room
|
|
5
|
+
Project-URL: Homepage, https://avarole.com
|
|
6
|
+
Project-URL: Documentation, https://github.com/Sbaig3229/livekit-plugins-avarole#readme
|
|
7
|
+
Project-URL: Source, https://github.com/Sbaig3229/livekit-plugins-avarole
|
|
8
|
+
Author-email: Avarole <support@avarole.com>
|
|
9
|
+
License-Expression: Apache-2.0
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agents,ai,avarole,avatar,livekit,realtime,voice,webrtc
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Multimedia :: Sound/Audio
|
|
20
|
+
Classifier: Topic :: Multimedia :: Video
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Requires-Dist: aiohttp>=3.9
|
|
24
|
+
Requires-Dist: livekit-agents>=1.0
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'test'
|
|
27
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# livekit-plugins-avarole
|
|
31
|
+
|
|
32
|
+
Avarole plugin for [LiveKit Agents](https://docs.livekit.io/agents/). It puts an **Avarole avatar** in your LiveKit room:
|
|
33
|
+
it listens to the people there, answers out loud with its face and voice, and can do everything an Avarole assistant
|
|
34
|
+
does on the platform (its knowledge base, tickets, scheduling, the tools you enable) plus **actions you give it in your
|
|
35
|
+
own app**.
|
|
36
|
+
|
|
37
|
+
Avarole runs the whole conversation. This package is the bridge: people in your room reach Avarole, and the avatar's
|
|
38
|
+
video and voice come back into your room as one participant.
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
people in your room ──(microphone, 16 kHz)──▶ Avarole
|
|
42
|
+
Avarole avatar ──(video + voice)──▶ published into your room
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Install
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install livekit-plugins-avarole
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
You need an Avarole assistant: sign in at [app.avarole.com](https://app.avarole.com), build one, and copy its **agent id**
|
|
52
|
+
and **API key** from *Integrations*.
|
|
53
|
+
|
|
54
|
+
## Quick start (LiveKit Agents)
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from livekit.agents import JobContext, WorkerOptions, cli
|
|
58
|
+
from livekit.plugins import avarole
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
async def entrypoint(ctx: JobContext):
|
|
62
|
+
await ctx.connect()
|
|
63
|
+
session = avarole.AvatarSession(agent_id="YOUR_AGENT_ID", api_key="sk_YOUR_KEY")
|
|
64
|
+
await session.start(ctx.room) # the avatar joins and greets whoever is in the room
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
if __name__ == "__main__":
|
|
68
|
+
cli.run_app(WorkerOptions(entrypoint_fnc=entrypoint))
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
The avatar listens to every person in the room (not to other agents), and the call ends when the room closes, you call
|
|
72
|
+
`await session.aclose()`, or the plan's session time runs out. Use `listen_to=lambda p: p.identity == "alice"` to narrow
|
|
73
|
+
who it hears.
|
|
74
|
+
|
|
75
|
+
Keep the key in an environment variable, not in code.
|
|
76
|
+
|
|
77
|
+
## Let the avatar act in your app
|
|
78
|
+
|
|
79
|
+
List what your app can do. The assistant can call these and nothing else. Anything that changes something should
|
|
80
|
+
set `confirm=True`: the user must allow it first.
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from livekit.plugins.avarole import Action
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
async def apply_coupon(args):
|
|
87
|
+
return await cart.apply(str(args["code"])) # validate arguments like any user input
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
session = avarole.AvatarSession(
|
|
91
|
+
agent_id=...,
|
|
92
|
+
api_key=...,
|
|
93
|
+
confirm=ask_the_user, # async (title, body) -> bool
|
|
94
|
+
actions=[
|
|
95
|
+
Action(
|
|
96
|
+
"apply_coupon",
|
|
97
|
+
"Apply a coupon code to the cart",
|
|
98
|
+
apply_coupon,
|
|
99
|
+
params={"type": "object", "properties": {"code": {"type": "string"}}, "required": ["code"]},
|
|
100
|
+
),
|
|
101
|
+
Action("cancel_order", "Cancel the current order", cancel_order, confirm=True),
|
|
102
|
+
],
|
|
103
|
+
)
|
|
104
|
+
session.set_screen(
|
|
105
|
+
avarole.ScreenInfo("Checkout", "Cart with 2 items", [{"id": "email", "label": "Email", "filled": True}])
|
|
106
|
+
)
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Handlers return data, or `{"ok": False, "status": "...", "reason": "..."}` to report a problem; the assistant reads it
|
|
110
|
+
and tells the user. At most 3 actions run at once and 30 per minute. `set_screen` takes labels only, never what the
|
|
111
|
+
user typed.
|
|
112
|
+
|
|
113
|
+
## Captions, transcripts and the rest
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
call = session.client
|
|
117
|
+
call.on("transcript", lambda text: print("user:", text)) # what the user said
|
|
118
|
+
call.on("caption", lambda text: print("avatar:", text)) # what the avatar is saying
|
|
119
|
+
call.on("agent_status", print) # "looking that up"
|
|
120
|
+
call.on("emotion", print)
|
|
121
|
+
call.on("interrupt", lambda _: print("barge-in"))
|
|
122
|
+
call.on("limit", lambda seconds: ...) # session length, then "ending" and "expired"
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Events: `status`, `session`, `agent_status`, `transcript`, `caption`, `emotion`, `interrupt`, `limit`, `ending`,
|
|
126
|
+
`expired`, `error`, `end`. Listeners may be plain functions or coroutines; one that raises never breaks the call.
|
|
127
|
+
|
|
128
|
+
## Without LiveKit
|
|
129
|
+
|
|
130
|
+
`AvaroleClient` is the same call for any app that can supply microphone audio (16 kHz, mono, 16-bit PCM):
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from livekit.plugins.avarole import AvaroleClient
|
|
134
|
+
|
|
135
|
+
async with AvaroleClient(agent_id="...", api_key="sk_...") as call:
|
|
136
|
+
call.on("caption", print)
|
|
137
|
+
await call.send_audio(pcm_bytes) # any chunk size; sent as 60 ms frames
|
|
138
|
+
join = await call.livekit_join() # url + token to watch the avatar yourself
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
It also reads account data: `await call.agent_config()` (the assistant's public settings) and
|
|
142
|
+
`await call.sessions(limit=50)` (past calls, newest first; needs the API key).
|
|
143
|
+
|
|
144
|
+
## Good to know
|
|
145
|
+
|
|
146
|
+
- Calls last at most the agent's session limit (set on the Build page); `limit` / `ending` / `expired` tell you.
|
|
147
|
+
- If the connection drops the client retries three times, then ends the call and emits `error`.
|
|
148
|
+
- Page tools (clicking, scrolling) only exist in a browser; here the assistant is told they are unavailable.
|
|
149
|
+
- Use one avatar per room. Another agent that also speaks in the room will not be heard by the avatar.
|
|
150
|
+
- Status: **beta (0.x)**. The call client is covered by tests against a stand-in for the call service; run your own
|
|
151
|
+
end-to-end check against your LiveKit server before going live.
|
|
152
|
+
|
|
153
|
+
## Development
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
pip install -e ".[test]"
|
|
157
|
+
pytest
|
|
158
|
+
ruff check . && ruff format --check .
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Apache-2.0. © Avarole.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
livekit/plugins/avarole/__init__.py,sha256=QPWqtTibFtncDlukoqmZuXue092l_LuWnpjR651Dmhg,1050
|
|
2
|
+
livekit/plugins/avarole/avatar.py,sha256=ZOWxwMsyVryxVGrPAQlPlJ-Lvc9gAiz2mNX__MZiWNw,7902
|
|
3
|
+
livekit/plugins/avarole/client.py,sha256=nbwdld4MPB2EA32bBDV9nAmk_9TjIH2JAj3bKNvp--Y,20998
|
|
4
|
+
livekit/plugins/avarole/log.py,sha256=zjBRUcThSrQuK7wiaW-NJSnACh3LZk7VT_DTH3-ohZY,70
|
|
5
|
+
livekit/plugins/avarole/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
livekit/plugins/avarole/version.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
|
|
7
|
+
livekit_plugins_avarole-0.1.0.dist-info/METADATA,sha256=3qC-pCBHY2JgSMs3V3yllM94uIpSb8mc4O7KVnJtE-8,6295
|
|
8
|
+
livekit_plugins_avarole-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
9
|
+
livekit_plugins_avarole-0.1.0.dist-info/licenses/LICENSE,sha256=l9B5fJBUBcogm5bQYq34EXx3pzPe7CWLKKfDXBtumB4,11385
|
|
10
|
+
livekit_plugins_avarole-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
|
203
|
+
|
|
204
|
+
Copyright 2026 Avarole
|