pipecat-spatialreal 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipecat_spatialreal/__init__.py +6 -0
- pipecat_spatialreal/driven.py +387 -0
- pipecat_spatialreal/envelope.py +257 -0
- pipecat_spatialreal/fingerprint.py +117 -0
- pipecat_spatialreal/onset.py +118 -0
- pipecat_spatialreal/processor.py +1379 -0
- pipecat_spatialreal/proto/__init__.py +1 -0
- pipecat_spatialreal/proto/generated/__init__.py +1 -0
- pipecat_spatialreal/proto/generated/avatar_pb2.py +53 -0
- pipecat_spatialreal/proto/generated/avatar_pb2.pyi +118 -0
- pipecat_spatialreal/py.typed +0 -0
- pipecat_spatialreal/version.py +3 -0
- pipecat_spatialreal-0.1.0.dist-info/METADATA +322 -0
- pipecat_spatialreal-0.1.0.dist-info/RECORD +16 -0
- pipecat_spatialreal-0.1.0.dist-info/WHEEL +4 -0
- pipecat_spatialreal-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
"""A thin asyncio layer over one SpatialReal driveningress session.
|
|
2
|
+
|
|
3
|
+
The processor streams the bot's speech to driveningress and gets the facial animation back. The
|
|
4
|
+
``spatialreal`` SDK does the work (the session token, the WebSocket, the protobuf messages); this
|
|
5
|
+
module narrows it to what the processor needs.
|
|
6
|
+
|
|
7
|
+
* Animation returns over the same WebSocket, which is what the SDK does when no egress is
|
|
8
|
+
configured. Each animation message from the server reaches the caller as a :class:`Segment`
|
|
9
|
+
whose ``wire`` is the SDK's bytes, untouched, so the processor can forward them without
|
|
10
|
+
serializing anything.
|
|
11
|
+
* A session that ends is reported once, through ``on_down``, with a short readable reason.
|
|
12
|
+
The SDK has no reconnect and a session object cannot be reused, so this layer has none either:
|
|
13
|
+
to come back, call :meth:`DrivenIngress.connect` again, which builds a new SDK session. When
|
|
14
|
+
to do that, and how often, is up to the caller.
|
|
15
|
+
* Nothing here waits for the socket for long. A link that stops taking data without closing (a
|
|
16
|
+
dropped flow, a server that stops reading) makes a write block in the kernel's send buffer,
|
|
17
|
+
and the SDK puts no limit on that. Every write and the closing handshake are bounded by
|
|
18
|
+
``wire_timeout_s``; one that runs out counts as a loss, and the socket is aborted, which is
|
|
19
|
+
the only way to end a write that never returns.
|
|
20
|
+
|
|
21
|
+
It decides nothing about turns: which audio belongs to which request, and when a request ends,
|
|
22
|
+
is the caller's.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
import asyncio
|
|
26
|
+
from collections.abc import Callable
|
|
27
|
+
from dataclasses import dataclass
|
|
28
|
+
from datetime import UTC, datetime, timedelta
|
|
29
|
+
from enum import Enum, auto
|
|
30
|
+
from functools import partial
|
|
31
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
32
|
+
|
|
33
|
+
import spatialreal
|
|
34
|
+
from loguru import logger
|
|
35
|
+
from spatialreal.proto.generated import message_pb2
|
|
36
|
+
from spatialreal.session import DRIVENINGRESS_PATH
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class Segment:
|
|
41
|
+
"""One animation message from driveningress.
|
|
42
|
+
|
|
43
|
+
The server answers a request's audio with one message per inference segment, each carrying
|
|
44
|
+
the keyframes for a stretch of that audio.
|
|
45
|
+
|
|
46
|
+
Attributes:
|
|
47
|
+
req_id: The request the animation belongs to: the id :meth:`DrivenIngress.send` returned
|
|
48
|
+
for its audio.
|
|
49
|
+
wire: The complete serialized ``driveningress.v2.Message``, exactly as received.
|
|
50
|
+
end: True on the last message the server sends for the request. That message is an
|
|
51
|
+
ordinary segment, usually with animation, not a marker of its own, and the server
|
|
52
|
+
may send none. It sends none when the end arrives with nothing left to infer (no
|
|
53
|
+
audio was sent, or the audio already filled its last segment exactly, and the
|
|
54
|
+
client sends the 16 kHz the inference runs at, so no resampler tail remains) or
|
|
55
|
+
when the last inference fails. A consumer must not wait for ``end`` alone to close
|
|
56
|
+
a turn.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
req_id: str
|
|
60
|
+
wire: bytes
|
|
61
|
+
end: bool
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class _State(Enum):
|
|
65
|
+
CONNECTING = auto() # connect() is still opening the session
|
|
66
|
+
UP = auto() # connected: the only state in which anything is delivered or sent
|
|
67
|
+
DOWN = auto() # lost, closed, or never opened
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class _Link:
|
|
71
|
+
"""What belongs to one SDK session.
|
|
72
|
+
|
|
73
|
+
A session is single-use, so everything that describes it lives here and the next session
|
|
74
|
+
starts from a new link. The callbacks the SDK is given carry the link, and a link that is
|
|
75
|
+
DOWN drops whatever its session still reports, so a late callback from a session that was
|
|
76
|
+
lost, closed or replaced cannot reach the caller.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
session: spatialreal.AvatarSession # set right after construction; the callbacks need the link
|
|
80
|
+
|
|
81
|
+
def __init__(self) -> None:
|
|
82
|
+
self.state = _State.CONNECTING
|
|
83
|
+
self.error: Exception | None = None # the last error the session reported
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
class DrivenIngress:
|
|
87
|
+
"""One driveningress session at a time, for the audio of one avatar.
|
|
88
|
+
|
|
89
|
+
Use it from the event loop that owns it; nothing here is thread-safe. The callbacks run on
|
|
90
|
+
that loop and must not block.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
def __init__(
|
|
94
|
+
self,
|
|
95
|
+
*,
|
|
96
|
+
api_key: str,
|
|
97
|
+
app_id: str,
|
|
98
|
+
avatar_id: str,
|
|
99
|
+
sample_rate: int,
|
|
100
|
+
environment: str = "us-west",
|
|
101
|
+
console_endpoint_url: str = "",
|
|
102
|
+
ingress_endpoint_url: str = "",
|
|
103
|
+
token_ttl_s: int = 86400,
|
|
104
|
+
wire_timeout_s: float = 2.0,
|
|
105
|
+
on_segment: Callable[[Segment], None],
|
|
106
|
+
on_down: Callable[[str], None],
|
|
107
|
+
):
|
|
108
|
+
"""Remember the settings; nothing connects until :meth:`connect`.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
api_key: SpatialReal API key, exchanged for a session token on every connect.
|
|
112
|
+
app_id: SpatialReal app id.
|
|
113
|
+
avatar_id: The avatar to drive.
|
|
114
|
+
sample_rate: Hz of the PCM16 mono audio that will be sent. It is fixed for a session
|
|
115
|
+
and the server accepts only some rates (8, 16, 22.05, 24, 32, 44.1 and 48 kHz).
|
|
116
|
+
environment: Which SpatialReal deployment to use, when the two URLs below are not
|
|
117
|
+
given.
|
|
118
|
+
console_endpoint_url: Origin of the console API that issues session tokens
|
|
119
|
+
(``https://<host>``; the SDK appends the route); empty takes it from
|
|
120
|
+
``environment``.
|
|
121
|
+
ingress_endpoint_url: The driveningress URL, ``https://<host>/v2/driveningress``;
|
|
122
|
+
empty takes it from ``environment``. An origin alone (``https://<host>``) is
|
|
123
|
+
accepted and given that path; a URL with any other path is used as given.
|
|
124
|
+
token_ttl_s: Seconds each session token stays valid.
|
|
125
|
+
wire_timeout_s: Longest wait, in seconds, for one write on the socket (audio, the end
|
|
126
|
+
of a request, an interrupt) and for the closing handshake. A write that takes
|
|
127
|
+
longer is a loss: the session's socket is aborted and ``on_down`` reports it.
|
|
128
|
+
on_segment: Called with every animation message, in the order it arrived. It is not
|
|
129
|
+
called once the session is lost or closed.
|
|
130
|
+
on_down: Called once for each session that ends without :meth:`close`, with a short
|
|
131
|
+
reason, for example ``creditsExhausted (4001)`` (the SDK's error code, then the
|
|
132
|
+
WebSocket close code when there is one; :func:`close_code` reads it back),
|
|
133
|
+
``send timed out`` or ``closed``. The reason comes from the last error the
|
|
134
|
+
session reported before it ended, or is ``closed`` when it reported none.
|
|
135
|
+
|
|
136
|
+
Raises:
|
|
137
|
+
ValueError: If ``wire_timeout_s`` is not positive.
|
|
138
|
+
"""
|
|
139
|
+
if wire_timeout_s <= 0:
|
|
140
|
+
raise ValueError(f"wire_timeout_s must be positive, got {wire_timeout_s}")
|
|
141
|
+
self._options = {
|
|
142
|
+
"api_key": api_key,
|
|
143
|
+
"app_id": app_id,
|
|
144
|
+
"avatar_id": avatar_id,
|
|
145
|
+
"sample_rate": sample_rate,
|
|
146
|
+
"environment": environment,
|
|
147
|
+
"console_endpoint_url": console_endpoint_url,
|
|
148
|
+
"ingress_endpoint_url": _ingress_url(ingress_endpoint_url),
|
|
149
|
+
}
|
|
150
|
+
self._token_ttl = timedelta(seconds=token_ttl_s)
|
|
151
|
+
self._wire_timeout = wire_timeout_s
|
|
152
|
+
self._on_segment = on_segment
|
|
153
|
+
self._on_down = on_down
|
|
154
|
+
self._link: _Link | None = None
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def connected(self) -> bool:
|
|
158
|
+
"""Whether a session is up: connected, and neither lost nor closed since."""
|
|
159
|
+
return self._link is not None and self._link.state is _State.UP
|
|
160
|
+
|
|
161
|
+
async def connect(self) -> None:
|
|
162
|
+
"""Open a new session, replacing whatever is left of the last one.
|
|
163
|
+
|
|
164
|
+
Calling it again after a loss is how the caller reconnects. A session that is still up
|
|
165
|
+
is closed first, without ``on_down``. Do not call it from two tasks at once.
|
|
166
|
+
|
|
167
|
+
Raises:
|
|
168
|
+
spatialreal.AvatarSDKError: The token exchange or the handshake failed; its ``code``
|
|
169
|
+
says why, and ``retryable`` whether trying again can help.
|
|
170
|
+
ValueError: ``api_key``, ``app_id`` or ``avatar_id`` is empty.
|
|
171
|
+
RuntimeError: :meth:`close` ran while this call was waiting and the call gave up.
|
|
172
|
+
A close that cuts the handshake short raises ``AvatarSDKError`` instead. Either
|
|
173
|
+
way no socket is left open.
|
|
174
|
+
"""
|
|
175
|
+
await self.close()
|
|
176
|
+
link = _Link()
|
|
177
|
+
link.session = spatialreal.new_avatar_session(
|
|
178
|
+
**self._options,
|
|
179
|
+
expire_at=datetime.now(UTC) + self._token_ttl,
|
|
180
|
+
audio_format="pcm_s16le",
|
|
181
|
+
transport_frames=partial(self._on_frame, link),
|
|
182
|
+
on_error=partial(self._on_error, link),
|
|
183
|
+
on_close=partial(self._on_close, link),
|
|
184
|
+
)
|
|
185
|
+
self._link = link # close() can reach the session from here on
|
|
186
|
+
try:
|
|
187
|
+
await link.session.init()
|
|
188
|
+
# A close() during the token exchange must not be followed by a socket: it bills.
|
|
189
|
+
if link.state is _State.CONNECTING:
|
|
190
|
+
await link.session.start()
|
|
191
|
+
except BaseException: # also a cancellation: the half-open session must not outlive it
|
|
192
|
+
link.state = _State.DOWN
|
|
193
|
+
if self._link is link:
|
|
194
|
+
self._link = None
|
|
195
|
+
await self._close_session(link)
|
|
196
|
+
raise
|
|
197
|
+
if link.state is _State.DOWN: # close() ran while this call waited
|
|
198
|
+
await self._close_session(link) # closes the socket too, if start() opened one
|
|
199
|
+
raise RuntimeError("driveningress closed while connecting")
|
|
200
|
+
link.state = _State.UP
|
|
201
|
+
|
|
202
|
+
async def send(self, pcm: bytes, *, end: bool = False) -> str:
|
|
203
|
+
"""Send audio to the current request.
|
|
204
|
+
|
|
205
|
+
The first call after a connect, after ``end=True`` or after an interrupt starts a new
|
|
206
|
+
request; the id is never reused. The call returns within ``wire_timeout_s``.
|
|
207
|
+
|
|
208
|
+
Args:
|
|
209
|
+
pcm: PCM16 little-endian mono audio at ``sample_rate``; may be empty.
|
|
210
|
+
end: True on the request's last chunk. The server's reply to it is not guaranteed to
|
|
211
|
+
be a :class:`Segment` with ``end`` set; see there.
|
|
212
|
+
|
|
213
|
+
Returns:
|
|
214
|
+
The id of the request the audio went into.
|
|
215
|
+
|
|
216
|
+
Raises:
|
|
217
|
+
RuntimeError: The session is not up, or the send failed or timed out. A failed or
|
|
218
|
+
timed-out send is a loss: it is reported through ``on_down`` like any other, and
|
|
219
|
+
the original error is the exception's ``__cause__``.
|
|
220
|
+
"""
|
|
221
|
+
link = self._link
|
|
222
|
+
if link is None or link.state is not _State.UP:
|
|
223
|
+
raise RuntimeError("driveningress not connected")
|
|
224
|
+
try:
|
|
225
|
+
async with asyncio.timeout(self._wire_timeout):
|
|
226
|
+
return await link.session.send_audio(pcm, end=end)
|
|
227
|
+
except (spatialreal.AvatarSDKError, ValueError) as exc:
|
|
228
|
+
self._lose(link, _describe(exc))
|
|
229
|
+
raise RuntimeError("driveningress not connected") from exc
|
|
230
|
+
except TimeoutError as exc:
|
|
231
|
+
self._cut(link, "send timed out")
|
|
232
|
+
raise RuntimeError("driveningress not connected") from exc
|
|
233
|
+
|
|
234
|
+
async def interrupt(self) -> str | None:
|
|
235
|
+
"""Tell the server to drop the audio it still holds for the latest request.
|
|
236
|
+
|
|
237
|
+
The next :meth:`send` starts a new request. The latest request is the SDK's: it takes the
|
|
238
|
+
id when a send starts, before the audio is written, so a send that was cancelled before
|
|
239
|
+
it returned (on a congested socket, say) is interrupted too, and the audio after the
|
|
240
|
+
interrupt cannot join it. This never raises for a missing or broken session: there is
|
|
241
|
+
then nothing left to interrupt, and the caller is usually in the middle of something
|
|
242
|
+
more urgent. The call returns within ``wire_timeout_s``; one that takes longer is a loss.
|
|
243
|
+
|
|
244
|
+
Returns:
|
|
245
|
+
The id of the request interrupted, or None when nothing was sent on this session or
|
|
246
|
+
the session is not up.
|
|
247
|
+
"""
|
|
248
|
+
link = self._link
|
|
249
|
+
if link is None or link.state is not _State.UP:
|
|
250
|
+
return None
|
|
251
|
+
try:
|
|
252
|
+
async with asyncio.timeout(self._wire_timeout):
|
|
253
|
+
return await link.session.interrupt()
|
|
254
|
+
except ValueError:
|
|
255
|
+
# Nothing was sent on this session yet. (The SDK says the same when its socket is
|
|
256
|
+
# already gone; the session's close reports that loss.)
|
|
257
|
+
return None
|
|
258
|
+
except spatialreal.AvatarSDKError as exc:
|
|
259
|
+
self._lose(link, _describe(exc))
|
|
260
|
+
return None
|
|
261
|
+
except TimeoutError:
|
|
262
|
+
self._cut(link, "interrupt timed out")
|
|
263
|
+
return None
|
|
264
|
+
|
|
265
|
+
async def close(self) -> None:
|
|
266
|
+
"""Close the session; safe at any time, including when there is none, and repeatedly.
|
|
267
|
+
|
|
268
|
+
``on_down`` is not called, because closing is not a loss. The closing handshake is given
|
|
269
|
+
``wire_timeout_s``; a link that does not complete it in that time has its socket aborted.
|
|
270
|
+
"""
|
|
271
|
+
link, self._link = self._link, None
|
|
272
|
+
if link is not None:
|
|
273
|
+
link.state = _State.DOWN
|
|
274
|
+
await self._close_session(link)
|
|
275
|
+
|
|
276
|
+
# --- the socket, when the SDK cannot get it to answer
|
|
277
|
+
|
|
278
|
+
async def _close_session(self, link: _Link) -> None:
|
|
279
|
+
"""Close the link's SDK session, aborting its socket if the handshake does not return.
|
|
280
|
+
|
|
281
|
+
The SDK's close sends a close frame and waits for the other end, with no bound on the
|
|
282
|
+
write: on a link that has stopped draining, it waits for good.
|
|
283
|
+
"""
|
|
284
|
+
connection = _connection_of(link) # the SDK forgets it before it waits
|
|
285
|
+
try:
|
|
286
|
+
async with asyncio.timeout(self._wire_timeout):
|
|
287
|
+
await link.session.close()
|
|
288
|
+
return
|
|
289
|
+
except TimeoutError:
|
|
290
|
+
logger.warning("driveningress close timed out; aborting the socket")
|
|
291
|
+
_abort(connection)
|
|
292
|
+
try:
|
|
293
|
+
async with asyncio.timeout(self._wire_timeout):
|
|
294
|
+
await link.session.close() # idempotent: finishes what the first call began
|
|
295
|
+
except TimeoutError:
|
|
296
|
+
logger.error("driveningress session could not be closed")
|
|
297
|
+
|
|
298
|
+
def _cut(self, link: _Link, reason: str) -> None:
|
|
299
|
+
"""Abort the socket of a link that does not answer, and report the loss."""
|
|
300
|
+
_abort(_connection_of(link))
|
|
301
|
+
self._lose(link, reason)
|
|
302
|
+
|
|
303
|
+
# --- what the SDK reports
|
|
304
|
+
|
|
305
|
+
def _on_frame(self, link: _Link, wire: bytes, is_last: bool) -> None:
|
|
306
|
+
if link.state is not _State.UP:
|
|
307
|
+
return
|
|
308
|
+
req_id = message_pb2.Message.FromString(wire).server_response_animation.req_id
|
|
309
|
+
self._on_segment(Segment(req_id=req_id, wire=wire, end=is_last))
|
|
310
|
+
|
|
311
|
+
def _on_error(self, link: _Link, exc: Exception) -> None:
|
|
312
|
+
# An error does not necessarily end the session: the server may report a failed request
|
|
313
|
+
# and carry on. What ends it is the close, so remember the error and let _on_close decide.
|
|
314
|
+
if link.state is not _State.UP:
|
|
315
|
+
return
|
|
316
|
+
link.error = exc
|
|
317
|
+
logger.warning("driveningress session error: {}", _describe(exc))
|
|
318
|
+
|
|
319
|
+
def _on_close(self, link: _Link) -> None:
|
|
320
|
+
self._lose(link, "closed" if link.error is None else _describe(link.error))
|
|
321
|
+
|
|
322
|
+
def _lose(self, link: _Link, reason: str) -> None:
|
|
323
|
+
"""Report that ``link`` ended, once; nothing if it is already DOWN."""
|
|
324
|
+
if link.state is not _State.UP:
|
|
325
|
+
return
|
|
326
|
+
link.state = _State.DOWN
|
|
327
|
+
try:
|
|
328
|
+
self._on_down(reason)
|
|
329
|
+
except Exception: # noqa: BLE001
|
|
330
|
+
# The callback is the caller's. Its bug must not reach the call that found the loss:
|
|
331
|
+
# send() promises RuntimeError and interrupt() promises not to raise.
|
|
332
|
+
logger.exception("on_down callback raised")
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _connection_of(link: _Link) -> object | None:
|
|
336
|
+
"""The WebSocket connection of the link's session, if it has one open.
|
|
337
|
+
|
|
338
|
+
The SDK gives no way to abort a session; its connection is the one thing this module reaches
|
|
339
|
+
into it for.
|
|
340
|
+
"""
|
|
341
|
+
return getattr(link.session, "_connection", None)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _abort(connection: object | None) -> None:
|
|
345
|
+
"""Close the connection's socket at once, without the closing handshake.
|
|
346
|
+
|
|
347
|
+
The pending writes are dropped and every call waiting on the socket returns; the SDK's read
|
|
348
|
+
loop sees the connection closed and finishes with the session.
|
|
349
|
+
"""
|
|
350
|
+
transport = getattr(connection, "transport", None)
|
|
351
|
+
if transport is not None:
|
|
352
|
+
transport.abort()
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _ingress_url(url: str) -> str:
|
|
356
|
+
"""``url`` with the driveningress path, when it is an origin without one."""
|
|
357
|
+
if not url:
|
|
358
|
+
return url
|
|
359
|
+
parts = urlsplit(url)
|
|
360
|
+
if parts.path not in ("", "/"):
|
|
361
|
+
return url
|
|
362
|
+
return urlunsplit((parts.scheme, parts.netloc, DRIVENINGRESS_PATH, parts.query, parts.fragment))
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def close_code(reason: str) -> int | None:
|
|
366
|
+
"""Return the WebSocket close code a reason given to ``on_down`` carries, if any.
|
|
367
|
+
|
|
368
|
+
Args:
|
|
369
|
+
reason: The reason as ``on_down`` received it, for example ``creditsExhausted (4001)``.
|
|
370
|
+
|
|
371
|
+
Returns:
|
|
372
|
+
The code, or None when the reason names no close code (``closed``, ``send timed out``,
|
|
373
|
+
an error with no close behind it).
|
|
374
|
+
"""
|
|
375
|
+
_, found, tail = reason.rpartition(" (")
|
|
376
|
+
if not found or not tail.endswith(")") or not tail[:-1].isdigit():
|
|
377
|
+
return None
|
|
378
|
+
return int(tail[:-1])
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _describe(exc: Exception) -> str:
|
|
382
|
+
"""Return a short reason for a log line or ``on_down``, such as ``creditsExhausted (4001)``."""
|
|
383
|
+
if isinstance(exc, spatialreal.AvatarSDKError):
|
|
384
|
+
if exc.close_code is None:
|
|
385
|
+
return exc.code.value
|
|
386
|
+
return f"{exc.code.value} ({exc.close_code})"
|
|
387
|
+
return f"{type(exc).__name__}: {exc}"
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
"""Builders for the JSON envelope the processor sends to the SpatialReal web SDK.
|
|
2
|
+
|
|
3
|
+
Every builder returns the dict to pass as ``RTVIServerMessageFrame(data=...)``. The dict is an
|
|
4
|
+
``AvatarMessage`` from ``pipecat/v1/avatar.proto`` rendered under the contract's JSON rules: wire
|
|
5
|
+
keys are the proto field names, exactly one payload member appears beside ``tag``, and every key
|
|
6
|
+
inside that member is present ("unset" is ``""``, ``0``, ``[]``, ``false`` or the
|
|
7
|
+
``_UNSPECIFIED`` enum name). Bytes are standard base64, milliseconds are doubles and enums travel
|
|
8
|
+
as their names.
|
|
9
|
+
|
|
10
|
+
The builders fill a generated ``AvatarMessage`` and convert it with :func:`to_dict`, so the schema
|
|
11
|
+
alone decides the shape and no builder spells a wire key by hand. The builders only serialize: when
|
|
12
|
+
to send a message (never an empty ``turn_onsets``, never ``STATUS_UNSPECIFIED``) is the caller's
|
|
13
|
+
rule.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from google.protobuf.json_format import MessageToDict
|
|
19
|
+
|
|
20
|
+
from pipecat_spatialreal.proto.generated import avatar_pb2
|
|
21
|
+
|
|
22
|
+
# Value of AvatarMessage.tag; lets a client skip other server messages.
|
|
23
|
+
TAG = "spatialreal-avatar"
|
|
24
|
+
|
|
25
|
+
# Most bytes that one frames chunk or one turn_envelope message carries. Base64 turns 2400
|
|
26
|
+
# bytes into 3200 characters (2400 is a multiple of 3, so a full chunk has no padding); with the
|
|
27
|
+
# keys, a UUID turn id and the RTVI wrapper that is about 3.4 KB, inside Daily's 4 KB app-message
|
|
28
|
+
# cap.
|
|
29
|
+
MAX_CHUNK_BYTES = 2400
|
|
30
|
+
|
|
31
|
+
# Keyframes per second of the animation the web SDK renders.
|
|
32
|
+
FPS = 25
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def to_dict(msg: avatar_pb2.AvatarMessage) -> dict[str, Any]:
|
|
36
|
+
"""Render an ``AvatarMessage`` as the JSON-ready dict the contract describes.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
msg: The message to render.
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
A dict keyed by proto field names in which every field of the set payload member is
|
|
43
|
+
present, bytes are base64 strings and enums are their names.
|
|
44
|
+
"""
|
|
45
|
+
return MessageToDict(
|
|
46
|
+
msg,
|
|
47
|
+
preserving_proto_field_name=True,
|
|
48
|
+
always_print_fields_with_no_presence=True,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def turn_start(
|
|
53
|
+
turn_id: str,
|
|
54
|
+
*,
|
|
55
|
+
sample_rate: int,
|
|
56
|
+
onsets_ms: list[float],
|
|
57
|
+
prev_turn_id: str = "",
|
|
58
|
+
prev_turn_pcm_ms: float = 0.0,
|
|
59
|
+
degraded: bool = False,
|
|
60
|
+
gap_ms: float = 0.0,
|
|
61
|
+
) -> dict[str, Any]:
|
|
62
|
+
"""Build the message that announces a bot turn.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
turn_id: The driveningress req_id of the turn.
|
|
66
|
+
sample_rate: Hz of the audio sent to inference, which is also the room track's rate.
|
|
67
|
+
onsets_ms: Offsets in ms from the turn's first sample of the audible onsets found so
|
|
68
|
+
far, ascending; empty when none was found yet. A prior only: the browser places the
|
|
69
|
+
frames by them until the turn's envelope locks.
|
|
70
|
+
prev_turn_id: The previous turn when this one follows it without a silence gap, or an
|
|
71
|
+
empty string.
|
|
72
|
+
prev_turn_pcm_ms: Exact PCM length of ``prev_turn_id`` in ms; 0 when that is empty.
|
|
73
|
+
degraded: True when no animation arrived in time, so the client plays audio only.
|
|
74
|
+
gap_ms: Silence in ms the processor itself released between the previous turn's last
|
|
75
|
+
sample and this turn's first, so the browser can join the two envelopes; 0 for a
|
|
76
|
+
chained turn, the first turn, after an interruption and when the room ran dry.
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
The ``turn_start`` envelope, with ``fps`` set to :data:`FPS`.
|
|
80
|
+
"""
|
|
81
|
+
return to_dict(
|
|
82
|
+
avatar_pb2.AvatarMessage(
|
|
83
|
+
tag=TAG,
|
|
84
|
+
turn_start=avatar_pb2.TurnStart(
|
|
85
|
+
turn_id=turn_id,
|
|
86
|
+
sample_rate=sample_rate,
|
|
87
|
+
fps=FPS,
|
|
88
|
+
onsets_ms=onsets_ms,
|
|
89
|
+
prev_turn_id=prev_turn_id,
|
|
90
|
+
prev_turn_pcm_ms=prev_turn_pcm_ms,
|
|
91
|
+
degraded=degraded,
|
|
92
|
+
gap_ms=gap_ms,
|
|
93
|
+
),
|
|
94
|
+
)
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def turn_onsets(turn_id: str, onsets_ms: list[float]) -> dict[str, Any]:
|
|
99
|
+
"""Build the message that reports onsets found after ``turn_start``.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
turn_id: The driveningress req_id of the turn.
|
|
103
|
+
onsets_ms: The new onsets only, ascending, in ms from the turn's first sample; none
|
|
104
|
+
repeats an earlier entry.
|
|
105
|
+
|
|
106
|
+
Returns:
|
|
107
|
+
The ``turn_onsets`` envelope.
|
|
108
|
+
"""
|
|
109
|
+
return to_dict(
|
|
110
|
+
avatar_pb2.AvatarMessage(
|
|
111
|
+
tag=TAG,
|
|
112
|
+
turn_onsets=avatar_pb2.TurnOnsets(turn_id=turn_id, onsets_ms=onsets_ms),
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def turn_envelope(
|
|
118
|
+
turn_id: str,
|
|
119
|
+
first_index: int,
|
|
120
|
+
values: bytes,
|
|
121
|
+
*,
|
|
122
|
+
hop_ms: int = 5,
|
|
123
|
+
window_ms: int = 10,
|
|
124
|
+
) -> list[dict[str, Any]]:
|
|
125
|
+
"""Split a run of envelope values into ``turn_envelope`` messages that each fit an app message.
|
|
126
|
+
|
|
127
|
+
The values are one byte each, so a chunk of ``n`` bytes covers ``n`` consecutive indices.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
turn_id: The driveningress req_id of the turn.
|
|
131
|
+
first_index: 0-based index, counted from the turn's first released sample, of
|
|
132
|
+
``values[0]``.
|
|
133
|
+
values: The envelope values, one byte per index, contiguous and in order.
|
|
134
|
+
hop_ms: Spacing in ms between consecutive values.
|
|
135
|
+
window_ms: Length in ms of the window each value is measured over.
|
|
136
|
+
|
|
137
|
+
Returns:
|
|
138
|
+
One envelope per :data:`MAX_CHUNK_BYTES` of ``values`` in index order, each with its own
|
|
139
|
+
``first_index``; an empty list when ``values`` is empty.
|
|
140
|
+
"""
|
|
141
|
+
return [
|
|
142
|
+
to_dict(
|
|
143
|
+
avatar_pb2.AvatarMessage(
|
|
144
|
+
tag=TAG,
|
|
145
|
+
turn_envelope=avatar_pb2.TurnEnvelope(
|
|
146
|
+
turn_id=turn_id,
|
|
147
|
+
first_index=first_index + offset,
|
|
148
|
+
hop_ms=hop_ms,
|
|
149
|
+
window_ms=window_ms,
|
|
150
|
+
values=values[offset : offset + MAX_CHUNK_BYTES],
|
|
151
|
+
),
|
|
152
|
+
)
|
|
153
|
+
)
|
|
154
|
+
for offset in range(0, len(values), MAX_CHUNK_BYTES)
|
|
155
|
+
]
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def frames(
|
|
159
|
+
turn_id: str,
|
|
160
|
+
segment_seq: int,
|
|
161
|
+
wire: bytes,
|
|
162
|
+
*,
|
|
163
|
+
end: bool,
|
|
164
|
+
max_chunk_bytes: int = MAX_CHUNK_BYTES,
|
|
165
|
+
) -> list[dict[str, Any]]:
|
|
166
|
+
"""Split one driveningress message into ``frames`` envelopes that each fit one app message.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
turn_id: The driveningress req_id of the turn.
|
|
170
|
+
segment_seq: 0-based position of this segment within the turn.
|
|
171
|
+
wire: The raw ``driveningress.v2.Message`` bytes, exactly as driveningress sent them.
|
|
172
|
+
end: True when this is the turn's last segment; it is set on every chunk.
|
|
173
|
+
max_chunk_bytes: Most bytes of ``wire`` that one chunk carries.
|
|
174
|
+
|
|
175
|
+
Returns:
|
|
176
|
+
``ceil(len(wire) / max_chunk_bytes)`` envelopes in chunk order, all with the same
|
|
177
|
+
``chunk_count``. An empty ``wire`` still gives one chunk, with empty data.
|
|
178
|
+
|
|
179
|
+
Raises:
|
|
180
|
+
ValueError: If ``max_chunk_bytes`` is less than 1.
|
|
181
|
+
"""
|
|
182
|
+
if max_chunk_bytes < 1:
|
|
183
|
+
raise ValueError(f"max_chunk_bytes must be at least 1, got {max_chunk_bytes}")
|
|
184
|
+
pieces = [wire[i : i + max_chunk_bytes] for i in range(0, len(wire), max_chunk_bytes)]
|
|
185
|
+
if not pieces:
|
|
186
|
+
pieces = [b""] # chunk_count is at least 1: an empty wire travels as one empty chunk
|
|
187
|
+
return [
|
|
188
|
+
to_dict(
|
|
189
|
+
avatar_pb2.AvatarMessage(
|
|
190
|
+
tag=TAG,
|
|
191
|
+
frames=avatar_pb2.Frames(
|
|
192
|
+
turn_id=turn_id,
|
|
193
|
+
segment_seq=segment_seq,
|
|
194
|
+
chunk_index=index,
|
|
195
|
+
chunk_count=len(pieces),
|
|
196
|
+
data=piece,
|
|
197
|
+
end=end,
|
|
198
|
+
),
|
|
199
|
+
)
|
|
200
|
+
)
|
|
201
|
+
for index, piece in enumerate(pieces)
|
|
202
|
+
]
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def turn_end(turn_id: str, *, pcm_ms: float, segment_count: int) -> dict[str, Any]:
|
|
206
|
+
"""Build the message that says a turn's audio has finished playing out.
|
|
207
|
+
|
|
208
|
+
Args:
|
|
209
|
+
turn_id: The driveningress req_id of the turn.
|
|
210
|
+
pcm_ms: Exact length of the turn's audio in ms, stall silence included.
|
|
211
|
+
segment_count: Segments delivered for this turn; may be 0 for a degraded turn.
|
|
212
|
+
|
|
213
|
+
Returns:
|
|
214
|
+
The ``turn_end`` envelope.
|
|
215
|
+
"""
|
|
216
|
+
return to_dict(
|
|
217
|
+
avatar_pb2.AvatarMessage(
|
|
218
|
+
tag=TAG,
|
|
219
|
+
turn_end=avatar_pb2.TurnEnd(
|
|
220
|
+
turn_id=turn_id,
|
|
221
|
+
pcm_ms=pcm_ms,
|
|
222
|
+
segment_count=segment_count,
|
|
223
|
+
),
|
|
224
|
+
)
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def turn_abort(turn_id: str) -> dict[str, Any]:
|
|
229
|
+
"""Build the message that tells the client a turn was interrupted.
|
|
230
|
+
|
|
231
|
+
The client drops the turn's remaining frames and blends to idle at once.
|
|
232
|
+
|
|
233
|
+
Args:
|
|
234
|
+
turn_id: The driveningress req_id of the interrupted turn.
|
|
235
|
+
|
|
236
|
+
Returns:
|
|
237
|
+
The ``turn_abort`` envelope.
|
|
238
|
+
"""
|
|
239
|
+
return to_dict(
|
|
240
|
+
avatar_pb2.AvatarMessage(tag=TAG, turn_abort=avatar_pb2.TurnAbort(turn_id=turn_id))
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def state(status: int, reason: str = "") -> dict[str, Any]:
|
|
245
|
+
"""Build the message that reports a change in the processor's health.
|
|
246
|
+
|
|
247
|
+
Args:
|
|
248
|
+
status: An ``avatar_pb2.State.Status`` value, ``STATUS_OK`` or ``STATUS_DEGRADED``; the
|
|
249
|
+
JSON carries its name.
|
|
250
|
+
reason: Free text for logs.
|
|
251
|
+
|
|
252
|
+
Returns:
|
|
253
|
+
The ``state`` envelope.
|
|
254
|
+
"""
|
|
255
|
+
return to_dict(
|
|
256
|
+
avatar_pb2.AvatarMessage(tag=TAG, state=avatar_pb2.State(status=status, reason=reason))
|
|
257
|
+
)
|