dv-pipecat-ai 0.0.82.dev815__py3-none-any.whl → 0.0.82.dev857__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dv-pipecat-ai might be problematic. Click here for more details.
- {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/METADATA +8 -3
- {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/RECORD +106 -79
- pipecat/adapters/base_llm_adapter.py +44 -6
- pipecat/adapters/services/anthropic_adapter.py +302 -2
- pipecat/adapters/services/aws_nova_sonic_adapter.py +40 -2
- pipecat/adapters/services/bedrock_adapter.py +40 -2
- pipecat/adapters/services/gemini_adapter.py +276 -6
- pipecat/adapters/services/open_ai_adapter.py +88 -7
- pipecat/adapters/services/open_ai_realtime_adapter.py +39 -1
- pipecat/audio/dtmf/__init__.py +0 -0
- pipecat/audio/dtmf/types.py +47 -0
- pipecat/audio/dtmf/utils.py +70 -0
- pipecat/audio/filters/aic_filter.py +199 -0
- pipecat/audio/utils.py +9 -7
- pipecat/extensions/ivr/__init__.py +0 -0
- pipecat/extensions/ivr/ivr_navigator.py +452 -0
- pipecat/frames/frames.py +156 -43
- pipecat/pipeline/llm_switcher.py +76 -0
- pipecat/pipeline/parallel_pipeline.py +3 -3
- pipecat/pipeline/service_switcher.py +144 -0
- pipecat/pipeline/task.py +68 -28
- pipecat/pipeline/task_observer.py +10 -0
- pipecat/processors/aggregators/dtmf_aggregator.py +2 -2
- pipecat/processors/aggregators/llm_context.py +277 -0
- pipecat/processors/aggregators/llm_response.py +48 -15
- pipecat/processors/aggregators/llm_response_universal.py +840 -0
- pipecat/processors/aggregators/openai_llm_context.py +3 -3
- pipecat/processors/dtmf_aggregator.py +0 -2
- pipecat/processors/filters/stt_mute_filter.py +0 -2
- pipecat/processors/frame_processor.py +18 -11
- pipecat/processors/frameworks/rtvi.py +17 -10
- pipecat/processors/metrics/sentry.py +2 -0
- pipecat/runner/daily.py +137 -36
- pipecat/runner/run.py +1 -1
- pipecat/runner/utils.py +7 -7
- pipecat/serializers/asterisk.py +20 -4
- pipecat/serializers/exotel.py +1 -1
- pipecat/serializers/plivo.py +1 -1
- pipecat/serializers/telnyx.py +1 -1
- pipecat/serializers/twilio.py +1 -1
- pipecat/services/__init__.py +2 -2
- pipecat/services/anthropic/llm.py +113 -28
- pipecat/services/asyncai/tts.py +4 -0
- pipecat/services/aws/llm.py +82 -8
- pipecat/services/aws/tts.py +0 -10
- pipecat/services/aws_nova_sonic/aws.py +5 -0
- pipecat/services/cartesia/tts.py +28 -16
- pipecat/services/cerebras/llm.py +15 -10
- pipecat/services/deepgram/stt.py +8 -0
- pipecat/services/deepseek/llm.py +13 -8
- pipecat/services/fireworks/llm.py +13 -8
- pipecat/services/fish/tts.py +8 -6
- pipecat/services/gemini_multimodal_live/gemini.py +5 -0
- pipecat/services/gladia/config.py +7 -1
- pipecat/services/gladia/stt.py +23 -15
- pipecat/services/google/llm.py +159 -59
- pipecat/services/google/llm_openai.py +18 -3
- pipecat/services/grok/llm.py +2 -1
- pipecat/services/llm_service.py +38 -3
- pipecat/services/mem0/memory.py +2 -1
- pipecat/services/mistral/llm.py +5 -6
- pipecat/services/nim/llm.py +2 -1
- pipecat/services/openai/base_llm.py +88 -26
- pipecat/services/openai/image.py +6 -1
- pipecat/services/openai_realtime_beta/openai.py +5 -2
- pipecat/services/openpipe/llm.py +6 -8
- pipecat/services/perplexity/llm.py +13 -8
- pipecat/services/playht/tts.py +9 -6
- pipecat/services/rime/tts.py +1 -1
- pipecat/services/sambanova/llm.py +18 -13
- pipecat/services/sarvam/tts.py +415 -10
- pipecat/services/speechmatics/stt.py +2 -2
- pipecat/services/tavus/video.py +1 -1
- pipecat/services/tts_service.py +15 -5
- pipecat/services/vistaar/llm.py +2 -5
- pipecat/transports/base_input.py +32 -19
- pipecat/transports/base_output.py +39 -5
- pipecat/transports/daily/__init__.py +0 -0
- pipecat/transports/daily/transport.py +2371 -0
- pipecat/transports/daily/utils.py +410 -0
- pipecat/transports/livekit/__init__.py +0 -0
- pipecat/transports/livekit/transport.py +1042 -0
- pipecat/transports/network/fastapi_websocket.py +12 -546
- pipecat/transports/network/small_webrtc.py +12 -922
- pipecat/transports/network/webrtc_connection.py +9 -595
- pipecat/transports/network/websocket_client.py +12 -481
- pipecat/transports/network/websocket_server.py +12 -487
- pipecat/transports/services/daily.py +9 -2334
- pipecat/transports/services/helpers/daily_rest.py +12 -396
- pipecat/transports/services/livekit.py +12 -975
- pipecat/transports/services/tavus.py +12 -757
- pipecat/transports/smallwebrtc/__init__.py +0 -0
- pipecat/transports/smallwebrtc/connection.py +612 -0
- pipecat/transports/smallwebrtc/transport.py +936 -0
- pipecat/transports/tavus/__init__.py +0 -0
- pipecat/transports/tavus/transport.py +770 -0
- pipecat/transports/websocket/__init__.py +0 -0
- pipecat/transports/websocket/client.py +494 -0
- pipecat/transports/websocket/fastapi.py +559 -0
- pipecat/transports/websocket/server.py +500 -0
- pipecat/transports/whatsapp/__init__.py +0 -0
- pipecat/transports/whatsapp/api.py +345 -0
- pipecat/transports/whatsapp/client.py +364 -0
- {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/WHEEL +0 -0
- {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/licenses/LICENSE +0 -0
- {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/top_level.txt +0 -0
|
@@ -11,760 +11,15 @@ AI applications with avatars. It manages conversation sessions and provides real
|
|
|
11
11
|
audio/video streaming capabilities through the Tavus API.
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
|
-
import
|
|
15
|
-
|
|
16
|
-
from
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
Frame,
|
|
27
|
-
InputAudioRawFrame,
|
|
28
|
-
OutputAudioRawFrame,
|
|
29
|
-
StartFrame,
|
|
30
|
-
StartInterruptionFrame,
|
|
31
|
-
TransportMessageFrame,
|
|
32
|
-
TransportMessageUrgentFrame,
|
|
33
|
-
)
|
|
34
|
-
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
|
|
35
|
-
from pipecat.transports.base_input import BaseInputTransport
|
|
36
|
-
from pipecat.transports.base_output import BaseOutputTransport
|
|
37
|
-
from pipecat.transports.base_transport import BaseTransport, TransportParams
|
|
38
|
-
from pipecat.transports.services.daily import (
|
|
39
|
-
DailyCallbacks,
|
|
40
|
-
DailyParams,
|
|
41
|
-
DailyTransportClient,
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
class TavusApi:
|
|
46
|
-
"""Helper class for interacting with the Tavus API (v2).
|
|
47
|
-
|
|
48
|
-
Provides methods for creating and managing conversations with Tavus avatars,
|
|
49
|
-
including conversation lifecycle management and persona information retrieval.
|
|
50
|
-
"""
|
|
51
|
-
|
|
52
|
-
BASE_URL = "https://tavusapi.com/v2"
|
|
53
|
-
MOCK_CONVERSATION_ID = "dev-conversation"
|
|
54
|
-
MOCK_PERSONA_NAME = "TestTavusTransport"
|
|
55
|
-
|
|
56
|
-
def __init__(self, api_key: str, session: aiohttp.ClientSession):
|
|
57
|
-
"""Initialize the TavusApi client.
|
|
58
|
-
|
|
59
|
-
Args:
|
|
60
|
-
api_key: Tavus API key for authentication.
|
|
61
|
-
session: An aiohttp session for making HTTP requests.
|
|
62
|
-
"""
|
|
63
|
-
self._api_key = api_key
|
|
64
|
-
self._session = session
|
|
65
|
-
self._headers = {"Content-Type": "application/json", "x-api-key": self._api_key}
|
|
66
|
-
# Only for development
|
|
67
|
-
self._dev_room_url = os.getenv("TAVUS_SAMPLE_ROOM_URL")
|
|
68
|
-
|
|
69
|
-
async def create_conversation(self, replica_id: str, persona_id: str) -> dict:
|
|
70
|
-
"""Create a new conversation with the specified replica and persona.
|
|
71
|
-
|
|
72
|
-
Args:
|
|
73
|
-
replica_id: ID of the replica to use in the conversation.
|
|
74
|
-
persona_id: ID of the persona to use in the conversation.
|
|
75
|
-
|
|
76
|
-
Returns:
|
|
77
|
-
Dictionary containing conversation_id and conversation_url.
|
|
78
|
-
"""
|
|
79
|
-
if self._dev_room_url:
|
|
80
|
-
return {
|
|
81
|
-
"conversation_id": self.MOCK_CONVERSATION_ID,
|
|
82
|
-
"conversation_url": self._dev_room_url,
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
logger.debug(f"Creating Tavus conversation: replica={replica_id}, persona={persona_id}")
|
|
86
|
-
url = f"{self.BASE_URL}/conversations"
|
|
87
|
-
payload = {
|
|
88
|
-
"replica_id": replica_id,
|
|
89
|
-
"persona_id": persona_id,
|
|
90
|
-
}
|
|
91
|
-
async with self._session.post(url, headers=self._headers, json=payload) as r:
|
|
92
|
-
r.raise_for_status()
|
|
93
|
-
response = await r.json()
|
|
94
|
-
logger.debug(f"Created Tavus conversation: {response}")
|
|
95
|
-
return response
|
|
96
|
-
|
|
97
|
-
async def end_conversation(self, conversation_id: str):
|
|
98
|
-
"""End an existing conversation.
|
|
99
|
-
|
|
100
|
-
Args:
|
|
101
|
-
conversation_id: ID of the conversation to end.
|
|
102
|
-
"""
|
|
103
|
-
if conversation_id is None or conversation_id == self.MOCK_CONVERSATION_ID:
|
|
104
|
-
return
|
|
105
|
-
|
|
106
|
-
url = f"{self.BASE_URL}/conversations/{conversation_id}/end"
|
|
107
|
-
async with self._session.post(url, headers=self._headers) as r:
|
|
108
|
-
r.raise_for_status()
|
|
109
|
-
logger.debug(f"Ended Tavus conversation {conversation_id}")
|
|
110
|
-
|
|
111
|
-
async def get_persona_name(self, persona_id: str) -> str:
|
|
112
|
-
"""Get the name of a persona by ID.
|
|
113
|
-
|
|
114
|
-
Args:
|
|
115
|
-
persona_id: ID of the persona to retrieve.
|
|
116
|
-
|
|
117
|
-
Returns:
|
|
118
|
-
The name of the persona.
|
|
119
|
-
"""
|
|
120
|
-
if self._dev_room_url is not None:
|
|
121
|
-
return self.MOCK_PERSONA_NAME
|
|
122
|
-
|
|
123
|
-
url = f"{self.BASE_URL}/personas/{persona_id}"
|
|
124
|
-
async with self._session.get(url, headers=self._headers) as r:
|
|
125
|
-
r.raise_for_status()
|
|
126
|
-
response = await r.json()
|
|
127
|
-
logger.debug(f"Fetched Tavus persona: {response}")
|
|
128
|
-
return response["persona_name"]
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
class TavusCallbacks(BaseModel):
|
|
132
|
-
"""Callback handlers for Tavus events.
|
|
133
|
-
|
|
134
|
-
Parameters:
|
|
135
|
-
on_participant_joined: Called when a participant joins the conversation.
|
|
136
|
-
on_participant_left: Called when a participant leaves the conversation.
|
|
137
|
-
"""
|
|
138
|
-
|
|
139
|
-
on_participant_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
|
|
140
|
-
on_participant_left: Callable[[Mapping[str, Any], str], Awaitable[None]]
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
class TavusParams(DailyParams):
|
|
144
|
-
"""Configuration parameters for the Tavus transport.
|
|
145
|
-
|
|
146
|
-
Parameters:
|
|
147
|
-
audio_in_enabled: Whether to enable audio input from participants.
|
|
148
|
-
audio_out_enabled: Whether to enable audio output to participants.
|
|
149
|
-
microphone_out_enabled: Whether to enable microphone output track.
|
|
150
|
-
"""
|
|
151
|
-
|
|
152
|
-
audio_in_enabled: bool = True
|
|
153
|
-
audio_out_enabled: bool = True
|
|
154
|
-
microphone_out_enabled: bool = False
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
class TavusTransportClient:
|
|
158
|
-
"""Transport client that integrates Pipecat with the Tavus platform.
|
|
159
|
-
|
|
160
|
-
A transport client that integrates a Pipecat Bot with the Tavus platform by managing
|
|
161
|
-
conversation sessions using the Tavus API.
|
|
162
|
-
|
|
163
|
-
This client uses `TavusApi` to interact with the Tavus backend services. When a conversation
|
|
164
|
-
is started via `TavusApi`, Tavus provides a `roomURL` that can be used to connect the Pipecat Bot
|
|
165
|
-
into the same virtual room where the TavusBot is operating.
|
|
166
|
-
"""
|
|
167
|
-
|
|
168
|
-
def __init__(
|
|
169
|
-
self,
|
|
170
|
-
*,
|
|
171
|
-
bot_name: str,
|
|
172
|
-
params: TavusParams = TavusParams(),
|
|
173
|
-
callbacks: TavusCallbacks,
|
|
174
|
-
api_key: str,
|
|
175
|
-
replica_id: str,
|
|
176
|
-
persona_id: str = "pipecat-stream",
|
|
177
|
-
session: aiohttp.ClientSession,
|
|
178
|
-
) -> None:
|
|
179
|
-
"""Initialize the Tavus transport client.
|
|
180
|
-
|
|
181
|
-
Args:
|
|
182
|
-
bot_name: The name of the Pipecat bot instance.
|
|
183
|
-
params: Optional parameters for Tavus operation.
|
|
184
|
-
callbacks: Callback handlers for Tavus-related events.
|
|
185
|
-
api_key: API key for authenticating with Tavus API.
|
|
186
|
-
replica_id: ID of the replica to use in the Tavus conversation.
|
|
187
|
-
persona_id: ID of the Tavus persona. Defaults to "pipecat-stream",
|
|
188
|
-
which signals Tavus to use the TTS voice of the Pipecat bot
|
|
189
|
-
instead of a Tavus persona voice.
|
|
190
|
-
session: The aiohttp session for making async HTTP requests.
|
|
191
|
-
"""
|
|
192
|
-
self._bot_name = bot_name
|
|
193
|
-
self._api = TavusApi(api_key, session)
|
|
194
|
-
self._replica_id = replica_id
|
|
195
|
-
self._persona_id = persona_id
|
|
196
|
-
self._conversation_id: Optional[str] = None
|
|
197
|
-
self._client: Optional[DailyTransportClient] = None
|
|
198
|
-
self._callbacks = callbacks
|
|
199
|
-
self._params = params
|
|
200
|
-
|
|
201
|
-
async def _initialize(self) -> str:
|
|
202
|
-
"""Initialize the conversation and return the room URL."""
|
|
203
|
-
response = await self._api.create_conversation(self._replica_id, self._persona_id)
|
|
204
|
-
self._conversation_id = response["conversation_id"]
|
|
205
|
-
return response["conversation_url"]
|
|
206
|
-
|
|
207
|
-
async def setup(self, setup: FrameProcessorSetup):
|
|
208
|
-
"""Setup the client and initialize the conversation.
|
|
209
|
-
|
|
210
|
-
Args:
|
|
211
|
-
setup: The frame processor setup configuration.
|
|
212
|
-
"""
|
|
213
|
-
if self._conversation_id is not None:
|
|
214
|
-
logger.debug(f"Conversation ID already defined: {self._conversation_id}")
|
|
215
|
-
return
|
|
216
|
-
try:
|
|
217
|
-
room_url = await self._initialize()
|
|
218
|
-
daily_callbacks = DailyCallbacks(
|
|
219
|
-
on_active_speaker_changed=partial(
|
|
220
|
-
self._on_handle_callback, "on_active_speaker_changed"
|
|
221
|
-
),
|
|
222
|
-
on_joined=self._on_joined,
|
|
223
|
-
on_left=self._on_left,
|
|
224
|
-
on_error=partial(self._on_handle_callback, "on_error"),
|
|
225
|
-
on_app_message=partial(self._on_handle_callback, "on_app_message"),
|
|
226
|
-
on_call_state_updated=partial(self._on_handle_callback, "on_call_state_updated"),
|
|
227
|
-
on_client_connected=partial(self._on_handle_callback, "on_client_connected"),
|
|
228
|
-
on_client_disconnected=partial(self._on_handle_callback, "on_client_disconnected"),
|
|
229
|
-
on_dialin_connected=partial(self._on_handle_callback, "on_dialin_connected"),
|
|
230
|
-
on_dialin_ready=partial(self._on_handle_callback, "on_dialin_ready"),
|
|
231
|
-
on_dialin_stopped=partial(self._on_handle_callback, "on_dialin_stopped"),
|
|
232
|
-
on_dialin_error=partial(self._on_handle_callback, "on_dialin_error"),
|
|
233
|
-
on_dialin_warning=partial(self._on_handle_callback, "on_dialin_warning"),
|
|
234
|
-
on_dialout_answered=partial(self._on_handle_callback, "on_dialout_answered"),
|
|
235
|
-
on_dialout_connected=partial(self._on_handle_callback, "on_dialout_connected"),
|
|
236
|
-
on_dialout_stopped=partial(self._on_handle_callback, "on_dialout_stopped"),
|
|
237
|
-
on_dialout_error=partial(self._on_handle_callback, "on_dialout_error"),
|
|
238
|
-
on_dialout_warning=partial(self._on_handle_callback, "on_dialout_warning"),
|
|
239
|
-
on_participant_joined=self._callbacks.on_participant_joined,
|
|
240
|
-
on_participant_left=self._callbacks.on_participant_left,
|
|
241
|
-
on_participant_updated=partial(self._on_handle_callback, "on_participant_updated"),
|
|
242
|
-
on_transcription_message=partial(
|
|
243
|
-
self._on_handle_callback, "on_transcription_message"
|
|
244
|
-
),
|
|
245
|
-
on_recording_started=partial(self._on_handle_callback, "on_recording_started"),
|
|
246
|
-
on_recording_stopped=partial(self._on_handle_callback, "on_recording_stopped"),
|
|
247
|
-
on_recording_error=partial(self._on_handle_callback, "on_recording_error"),
|
|
248
|
-
on_transcription_stopped=partial(
|
|
249
|
-
self._on_handle_callback, "on_transcription_stopped"
|
|
250
|
-
),
|
|
251
|
-
on_transcription_error=partial(self._on_handle_callback, "on_transcription_error"),
|
|
252
|
-
)
|
|
253
|
-
self._client = DailyTransportClient(
|
|
254
|
-
room_url, None, "Pipecat", self._params, daily_callbacks, self._bot_name
|
|
255
|
-
)
|
|
256
|
-
await self._client.setup(setup)
|
|
257
|
-
except Exception as e:
|
|
258
|
-
logger.error(f"Failed to setup TavusTransportClient: {e}")
|
|
259
|
-
await self._api.end_conversation(self._conversation_id)
|
|
260
|
-
self._conversation_id = None
|
|
261
|
-
|
|
262
|
-
async def cleanup(self):
|
|
263
|
-
"""Cleanup client resources."""
|
|
264
|
-
try:
|
|
265
|
-
await self._client.cleanup()
|
|
266
|
-
except Exception as e:
|
|
267
|
-
logger.exception(f"Exception during cleanup: {e}")
|
|
268
|
-
|
|
269
|
-
async def _on_joined(self, data):
|
|
270
|
-
"""Handle joined event."""
|
|
271
|
-
logger.debug("TavusTransportClient joined!")
|
|
272
|
-
|
|
273
|
-
async def _on_left(self):
|
|
274
|
-
"""Handle left event."""
|
|
275
|
-
logger.debug("TavusTransportClient left!")
|
|
276
|
-
|
|
277
|
-
async def _on_handle_callback(self, event_name, *args, **kwargs):
|
|
278
|
-
"""Handle generic callback events."""
|
|
279
|
-
logger.trace(f"[Callback] {event_name} called with args={args}, kwargs={kwargs}")
|
|
280
|
-
|
|
281
|
-
async def get_persona_name(self) -> str:
|
|
282
|
-
"""Get the persona name from the API.
|
|
283
|
-
|
|
284
|
-
Returns:
|
|
285
|
-
The name of the current persona.
|
|
286
|
-
"""
|
|
287
|
-
return await self._api.get_persona_name(self._persona_id)
|
|
288
|
-
|
|
289
|
-
async def start(self, frame: StartFrame):
|
|
290
|
-
"""Start the client and join the room.
|
|
291
|
-
|
|
292
|
-
Args:
|
|
293
|
-
frame: The start frame containing initialization parameters.
|
|
294
|
-
"""
|
|
295
|
-
logger.debug("TavusTransportClient start invoked!")
|
|
296
|
-
await self._client.start(frame)
|
|
297
|
-
await self._client.join()
|
|
298
|
-
|
|
299
|
-
async def stop(self):
|
|
300
|
-
"""Stop the client and end the conversation."""
|
|
301
|
-
await self._client.leave()
|
|
302
|
-
await self._api.end_conversation(self._conversation_id)
|
|
303
|
-
self._conversation_id = None
|
|
304
|
-
|
|
305
|
-
async def capture_participant_video(
|
|
306
|
-
self,
|
|
307
|
-
participant_id: str,
|
|
308
|
-
callback: Callable,
|
|
309
|
-
framerate: int = 30,
|
|
310
|
-
video_source: str = "camera",
|
|
311
|
-
color_format: str = "RGB",
|
|
312
|
-
):
|
|
313
|
-
"""Capture video from a participant.
|
|
314
|
-
|
|
315
|
-
Args:
|
|
316
|
-
participant_id: ID of the participant to capture video from.
|
|
317
|
-
callback: Callback function to handle video frames.
|
|
318
|
-
framerate: Desired framerate for video capture.
|
|
319
|
-
video_source: Video source to capture from.
|
|
320
|
-
color_format: Color format for video frames.
|
|
321
|
-
"""
|
|
322
|
-
await self._client.capture_participant_video(
|
|
323
|
-
participant_id, callback, framerate, video_source, color_format
|
|
324
|
-
)
|
|
325
|
-
|
|
326
|
-
async def capture_participant_audio(
|
|
327
|
-
self,
|
|
328
|
-
participant_id: str,
|
|
329
|
-
callback: Callable,
|
|
330
|
-
audio_source: str = "microphone",
|
|
331
|
-
sample_rate: int = 16000,
|
|
332
|
-
callback_interval_ms: int = 20,
|
|
333
|
-
):
|
|
334
|
-
"""Capture audio from a participant.
|
|
335
|
-
|
|
336
|
-
Args:
|
|
337
|
-
participant_id: ID of the participant to capture audio from.
|
|
338
|
-
callback: Callback function to handle audio data.
|
|
339
|
-
audio_source: Audio source to capture from.
|
|
340
|
-
sample_rate: Desired sample rate for audio capture.
|
|
341
|
-
callback_interval_ms: Interval between audio callbacks in milliseconds.
|
|
342
|
-
"""
|
|
343
|
-
await self._client.capture_participant_audio(
|
|
344
|
-
participant_id, callback, audio_source, sample_rate, callback_interval_ms
|
|
345
|
-
)
|
|
346
|
-
|
|
347
|
-
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
|
348
|
-
"""Send a message to participants.
|
|
349
|
-
|
|
350
|
-
Args:
|
|
351
|
-
frame: The message frame to send.
|
|
352
|
-
"""
|
|
353
|
-
await self._client.send_message(frame)
|
|
354
|
-
|
|
355
|
-
@property
|
|
356
|
-
def out_sample_rate(self) -> int:
|
|
357
|
-
"""Get the output sample rate.
|
|
358
|
-
|
|
359
|
-
Returns:
|
|
360
|
-
The output sample rate in Hz.
|
|
361
|
-
"""
|
|
362
|
-
return self._client.out_sample_rate
|
|
363
|
-
|
|
364
|
-
@property
|
|
365
|
-
def in_sample_rate(self) -> int:
|
|
366
|
-
"""Get the input sample rate.
|
|
367
|
-
|
|
368
|
-
Returns:
|
|
369
|
-
The input sample rate in Hz.
|
|
370
|
-
"""
|
|
371
|
-
return self._client.in_sample_rate
|
|
372
|
-
|
|
373
|
-
async def send_interrupt_message(self) -> None:
|
|
374
|
-
"""Send an interrupt message to the conversation."""
|
|
375
|
-
transport_frame = TransportMessageUrgentFrame(
|
|
376
|
-
message={
|
|
377
|
-
"message_type": "conversation",
|
|
378
|
-
"event_type": "conversation.interrupt",
|
|
379
|
-
"conversation_id": self._conversation_id,
|
|
380
|
-
}
|
|
381
|
-
)
|
|
382
|
-
await self.send_message(transport_frame)
|
|
383
|
-
|
|
384
|
-
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
|
|
385
|
-
"""Update subscription settings for participants.
|
|
386
|
-
|
|
387
|
-
Args:
|
|
388
|
-
participant_settings: Per-participant subscription settings.
|
|
389
|
-
profile_settings: Global subscription profile settings.
|
|
390
|
-
"""
|
|
391
|
-
if not self._client:
|
|
392
|
-
return
|
|
393
|
-
|
|
394
|
-
await self._client.update_subscriptions(
|
|
395
|
-
participant_settings=participant_settings, profile_settings=profile_settings
|
|
396
|
-
)
|
|
397
|
-
|
|
398
|
-
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
|
399
|
-
"""Write an audio frame to the transport.
|
|
400
|
-
|
|
401
|
-
Args:
|
|
402
|
-
frame: The audio frame to write.
|
|
403
|
-
"""
|
|
404
|
-
if not self._client:
|
|
405
|
-
return
|
|
406
|
-
await self._client.write_audio_frame(frame)
|
|
407
|
-
|
|
408
|
-
async def register_audio_destination(self, destination: str):
|
|
409
|
-
"""Register an audio destination for output.
|
|
410
|
-
|
|
411
|
-
Args:
|
|
412
|
-
destination: The destination identifier to register.
|
|
413
|
-
"""
|
|
414
|
-
if not self._client:
|
|
415
|
-
return
|
|
416
|
-
|
|
417
|
-
await self._client.register_audio_destination(destination)
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
class TavusInputTransport(BaseInputTransport):
|
|
421
|
-
"""Input transport for receiving audio and events from Tavus conversations.
|
|
422
|
-
|
|
423
|
-
Handles incoming audio streams from participants and manages audio capture
|
|
424
|
-
from the Daily room connected to the Tavus conversation.
|
|
425
|
-
"""
|
|
426
|
-
|
|
427
|
-
def __init__(
|
|
428
|
-
self,
|
|
429
|
-
client: TavusTransportClient,
|
|
430
|
-
params: TransportParams,
|
|
431
|
-
**kwargs,
|
|
432
|
-
):
|
|
433
|
-
"""Initialize the Tavus input transport.
|
|
434
|
-
|
|
435
|
-
Args:
|
|
436
|
-
client: The Tavus transport client instance.
|
|
437
|
-
params: Transport configuration parameters.
|
|
438
|
-
**kwargs: Additional arguments passed to parent class.
|
|
439
|
-
"""
|
|
440
|
-
super().__init__(params, **kwargs)
|
|
441
|
-
self._client = client
|
|
442
|
-
self._params = params
|
|
443
|
-
# Whether we have seen a StartFrame already.
|
|
444
|
-
self._initialized = False
|
|
445
|
-
|
|
446
|
-
async def setup(self, setup: FrameProcessorSetup):
|
|
447
|
-
"""Setup the input transport.
|
|
448
|
-
|
|
449
|
-
Args:
|
|
450
|
-
setup: The frame processor setup configuration.
|
|
451
|
-
"""
|
|
452
|
-
await super().setup(setup)
|
|
453
|
-
await self._client.setup(setup)
|
|
454
|
-
|
|
455
|
-
async def cleanup(self):
|
|
456
|
-
"""Cleanup input transport resources."""
|
|
457
|
-
await super().cleanup()
|
|
458
|
-
await self._client.cleanup()
|
|
459
|
-
|
|
460
|
-
async def start(self, frame: StartFrame):
|
|
461
|
-
"""Start the input transport.
|
|
462
|
-
|
|
463
|
-
Args:
|
|
464
|
-
frame: The start frame containing initialization parameters.
|
|
465
|
-
"""
|
|
466
|
-
await super().start(frame)
|
|
467
|
-
|
|
468
|
-
if self._initialized:
|
|
469
|
-
return
|
|
470
|
-
|
|
471
|
-
self._initialized = True
|
|
472
|
-
|
|
473
|
-
await self._client.start(frame)
|
|
474
|
-
await self.set_transport_ready(frame)
|
|
475
|
-
|
|
476
|
-
async def stop(self, frame: EndFrame):
|
|
477
|
-
"""Stop the input transport.
|
|
478
|
-
|
|
479
|
-
Args:
|
|
480
|
-
frame: The end frame signaling transport shutdown.
|
|
481
|
-
"""
|
|
482
|
-
await super().stop(frame)
|
|
483
|
-
await self._client.stop()
|
|
484
|
-
|
|
485
|
-
async def cancel(self, frame: CancelFrame):
|
|
486
|
-
"""Cancel the input transport.
|
|
487
|
-
|
|
488
|
-
Args:
|
|
489
|
-
frame: The cancel frame signaling immediate cancellation.
|
|
490
|
-
"""
|
|
491
|
-
await super().cancel(frame)
|
|
492
|
-
await self._client.stop()
|
|
493
|
-
|
|
494
|
-
async def start_capturing_audio(self, participant):
|
|
495
|
-
"""Start capturing audio from a participant.
|
|
496
|
-
|
|
497
|
-
Args:
|
|
498
|
-
participant: The participant to capture audio from.
|
|
499
|
-
"""
|
|
500
|
-
if self._params.audio_in_enabled:
|
|
501
|
-
logger.info(
|
|
502
|
-
f"TavusTransportClient start capturing audio for participant {participant['id']}"
|
|
503
|
-
)
|
|
504
|
-
await self._client.capture_participant_audio(
|
|
505
|
-
participant_id=participant["id"],
|
|
506
|
-
callback=self._on_participant_audio_data,
|
|
507
|
-
sample_rate=self._client.in_sample_rate,
|
|
508
|
-
)
|
|
509
|
-
|
|
510
|
-
async def _on_participant_audio_data(
|
|
511
|
-
self, participant_id: str, audio: AudioData, audio_source: str
|
|
512
|
-
):
|
|
513
|
-
"""Handle received participant audio data."""
|
|
514
|
-
frame = InputAudioRawFrame(
|
|
515
|
-
audio=audio.audio_frames,
|
|
516
|
-
sample_rate=audio.audio_frames,
|
|
517
|
-
num_channels=audio.num_channels,
|
|
518
|
-
)
|
|
519
|
-
frame.transport_source = audio_source
|
|
520
|
-
await self.push_audio_frame(frame)
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
class TavusOutputTransport(BaseOutputTransport):
|
|
524
|
-
"""Output transport for sending audio and events to Tavus conversations.
|
|
525
|
-
|
|
526
|
-
Handles outgoing audio streams to participants and manages the custom
|
|
527
|
-
audio track expected by the Tavus platform.
|
|
528
|
-
"""
|
|
529
|
-
|
|
530
|
-
def __init__(
|
|
531
|
-
self,
|
|
532
|
-
client: TavusTransportClient,
|
|
533
|
-
params: TransportParams,
|
|
534
|
-
**kwargs,
|
|
535
|
-
):
|
|
536
|
-
"""Initialize the Tavus output transport.
|
|
537
|
-
|
|
538
|
-
Args:
|
|
539
|
-
client: The Tavus transport client instance.
|
|
540
|
-
params: Transport configuration parameters.
|
|
541
|
-
**kwargs: Additional arguments passed to parent class.
|
|
542
|
-
"""
|
|
543
|
-
super().__init__(params, **kwargs)
|
|
544
|
-
self._client = client
|
|
545
|
-
self._params = params
|
|
546
|
-
|
|
547
|
-
# Whether we have seen a StartFrame already.
|
|
548
|
-
self._initialized = False
|
|
549
|
-
# This is the custom track destination expected by Tavus
|
|
550
|
-
self._transport_destination: Optional[str] = "stream"
|
|
551
|
-
|
|
552
|
-
async def setup(self, setup: FrameProcessorSetup):
|
|
553
|
-
"""Setup the output transport.
|
|
554
|
-
|
|
555
|
-
Args:
|
|
556
|
-
setup: The frame processor setup configuration.
|
|
557
|
-
"""
|
|
558
|
-
await super().setup(setup)
|
|
559
|
-
await self._client.setup(setup)
|
|
560
|
-
|
|
561
|
-
async def cleanup(self):
|
|
562
|
-
"""Cleanup output transport resources."""
|
|
563
|
-
await super().cleanup()
|
|
564
|
-
await self._client.cleanup()
|
|
565
|
-
|
|
566
|
-
async def start(self, frame: StartFrame):
|
|
567
|
-
"""Start the output transport.
|
|
568
|
-
|
|
569
|
-
Args:
|
|
570
|
-
frame: The start frame containing initialization parameters.
|
|
571
|
-
"""
|
|
572
|
-
await super().start(frame)
|
|
573
|
-
|
|
574
|
-
if self._initialized:
|
|
575
|
-
return
|
|
576
|
-
|
|
577
|
-
self._initialized = True
|
|
578
|
-
|
|
579
|
-
await self._client.start(frame)
|
|
580
|
-
|
|
581
|
-
if self._transport_destination:
|
|
582
|
-
await self._client.register_audio_destination(self._transport_destination)
|
|
583
|
-
|
|
584
|
-
await self.set_transport_ready(frame)
|
|
585
|
-
|
|
586
|
-
async def stop(self, frame: EndFrame):
|
|
587
|
-
"""Stop the output transport.
|
|
588
|
-
|
|
589
|
-
Args:
|
|
590
|
-
frame: The end frame signaling transport shutdown.
|
|
591
|
-
"""
|
|
592
|
-
await super().stop(frame)
|
|
593
|
-
await self._client.stop()
|
|
594
|
-
|
|
595
|
-
async def cancel(self, frame: CancelFrame):
|
|
596
|
-
"""Cancel the output transport.
|
|
597
|
-
|
|
598
|
-
Args:
|
|
599
|
-
frame: The cancel frame signaling immediate cancellation.
|
|
600
|
-
"""
|
|
601
|
-
await super().cancel(frame)
|
|
602
|
-
await self._client.stop()
|
|
603
|
-
|
|
604
|
-
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
|
605
|
-
"""Send a message to participants.
|
|
606
|
-
|
|
607
|
-
Args:
|
|
608
|
-
frame: The message frame to send.
|
|
609
|
-
"""
|
|
610
|
-
logger.info(f"TavusOutputTransport sending message {frame}")
|
|
611
|
-
await self._client.send_message(frame)
|
|
612
|
-
|
|
613
|
-
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
|
614
|
-
"""Process frames and handle interruptions.
|
|
615
|
-
|
|
616
|
-
Args:
|
|
617
|
-
frame: The frame to process.
|
|
618
|
-
direction: The direction of frame flow in the pipeline.
|
|
619
|
-
"""
|
|
620
|
-
await super().process_frame(frame, direction)
|
|
621
|
-
if isinstance(frame, StartInterruptionFrame):
|
|
622
|
-
await self._handle_interruptions()
|
|
623
|
-
|
|
624
|
-
async def _handle_interruptions(self):
|
|
625
|
-
"""Handle interruption events by sending interrupt message."""
|
|
626
|
-
await self._client.send_interrupt_message()
|
|
627
|
-
|
|
628
|
-
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
|
629
|
-
"""Write an audio frame to the Tavus transport.
|
|
630
|
-
|
|
631
|
-
Args:
|
|
632
|
-
frame: The audio frame to write.
|
|
633
|
-
"""
|
|
634
|
-
# This is the custom track destination expected by Tavus
|
|
635
|
-
frame.transport_destination = self._transport_destination
|
|
636
|
-
await self._client.write_audio_frame(frame)
|
|
637
|
-
|
|
638
|
-
async def register_audio_destination(self, destination: str):
|
|
639
|
-
"""Register an audio destination.
|
|
640
|
-
|
|
641
|
-
Args:
|
|
642
|
-
destination: The destination identifier to register.
|
|
643
|
-
"""
|
|
644
|
-
await self._client.register_audio_destination(destination)
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
class TavusTransport(BaseTransport):
|
|
648
|
-
"""Transport implementation for Tavus video calls.
|
|
649
|
-
|
|
650
|
-
When used, the Pipecat bot joins the same virtual room as the Tavus Avatar and the user.
|
|
651
|
-
This is achieved by using `TavusTransportClient`, which initiates the conversation via
|
|
652
|
-
`TavusApi` and obtains a room URL that all participants connect to.
|
|
653
|
-
"""
|
|
654
|
-
|
|
655
|
-
def __init__(
|
|
656
|
-
self,
|
|
657
|
-
bot_name: str,
|
|
658
|
-
session: aiohttp.ClientSession,
|
|
659
|
-
api_key: str,
|
|
660
|
-
replica_id: str,
|
|
661
|
-
persona_id: str = "pipecat-stream",
|
|
662
|
-
params: TavusParams = TavusParams(),
|
|
663
|
-
input_name: Optional[str] = None,
|
|
664
|
-
output_name: Optional[str] = None,
|
|
665
|
-
):
|
|
666
|
-
"""Initialize the Tavus transport.
|
|
667
|
-
|
|
668
|
-
Args:
|
|
669
|
-
bot_name: The name of the Pipecat bot.
|
|
670
|
-
session: aiohttp session used for async HTTP requests.
|
|
671
|
-
api_key: Tavus API key for authentication.
|
|
672
|
-
replica_id: ID of the replica model used for voice generation.
|
|
673
|
-
persona_id: ID of the Tavus persona. Defaults to "pipecat-stream"
|
|
674
|
-
to use the Pipecat TTS voice.
|
|
675
|
-
params: Optional Tavus-specific configuration parameters.
|
|
676
|
-
input_name: Optional name for the input transport.
|
|
677
|
-
output_name: Optional name for the output transport.
|
|
678
|
-
"""
|
|
679
|
-
super().__init__(input_name=input_name, output_name=output_name)
|
|
680
|
-
self._params = params
|
|
681
|
-
|
|
682
|
-
callbacks = TavusCallbacks(
|
|
683
|
-
on_participant_joined=self._on_participant_joined,
|
|
684
|
-
on_participant_left=self._on_participant_left,
|
|
685
|
-
)
|
|
686
|
-
self._client = TavusTransportClient(
|
|
687
|
-
bot_name="Pipecat",
|
|
688
|
-
callbacks=callbacks,
|
|
689
|
-
api_key=api_key,
|
|
690
|
-
replica_id=replica_id,
|
|
691
|
-
persona_id=persona_id,
|
|
692
|
-
session=session,
|
|
693
|
-
params=params,
|
|
694
|
-
)
|
|
695
|
-
self._input: Optional[TavusInputTransport] = None
|
|
696
|
-
self._output: Optional[TavusOutputTransport] = None
|
|
697
|
-
self._tavus_participant_id = None
|
|
698
|
-
|
|
699
|
-
# Register supported handlers. The user will only be able to register
|
|
700
|
-
# these handlers.
|
|
701
|
-
self._register_event_handler("on_client_connected")
|
|
702
|
-
self._register_event_handler("on_client_disconnected")
|
|
703
|
-
|
|
704
|
-
async def _on_participant_left(self, participant, reason):
|
|
705
|
-
"""Handle participant left events."""
|
|
706
|
-
persona_name = await self._client.get_persona_name()
|
|
707
|
-
if participant.get("info", {}).get("userName", "") != persona_name:
|
|
708
|
-
await self._on_client_disconnected(participant)
|
|
709
|
-
|
|
710
|
-
async def _on_participant_joined(self, participant):
|
|
711
|
-
"""Handle participant joined events."""
|
|
712
|
-
# get persona, look up persona_name, set this as the bot name to ignore
|
|
713
|
-
persona_name = await self._client.get_persona_name()
|
|
714
|
-
|
|
715
|
-
# Ignore the Tavus replica's microphone
|
|
716
|
-
if participant.get("info", {}).get("userName", "") == persona_name:
|
|
717
|
-
self._tavus_participant_id = participant["id"]
|
|
718
|
-
else:
|
|
719
|
-
await self._on_client_connected(participant)
|
|
720
|
-
if self._tavus_participant_id:
|
|
721
|
-
logger.debug(f"Ignoring {self._tavus_participant_id}'s microphone")
|
|
722
|
-
await self.update_subscriptions(
|
|
723
|
-
participant_settings={
|
|
724
|
-
self._tavus_participant_id: {
|
|
725
|
-
"media": {"microphone": "unsubscribed"},
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
)
|
|
729
|
-
if self._input:
|
|
730
|
-
await self._input.start_capturing_audio(participant)
|
|
731
|
-
|
|
732
|
-
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
|
|
733
|
-
"""Update subscription settings for participants.
|
|
734
|
-
|
|
735
|
-
Args:
|
|
736
|
-
participant_settings: Per-participant subscription settings.
|
|
737
|
-
profile_settings: Global subscription profile settings.
|
|
738
|
-
"""
|
|
739
|
-
await self._client.update_subscriptions(
|
|
740
|
-
participant_settings=participant_settings,
|
|
741
|
-
profile_settings=profile_settings,
|
|
742
|
-
)
|
|
743
|
-
|
|
744
|
-
def input(self) -> FrameProcessor:
|
|
745
|
-
"""Get the input transport for receiving media and events.
|
|
746
|
-
|
|
747
|
-
Returns:
|
|
748
|
-
The Tavus input transport instance.
|
|
749
|
-
"""
|
|
750
|
-
if not self._input:
|
|
751
|
-
self._input = TavusInputTransport(client=self._client, params=self._params)
|
|
752
|
-
return self._input
|
|
753
|
-
|
|
754
|
-
def output(self) -> FrameProcessor:
|
|
755
|
-
"""Get the output transport for sending media and events.
|
|
756
|
-
|
|
757
|
-
Returns:
|
|
758
|
-
The Tavus output transport instance.
|
|
759
|
-
"""
|
|
760
|
-
if not self._output:
|
|
761
|
-
self._output = TavusOutputTransport(client=self._client, params=self._params)
|
|
762
|
-
return self._output
|
|
763
|
-
|
|
764
|
-
async def _on_client_connected(self, participant: Any):
|
|
765
|
-
"""Handle client connected events."""
|
|
766
|
-
await self._call_event_handler("on_client_connected", participant)
|
|
767
|
-
|
|
768
|
-
async def _on_client_disconnected(self, participant: Any):
|
|
769
|
-
"""Handle client disconnected events."""
|
|
770
|
-
await self._call_event_handler("on_client_disconnected", participant)
|
|
14
|
+
import warnings
|
|
15
|
+
|
|
16
|
+
from pipecat.transports.tavus.transport import *
|
|
17
|
+
|
|
18
|
+
with warnings.catch_warnings():
|
|
19
|
+
warnings.simplefilter("always")
|
|
20
|
+
warnings.warn(
|
|
21
|
+
"Module `pipecat.transports.services.tavus` is deprecated, "
|
|
22
|
+
"use `pipecat.transports.tavus.transport` instead.",
|
|
23
|
+
DeprecationWarning,
|
|
24
|
+
stacklevel=2,
|
|
25
|
+
)
|