dv-pipecat-ai 0.0.82.dev815__py3-none-any.whl → 0.0.82.dev857__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dv-pipecat-ai might be problematic. Click here for more details.

Files changed (106) hide show
  1. {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/METADATA +8 -3
  2. {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/RECORD +106 -79
  3. pipecat/adapters/base_llm_adapter.py +44 -6
  4. pipecat/adapters/services/anthropic_adapter.py +302 -2
  5. pipecat/adapters/services/aws_nova_sonic_adapter.py +40 -2
  6. pipecat/adapters/services/bedrock_adapter.py +40 -2
  7. pipecat/adapters/services/gemini_adapter.py +276 -6
  8. pipecat/adapters/services/open_ai_adapter.py +88 -7
  9. pipecat/adapters/services/open_ai_realtime_adapter.py +39 -1
  10. pipecat/audio/dtmf/__init__.py +0 -0
  11. pipecat/audio/dtmf/types.py +47 -0
  12. pipecat/audio/dtmf/utils.py +70 -0
  13. pipecat/audio/filters/aic_filter.py +199 -0
  14. pipecat/audio/utils.py +9 -7
  15. pipecat/extensions/ivr/__init__.py +0 -0
  16. pipecat/extensions/ivr/ivr_navigator.py +452 -0
  17. pipecat/frames/frames.py +156 -43
  18. pipecat/pipeline/llm_switcher.py +76 -0
  19. pipecat/pipeline/parallel_pipeline.py +3 -3
  20. pipecat/pipeline/service_switcher.py +144 -0
  21. pipecat/pipeline/task.py +68 -28
  22. pipecat/pipeline/task_observer.py +10 -0
  23. pipecat/processors/aggregators/dtmf_aggregator.py +2 -2
  24. pipecat/processors/aggregators/llm_context.py +277 -0
  25. pipecat/processors/aggregators/llm_response.py +48 -15
  26. pipecat/processors/aggregators/llm_response_universal.py +840 -0
  27. pipecat/processors/aggregators/openai_llm_context.py +3 -3
  28. pipecat/processors/dtmf_aggregator.py +0 -2
  29. pipecat/processors/filters/stt_mute_filter.py +0 -2
  30. pipecat/processors/frame_processor.py +18 -11
  31. pipecat/processors/frameworks/rtvi.py +17 -10
  32. pipecat/processors/metrics/sentry.py +2 -0
  33. pipecat/runner/daily.py +137 -36
  34. pipecat/runner/run.py +1 -1
  35. pipecat/runner/utils.py +7 -7
  36. pipecat/serializers/asterisk.py +20 -4
  37. pipecat/serializers/exotel.py +1 -1
  38. pipecat/serializers/plivo.py +1 -1
  39. pipecat/serializers/telnyx.py +1 -1
  40. pipecat/serializers/twilio.py +1 -1
  41. pipecat/services/__init__.py +2 -2
  42. pipecat/services/anthropic/llm.py +113 -28
  43. pipecat/services/asyncai/tts.py +4 -0
  44. pipecat/services/aws/llm.py +82 -8
  45. pipecat/services/aws/tts.py +0 -10
  46. pipecat/services/aws_nova_sonic/aws.py +5 -0
  47. pipecat/services/cartesia/tts.py +28 -16
  48. pipecat/services/cerebras/llm.py +15 -10
  49. pipecat/services/deepgram/stt.py +8 -0
  50. pipecat/services/deepseek/llm.py +13 -8
  51. pipecat/services/fireworks/llm.py +13 -8
  52. pipecat/services/fish/tts.py +8 -6
  53. pipecat/services/gemini_multimodal_live/gemini.py +5 -0
  54. pipecat/services/gladia/config.py +7 -1
  55. pipecat/services/gladia/stt.py +23 -15
  56. pipecat/services/google/llm.py +159 -59
  57. pipecat/services/google/llm_openai.py +18 -3
  58. pipecat/services/grok/llm.py +2 -1
  59. pipecat/services/llm_service.py +38 -3
  60. pipecat/services/mem0/memory.py +2 -1
  61. pipecat/services/mistral/llm.py +5 -6
  62. pipecat/services/nim/llm.py +2 -1
  63. pipecat/services/openai/base_llm.py +88 -26
  64. pipecat/services/openai/image.py +6 -1
  65. pipecat/services/openai_realtime_beta/openai.py +5 -2
  66. pipecat/services/openpipe/llm.py +6 -8
  67. pipecat/services/perplexity/llm.py +13 -8
  68. pipecat/services/playht/tts.py +9 -6
  69. pipecat/services/rime/tts.py +1 -1
  70. pipecat/services/sambanova/llm.py +18 -13
  71. pipecat/services/sarvam/tts.py +415 -10
  72. pipecat/services/speechmatics/stt.py +2 -2
  73. pipecat/services/tavus/video.py +1 -1
  74. pipecat/services/tts_service.py +15 -5
  75. pipecat/services/vistaar/llm.py +2 -5
  76. pipecat/transports/base_input.py +32 -19
  77. pipecat/transports/base_output.py +39 -5
  78. pipecat/transports/daily/__init__.py +0 -0
  79. pipecat/transports/daily/transport.py +2371 -0
  80. pipecat/transports/daily/utils.py +410 -0
  81. pipecat/transports/livekit/__init__.py +0 -0
  82. pipecat/transports/livekit/transport.py +1042 -0
  83. pipecat/transports/network/fastapi_websocket.py +12 -546
  84. pipecat/transports/network/small_webrtc.py +12 -922
  85. pipecat/transports/network/webrtc_connection.py +9 -595
  86. pipecat/transports/network/websocket_client.py +12 -481
  87. pipecat/transports/network/websocket_server.py +12 -487
  88. pipecat/transports/services/daily.py +9 -2334
  89. pipecat/transports/services/helpers/daily_rest.py +12 -396
  90. pipecat/transports/services/livekit.py +12 -975
  91. pipecat/transports/services/tavus.py +12 -757
  92. pipecat/transports/smallwebrtc/__init__.py +0 -0
  93. pipecat/transports/smallwebrtc/connection.py +612 -0
  94. pipecat/transports/smallwebrtc/transport.py +936 -0
  95. pipecat/transports/tavus/__init__.py +0 -0
  96. pipecat/transports/tavus/transport.py +770 -0
  97. pipecat/transports/websocket/__init__.py +0 -0
  98. pipecat/transports/websocket/client.py +494 -0
  99. pipecat/transports/websocket/fastapi.py +559 -0
  100. pipecat/transports/websocket/server.py +500 -0
  101. pipecat/transports/whatsapp/__init__.py +0 -0
  102. pipecat/transports/whatsapp/api.py +345 -0
  103. pipecat/transports/whatsapp/client.py +364 -0
  104. {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/WHEEL +0 -0
  105. {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/licenses/LICENSE +0 -0
  106. {dv_pipecat_ai-0.0.82.dev815.dist-info → dv_pipecat_ai-0.0.82.dev857.dist-info}/top_level.txt +0 -0
@@ -11,978 +11,15 @@ including audio streaming, data messaging, participant management, and room
11
11
  event handling for conversational AI applications.
12
12
  """
13
13
 
14
- import asyncio
15
- from dataclasses import dataclass
16
- from typing import Any, Awaitable, Callable, List, Optional
17
-
18
- from loguru import logger
19
- from pydantic import BaseModel
20
-
21
- from pipecat.audio.utils import create_stream_resampler
22
- from pipecat.audio.vad.vad_analyzer import VADAnalyzer
23
- from pipecat.frames.frames import (
24
- AudioRawFrame,
25
- CancelFrame,
26
- EndFrame,
27
- OutputAudioRawFrame,
28
- StartFrame,
29
- TransportMessageFrame,
30
- TransportMessageUrgentFrame,
31
- UserAudioRawFrame,
32
- )
33
- from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup
34
- from pipecat.transports.base_input import BaseInputTransport
35
- from pipecat.transports.base_output import BaseOutputTransport
36
- from pipecat.transports.base_transport import BaseTransport, TransportParams
37
- from pipecat.utils.asyncio.task_manager import BaseTaskManager
38
-
39
- try:
40
- from livekit import rtc
41
- from tenacity import retry, stop_after_attempt, wait_exponential
42
- except ModuleNotFoundError as e:
43
- logger.error(f"Exception: {e}")
44
- logger.error("In order to use LiveKit, you need to `pip install pipecat-ai[livekit]`.")
45
- raise Exception(f"Missing module: {e}")
46
-
47
-
48
- @dataclass
49
- class LiveKitTransportMessageFrame(TransportMessageFrame):
50
- """Frame for transport messages in LiveKit rooms.
51
-
52
- Parameters:
53
- participant_id: Optional ID of the participant this message is for/from.
54
- """
55
-
56
- participant_id: Optional[str] = None
57
-
58
-
59
- @dataclass
60
- class LiveKitTransportMessageUrgentFrame(TransportMessageUrgentFrame):
61
- """Frame for urgent transport messages in LiveKit rooms.
62
-
63
- Parameters:
64
- participant_id: Optional ID of the participant this message is for/from.
65
- """
66
-
67
- participant_id: Optional[str] = None
68
-
69
-
70
- class LiveKitParams(TransportParams):
71
- """Configuration parameters for LiveKit transport.
72
-
73
- Inherits all parameters from TransportParams without additional configuration.
74
- """
75
-
76
- pass
77
-
78
-
79
- class LiveKitCallbacks(BaseModel):
80
- """Callback handlers for LiveKit events.
81
-
82
- Parameters:
83
- on_connected: Called when connected to the LiveKit room.
84
- on_disconnected: Called when disconnected from the LiveKit room.
85
- on_participant_connected: Called when a participant joins the room.
86
- on_participant_disconnected: Called when a participant leaves the room.
87
- on_audio_track_subscribed: Called when an audio track is subscribed.
88
- on_audio_track_unsubscribed: Called when an audio track is unsubscribed.
89
- on_data_received: Called when data is received from a participant.
90
- on_first_participant_joined: Called when the first participant joins.
91
- """
92
-
93
- on_connected: Callable[[], Awaitable[None]]
94
- on_disconnected: Callable[[], Awaitable[None]]
95
- on_participant_connected: Callable[[str], Awaitable[None]]
96
- on_participant_disconnected: Callable[[str], Awaitable[None]]
97
- on_audio_track_subscribed: Callable[[str], Awaitable[None]]
98
- on_audio_track_unsubscribed: Callable[[str], Awaitable[None]]
99
- on_data_received: Callable[[bytes, str], Awaitable[None]]
100
- on_first_participant_joined: Callable[[str], Awaitable[None]]
101
-
102
-
103
- class LiveKitTransportClient:
104
- """Core client for interacting with LiveKit rooms.
105
-
106
- Manages the connection to LiveKit rooms and handles all low-level API interactions
107
- including room management, audio streaming, data messaging, and event handling.
108
- """
109
-
110
- def __init__(
111
- self,
112
- url: str,
113
- token: str,
114
- room_name: str,
115
- params: LiveKitParams,
116
- callbacks: LiveKitCallbacks,
117
- transport_name: str,
118
- ):
119
- """Initialize the LiveKit transport client.
120
-
121
- Args:
122
- url: LiveKit server URL to connect to.
123
- token: Authentication token for the room.
124
- room_name: Name of the LiveKit room to join.
125
- params: Configuration parameters for the transport.
126
- callbacks: Event callback handlers.
127
- transport_name: Name identifier for the transport.
128
- """
129
- self._url = url
130
- self._token = token
131
- self._room_name = room_name
132
- self._params = params
133
- self._callbacks = callbacks
134
- self._transport_name = transport_name
135
- self._room: Optional[rtc.Room] = None
136
- self._participant_id: str = ""
137
- self._connected = False
138
- self._disconnect_counter = 0
139
- self._audio_source: Optional[rtc.AudioSource] = None
140
- self._audio_track: Optional[rtc.LocalAudioTrack] = None
141
- self._audio_tracks = {}
142
- self._audio_queue = asyncio.Queue()
143
- self._other_participant_has_joined = False
144
- self._task_manager: Optional[BaseTaskManager] = None
145
-
146
- @property
147
- def participant_id(self) -> str:
148
- """Get the participant ID for this client.
149
-
150
- Returns:
151
- The participant ID assigned by LiveKit.
152
- """
153
- return self._participant_id
154
-
155
- @property
156
- def room(self) -> rtc.Room:
157
- """Get the LiveKit room instance.
158
-
159
- Returns:
160
- The LiveKit room object.
161
-
162
- Raises:
163
- Exception: If room object is not available.
164
- """
165
- if not self._room:
166
- raise Exception(f"{self}: missing room object (pipeline not started?)")
167
- return self._room
168
-
169
- async def setup(self, setup: FrameProcessorSetup):
170
- """Setup the client with task manager and room initialization.
171
-
172
- Args:
173
- setup: The frame processor setup configuration.
174
- """
175
- if self._task_manager:
176
- return
177
-
178
- self._task_manager = setup.task_manager
179
- self._room = rtc.Room(loop=self._task_manager.get_event_loop())
180
-
181
- # Set up room event handlers
182
- self.room.on("participant_connected")(self._on_participant_connected_wrapper)
183
- self.room.on("participant_disconnected")(self._on_participant_disconnected_wrapper)
184
- self.room.on("track_subscribed")(self._on_track_subscribed_wrapper)
185
- self.room.on("track_unsubscribed")(self._on_track_unsubscribed_wrapper)
186
- self.room.on("data_received")(self._on_data_received_wrapper)
187
- self.room.on("connected")(self._on_connected_wrapper)
188
- self.room.on("disconnected")(self._on_disconnected_wrapper)
189
-
190
- async def cleanup(self):
191
- """Cleanup client resources."""
192
- await self.disconnect()
193
-
194
- async def start(self, frame: StartFrame):
195
- """Start the client and initialize audio components.
196
-
197
- Args:
198
- frame: The start frame containing initialization parameters.
199
- """
200
- self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
201
-
202
- @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
203
- async def connect(self):
204
- """Connect to the LiveKit room with retry logic."""
205
- if self._connected:
206
- # Increment disconnect counter if already connected.
207
- self._disconnect_counter += 1
208
- return
209
-
210
- logger.info(f"Connecting to {self._room_name}")
211
-
212
- try:
213
- await self.room.connect(
214
- self._url,
215
- self._token,
216
- options=rtc.RoomOptions(auto_subscribe=True),
217
- )
218
- self._connected = True
219
- # Increment disconnect counter if we successfully connected.
220
- self._disconnect_counter += 1
221
-
222
- self._participant_id = self.room.local_participant.sid
223
- logger.info(f"Connected to {self._room_name}")
224
-
225
- # Set up audio source and track
226
- self._audio_source = rtc.AudioSource(
227
- self._out_sample_rate, self._params.audio_out_channels
228
- )
229
- self._audio_track = rtc.LocalAudioTrack.create_audio_track(
230
- "pipecat-audio", self._audio_source
231
- )
232
- options = rtc.TrackPublishOptions()
233
- options.source = rtc.TrackSource.SOURCE_MICROPHONE
234
- await self.room.local_participant.publish_track(self._audio_track, options)
235
-
236
- await self._callbacks.on_connected()
237
-
238
- # Check if there are already participants in the room
239
- participants = self.get_participants()
240
- if participants and not self._other_participant_has_joined:
241
- self._other_participant_has_joined = True
242
- await self._callbacks.on_first_participant_joined(participants[0])
243
- except Exception as e:
244
- logger.error(f"Error connecting to {self._room_name}: {e}")
245
- raise
246
-
247
- async def disconnect(self):
248
- """Disconnect from the LiveKit room."""
249
- # Decrement leave counter when leaving.
250
- self._disconnect_counter -= 1
251
-
252
- if not self._connected or self._disconnect_counter > 0:
253
- return
254
-
255
- logger.info(f"Disconnecting from {self._room_name}")
256
- await self.room.disconnect()
257
- self._connected = False
258
- logger.info(f"Disconnected from {self._room_name}")
259
- await self._callbacks.on_disconnected()
260
-
261
- async def send_data(self, data: bytes, participant_id: Optional[str] = None):
262
- """Send data to participants in the room.
263
-
264
- Args:
265
- data: The data bytes to send.
266
- participant_id: Optional specific participant to send to.
267
- """
268
- if not self._connected:
269
- return
270
-
271
- try:
272
- if participant_id:
273
- await self.room.local_participant.publish_data(
274
- data, reliable=True, destination_identities=[participant_id]
275
- )
276
- else:
277
- await self.room.local_participant.publish_data(data, reliable=True)
278
- except Exception as e:
279
- logger.error(f"Error sending data: {e}")
280
-
281
- async def publish_audio(self, audio_frame: rtc.AudioFrame):
282
- """Publish an audio frame to the room.
283
-
284
- Args:
285
- audio_frame: The LiveKit audio frame to publish.
286
- """
287
- if not self._connected or not self._audio_source:
288
- return
289
-
290
- try:
291
- await self._audio_source.capture_frame(audio_frame)
292
- except Exception as e:
293
- logger.error(f"Error publishing audio: {e}")
294
-
295
- def get_participants(self) -> List[str]:
296
- """Get list of participant IDs in the room.
297
-
298
- Returns:
299
- List of participant IDs.
300
- """
301
- return [p.sid for p in self.room.remote_participants.values()]
302
-
303
- async def get_participant_metadata(self, participant_id: str) -> dict:
304
- """Get metadata for a specific participant.
305
-
306
- Args:
307
- participant_id: ID of the participant to get metadata for.
308
-
309
- Returns:
310
- Dictionary containing participant metadata.
311
- """
312
- participant = self.room.remote_participants.get(participant_id)
313
- if participant:
314
- return {
315
- "id": participant.sid,
316
- "name": participant.name,
317
- "metadata": participant.metadata,
318
- "is_speaking": participant.is_speaking,
319
- }
320
- return {}
321
-
322
- async def set_participant_metadata(self, metadata: str):
323
- """Set metadata for the local participant.
324
-
325
- Args:
326
- metadata: Metadata string to set.
327
- """
328
- await self.room.local_participant.set_metadata(metadata)
329
-
330
- async def mute_participant(self, participant_id: str):
331
- """Mute a specific participant's audio tracks.
332
-
333
- Args:
334
- participant_id: ID of the participant to mute.
335
- """
336
- participant = self.room.remote_participants.get(participant_id)
337
- if participant:
338
- for track in participant.tracks.values():
339
- if track.kind == "audio":
340
- await track.set_enabled(False)
341
-
342
- async def unmute_participant(self, participant_id: str):
343
- """Unmute a specific participant's audio tracks.
344
-
345
- Args:
346
- participant_id: ID of the participant to unmute.
347
- """
348
- participant = self.room.remote_participants.get(participant_id)
349
- if participant:
350
- for track in participant.tracks.values():
351
- if track.kind == "audio":
352
- await track.set_enabled(True)
353
-
354
- # Wrapper methods for event handlers
355
- def _on_participant_connected_wrapper(self, participant: rtc.RemoteParticipant):
356
- """Wrapper for participant connected events."""
357
- self._task_manager.create_task(
358
- self._async_on_participant_connected(participant),
359
- f"{self}::_async_on_participant_connected",
360
- )
361
-
362
- def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant):
363
- """Wrapper for participant disconnected events."""
364
- self._task_manager.create_task(
365
- self._async_on_participant_disconnected(participant),
366
- f"{self}::_async_on_participant_disconnected",
367
- )
368
-
369
- def _on_track_subscribed_wrapper(
370
- self,
371
- track: rtc.Track,
372
- publication: rtc.RemoteTrackPublication,
373
- participant: rtc.RemoteParticipant,
374
- ):
375
- """Wrapper for track subscribed events."""
376
- self._task_manager.create_task(
377
- self._async_on_track_subscribed(track, publication, participant),
378
- f"{self}::_async_on_track_subscribed",
379
- )
380
-
381
- def _on_track_unsubscribed_wrapper(
382
- self,
383
- track: rtc.Track,
384
- publication: rtc.RemoteTrackPublication,
385
- participant: rtc.RemoteParticipant,
386
- ):
387
- """Wrapper for track unsubscribed events."""
388
- self._task_manager.create_task(
389
- self._async_on_track_unsubscribed(track, publication, participant),
390
- f"{self}::_async_on_track_unsubscribed",
391
- )
392
-
393
- def _on_data_received_wrapper(self, data: rtc.DataPacket):
394
- """Wrapper for data received events."""
395
- self._task_manager.create_task(
396
- self._async_on_data_received(data),
397
- f"{self}::_async_on_data_received",
398
- )
399
-
400
- def _on_connected_wrapper(self):
401
- """Wrapper for connected events."""
402
- self._task_manager.create_task(self._async_on_connected(), f"{self}::_async_on_connected")
403
-
404
- def _on_disconnected_wrapper(self):
405
- """Wrapper for disconnected events."""
406
- self._task_manager.create_task(
407
- self._async_on_disconnected(), f"{self}::_async_on_disconnected"
408
- )
409
-
410
- # Async methods for event handling
411
- async def _async_on_participant_connected(self, participant: rtc.RemoteParticipant):
412
- """Handle participant connected events."""
413
- logger.info(f"Participant connected: {participant.identity}")
414
- await self._callbacks.on_participant_connected(participant.sid)
415
- if not self._other_participant_has_joined:
416
- self._other_participant_has_joined = True
417
- await self._callbacks.on_first_participant_joined(participant.sid)
418
-
419
- async def _async_on_participant_disconnected(self, participant: rtc.RemoteParticipant):
420
- """Handle participant disconnected events."""
421
- logger.info(f"Participant disconnected: {participant.identity}")
422
- await self._callbacks.on_participant_disconnected(participant.sid)
423
- if len(self.get_participants()) == 0:
424
- self._other_participant_has_joined = False
425
-
426
- async def _async_on_track_subscribed(
427
- self,
428
- track: rtc.Track,
429
- publication: rtc.RemoteTrackPublication,
430
- participant: rtc.RemoteParticipant,
431
- ):
432
- """Handle track subscribed events."""
433
- if track.kind == rtc.TrackKind.KIND_AUDIO:
434
- logger.info(f"Audio track subscribed: {track.sid} from participant {participant.sid}")
435
- self._audio_tracks[participant.sid] = track
436
- audio_stream = rtc.AudioStream(track)
437
- self._task_manager.create_task(
438
- self._process_audio_stream(audio_stream, participant.sid),
439
- f"{self}::_process_audio_stream",
440
- )
441
- await self._callbacks.on_audio_track_subscribed(participant.sid)
442
-
443
- async def _async_on_track_unsubscribed(
444
- self,
445
- track: rtc.Track,
446
- publication: rtc.RemoteTrackPublication,
447
- participant: rtc.RemoteParticipant,
448
- ):
449
- """Handle track unsubscribed events."""
450
- logger.info(f"Track unsubscribed: {publication.sid} from {participant.identity}")
451
- if track.kind == rtc.TrackKind.KIND_AUDIO:
452
- await self._callbacks.on_audio_track_unsubscribed(participant.sid)
453
-
454
- async def _async_on_data_received(self, data: rtc.DataPacket):
455
- """Handle data received events."""
456
- await self._callbacks.on_data_received(data.data, data.participant.sid)
457
-
458
- async def _async_on_connected(self):
459
- """Handle connected events."""
460
- await self._callbacks.on_connected()
461
-
462
- async def _async_on_disconnected(self, reason=None):
463
- """Handle disconnected events."""
464
- self._connected = False
465
- logger.info(f"Disconnected from {self._room_name}. Reason: {reason}")
466
- await self._callbacks.on_disconnected()
467
-
468
- async def _process_audio_stream(self, audio_stream: rtc.AudioStream, participant_id: str):
469
- """Process incoming audio stream from a participant."""
470
- logger.info(f"Started processing audio stream for participant {participant_id}")
471
- async for event in audio_stream:
472
- if isinstance(event, rtc.AudioFrameEvent):
473
- await self._audio_queue.put((event, participant_id))
474
- else:
475
- logger.warning(f"Received unexpected event type: {type(event)}")
476
-
477
- async def get_next_audio_frame(self):
478
- """Get the next audio frame from the queue."""
479
- while True:
480
- frame, participant_id = await self._audio_queue.get()
481
- yield frame, participant_id
482
-
483
- def __str__(self):
484
- """String representation of the LiveKit transport client."""
485
- return f"{self._transport_name}::LiveKitTransportClient"
486
-
487
-
488
- class LiveKitInputTransport(BaseInputTransport):
489
- """Handles incoming media streams and events from LiveKit rooms.
490
-
491
- Processes incoming audio streams from room participants and forwards them
492
- as Pipecat frames, including audio resampling and VAD integration.
493
- """
494
-
495
- def __init__(
496
- self,
497
- transport: BaseTransport,
498
- client: LiveKitTransportClient,
499
- params: LiveKitParams,
500
- **kwargs,
501
- ):
502
- """Initialize the LiveKit input transport.
503
-
504
- Args:
505
- transport: The parent transport instance.
506
- client: LiveKitTransportClient instance.
507
- params: Configuration parameters.
508
- **kwargs: Additional arguments passed to parent class.
509
- """
510
- super().__init__(params, **kwargs)
511
- self._transport = transport
512
- self._client = client
513
-
514
- self._audio_in_task = None
515
- self._vad_analyzer: Optional[VADAnalyzer] = params.vad_analyzer
516
- self._resampler = create_stream_resampler()
517
-
518
- # Whether we have seen a StartFrame already.
519
- self._initialized = False
520
-
521
- @property
522
- def vad_analyzer(self) -> Optional[VADAnalyzer]:
523
- """Get the Voice Activity Detection analyzer.
524
-
525
- Returns:
526
- The VAD analyzer instance if configured.
527
- """
528
- return self._vad_analyzer
529
-
530
- async def start(self, frame: StartFrame):
531
- """Start the input transport and connect to LiveKit room.
532
-
533
- Args:
534
- frame: The start frame containing initialization parameters.
535
- """
536
- await super().start(frame)
537
-
538
- if self._initialized:
539
- return
540
-
541
- self._initialized = True
542
-
543
- await self._client.start(frame)
544
- await self._client.connect()
545
- if not self._audio_in_task and self._params.audio_in_enabled:
546
- self._audio_in_task = self.create_task(self._audio_in_task_handler())
547
- await self.set_transport_ready(frame)
548
- logger.info("LiveKitInputTransport started")
549
-
550
- async def stop(self, frame: EndFrame):
551
- """Stop the input transport and disconnect from LiveKit room.
552
-
553
- Args:
554
- frame: The end frame signaling transport shutdown.
555
- """
556
- await super().stop(frame)
557
- await self._client.disconnect()
558
- if self._audio_in_task:
559
- await self.cancel_task(self._audio_in_task)
560
- logger.info("LiveKitInputTransport stopped")
561
-
562
- async def cancel(self, frame: CancelFrame):
563
- """Cancel the input transport and disconnect from LiveKit room.
564
-
565
- Args:
566
- frame: The cancel frame signaling immediate cancellation.
567
- """
568
- await super().cancel(frame)
569
- await self._client.disconnect()
570
- if self._audio_in_task and self._params.audio_in_enabled:
571
- await self.cancel_task(self._audio_in_task)
572
-
573
- async def setup(self, setup: FrameProcessorSetup):
574
- """Setup the input transport with shared client setup.
575
-
576
- Args:
577
- setup: The frame processor setup configuration.
578
- """
579
- await super().setup(setup)
580
- await self._client.setup(setup)
581
-
582
- async def cleanup(self):
583
- """Cleanup input transport and shared resources."""
584
- await super().cleanup()
585
- await self._transport.cleanup()
586
-
587
- async def push_app_message(self, message: Any, sender: str):
588
- """Push an application message as an urgent transport frame.
589
-
590
- Args:
591
- message: The message data to send.
592
- sender: ID of the message sender.
593
- """
594
- frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender)
595
- await self.push_frame(frame)
596
-
597
- async def _audio_in_task_handler(self):
598
- """Handle incoming audio frames from participants."""
599
- logger.info("Audio input task started")
600
- audio_iterator = self._client.get_next_audio_frame()
601
- async for audio_data in audio_iterator:
602
- if audio_data:
603
- audio_frame_event, participant_id = audio_data
604
- pipecat_audio_frame = await self._convert_livekit_audio_to_pipecat(
605
- audio_frame_event
606
- )
607
-
608
- # Skip frames with no audio data
609
- if len(pipecat_audio_frame.audio) == 0:
610
- continue
611
-
612
- input_audio_frame = UserAudioRawFrame(
613
- user_id=participant_id,
614
- audio=pipecat_audio_frame.audio,
615
- sample_rate=pipecat_audio_frame.sample_rate,
616
- num_channels=pipecat_audio_frame.num_channels,
617
- )
618
- await self.push_audio_frame(input_audio_frame)
619
-
620
- async def _convert_livekit_audio_to_pipecat(
621
- self, audio_frame_event: rtc.AudioFrameEvent
622
- ) -> AudioRawFrame:
623
- """Convert LiveKit audio frame to Pipecat audio frame."""
624
- audio_frame = audio_frame_event.frame
625
-
626
- audio_data = await self._resampler.resample(
627
- audio_frame.data.tobytes(), audio_frame.sample_rate, self.sample_rate
628
- )
629
-
630
- return AudioRawFrame(
631
- audio=audio_data,
632
- sample_rate=self.sample_rate,
633
- num_channels=audio_frame.num_channels,
634
- )
635
-
636
-
637
- class LiveKitOutputTransport(BaseOutputTransport):
638
- """Handles outgoing media streams and events to LiveKit rooms.
639
-
640
- Manages sending audio frames and data messages to LiveKit room participants,
641
- including audio format conversion for LiveKit compatibility.
642
- """
643
-
644
- def __init__(
645
- self,
646
- transport: BaseTransport,
647
- client: LiveKitTransportClient,
648
- params: LiveKitParams,
649
- **kwargs,
650
- ):
651
- """Initialize the LiveKit output transport.
652
-
653
- Args:
654
- transport: The parent transport instance.
655
- client: LiveKitTransportClient instance.
656
- params: Configuration parameters.
657
- **kwargs: Additional arguments passed to parent class.
658
- """
659
- super().__init__(params, **kwargs)
660
- self._transport = transport
661
- self._client = client
662
-
663
- # Whether we have seen a StartFrame already.
664
- self._initialized = False
665
-
666
- async def start(self, frame: StartFrame):
667
- """Start the output transport and connect to LiveKit room.
668
-
669
- Args:
670
- frame: The start frame containing initialization parameters.
671
- """
672
- await super().start(frame)
673
-
674
- if self._initialized:
675
- return
676
-
677
- self._initialized = True
678
-
679
- await self._client.start(frame)
680
- await self._client.connect()
681
- await self.set_transport_ready(frame)
682
- logger.info("LiveKitOutputTransport started")
683
-
684
- async def stop(self, frame: EndFrame):
685
- """Stop the output transport and disconnect from LiveKit room.
686
-
687
- Args:
688
- frame: The end frame signaling transport shutdown.
689
- """
690
- await super().stop(frame)
691
- await self._client.disconnect()
692
- logger.info("LiveKitOutputTransport stopped")
693
-
694
- async def cancel(self, frame: CancelFrame):
695
- """Cancel the output transport and disconnect from LiveKit room.
696
-
697
- Args:
698
- frame: The cancel frame signaling immediate cancellation.
699
- """
700
- await super().cancel(frame)
701
- await self._client.disconnect()
702
-
703
- async def setup(self, setup: FrameProcessorSetup):
704
- """Setup the output transport with shared client setup.
705
-
706
- Args:
707
- setup: The frame processor setup configuration.
708
- """
709
- await super().setup(setup)
710
- await self._client.setup(setup)
711
-
712
- async def cleanup(self):
713
- """Cleanup output transport and shared resources."""
714
- await super().cleanup()
715
- await self._transport.cleanup()
716
-
717
- async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
718
- """Send a transport message to participants.
719
-
720
- Args:
721
- frame: The transport message frame to send.
722
- """
723
- if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)):
724
- await self._client.send_data(frame.message.encode(), frame.participant_id)
725
- else:
726
- await self._client.send_data(frame.message.encode())
727
-
728
- async def write_audio_frame(self, frame: OutputAudioRawFrame):
729
- """Write an audio frame to the LiveKit room.
730
-
731
- Args:
732
- frame: The audio frame to write.
733
- """
734
- livekit_audio = self._convert_pipecat_audio_to_livekit(frame.audio)
735
- await self._client.publish_audio(livekit_audio)
736
-
737
- def _convert_pipecat_audio_to_livekit(self, pipecat_audio: bytes) -> rtc.AudioFrame:
738
- """Convert Pipecat audio data to LiveKit audio frame."""
739
- bytes_per_sample = 2 # Assuming 16-bit audio
740
- total_samples = len(pipecat_audio) // bytes_per_sample
741
- samples_per_channel = total_samples // self._params.audio_out_channels
742
-
743
- return rtc.AudioFrame(
744
- data=pipecat_audio,
745
- sample_rate=self.sample_rate,
746
- num_channels=self._params.audio_out_channels,
747
- samples_per_channel=samples_per_channel,
748
- )
749
-
750
-
751
- class LiveKitTransport(BaseTransport):
752
- """Transport implementation for LiveKit real-time communication.
753
-
754
- Provides comprehensive LiveKit integration including audio streaming, data
755
- messaging, participant management, and room event handling for conversational
756
- AI applications.
757
- """
758
-
759
- def __init__(
760
- self,
761
- url: str,
762
- token: str,
763
- room_name: str,
764
- params: Optional[LiveKitParams] = None,
765
- input_name: Optional[str] = None,
766
- output_name: Optional[str] = None,
767
- ):
768
- """Initialize the LiveKit transport.
769
-
770
- Args:
771
- url: LiveKit server URL to connect to.
772
- token: Authentication token for the room.
773
- room_name: Name of the LiveKit room to join.
774
- params: Configuration parameters for the transport.
775
- input_name: Optional name for the input transport.
776
- output_name: Optional name for the output transport.
777
- """
778
- super().__init__(input_name=input_name, output_name=output_name)
779
-
780
- callbacks = LiveKitCallbacks(
781
- on_connected=self._on_connected,
782
- on_disconnected=self._on_disconnected,
783
- on_participant_connected=self._on_participant_connected,
784
- on_participant_disconnected=self._on_participant_disconnected,
785
- on_audio_track_subscribed=self._on_audio_track_subscribed,
786
- on_audio_track_unsubscribed=self._on_audio_track_unsubscribed,
787
- on_data_received=self._on_data_received,
788
- on_first_participant_joined=self._on_first_participant_joined,
789
- )
790
- self._params = params or LiveKitParams()
791
-
792
- self._client = LiveKitTransportClient(
793
- url, token, room_name, self._params, callbacks, self.name
794
- )
795
- self._input: Optional[LiveKitInputTransport] = None
796
- self._output: Optional[LiveKitOutputTransport] = None
797
-
798
- self._register_event_handler("on_connected")
799
- self._register_event_handler("on_disconnected")
800
- self._register_event_handler("on_participant_connected")
801
- self._register_event_handler("on_participant_disconnected")
802
- self._register_event_handler("on_audio_track_subscribed")
803
- self._register_event_handler("on_audio_track_unsubscribed")
804
- self._register_event_handler("on_data_received")
805
- self._register_event_handler("on_first_participant_joined")
806
- self._register_event_handler("on_participant_left")
807
- self._register_event_handler("on_call_state_updated")
808
-
809
- def input(self) -> LiveKitInputTransport:
810
- """Get the input transport for receiving media and events.
811
-
812
- Returns:
813
- The LiveKit input transport instance.
814
- """
815
- if not self._input:
816
- self._input = LiveKitInputTransport(
817
- self, self._client, self._params, name=self._input_name
818
- )
819
- return self._input
820
-
821
- def output(self) -> LiveKitOutputTransport:
822
- """Get the output transport for sending media and events.
823
-
824
- Returns:
825
- The LiveKit output transport instance.
826
- """
827
- if not self._output:
828
- self._output = LiveKitOutputTransport(
829
- self, self._client, self._params, name=self._output_name
830
- )
831
- return self._output
832
-
833
- @property
834
- def participant_id(self) -> str:
835
- """Get the participant ID for this transport.
836
-
837
- Returns:
838
- The participant ID assigned by LiveKit.
839
- """
840
- return self._client.participant_id
841
-
842
- async def send_audio(self, frame: OutputAudioRawFrame):
843
- """Send an audio frame to the LiveKit room.
844
-
845
- Args:
846
- frame: The audio frame to send.
847
- """
848
- if self._output:
849
- await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
850
-
851
- def get_participants(self) -> List[str]:
852
- """Get list of participant IDs in the room.
853
-
854
- Returns:
855
- List of participant IDs.
856
- """
857
- return self._client.get_participants()
858
-
859
- async def get_participant_metadata(self, participant_id: str) -> dict:
860
- """Get metadata for a specific participant.
861
-
862
- Args:
863
- participant_id: ID of the participant to get metadata for.
864
-
865
- Returns:
866
- Dictionary containing participant metadata.
867
- """
868
- return await self._client.get_participant_metadata(participant_id)
869
-
870
- async def set_metadata(self, metadata: str):
871
- """Set metadata for the local participant.
872
-
873
- Args:
874
- metadata: Metadata string to set.
875
- """
876
- await self._client.set_participant_metadata(metadata)
877
-
878
- async def mute_participant(self, participant_id: str):
879
- """Mute a specific participant's audio tracks.
880
-
881
- Args:
882
- participant_id: ID of the participant to mute.
883
- """
884
- await self._client.mute_participant(participant_id)
885
-
886
- async def unmute_participant(self, participant_id: str):
887
- """Unmute a specific participant's audio tracks.
888
-
889
- Args:
890
- participant_id: ID of the participant to unmute.
891
- """
892
- await self._client.unmute_participant(participant_id)
893
-
894
- async def _on_connected(self):
895
- """Handle room connected events."""
896
- await self._call_event_handler("on_connected")
897
-
898
- async def _on_disconnected(self):
899
- """Handle room disconnected events."""
900
- await self._call_event_handler("on_disconnected")
901
-
902
- async def _on_participant_connected(self, participant_id: str):
903
- """Handle participant connected events."""
904
- await self._call_event_handler("on_participant_connected", participant_id)
905
-
906
- async def _on_participant_disconnected(self, participant_id: str):
907
- """Handle participant disconnected events."""
908
- await self._call_event_handler("on_participant_disconnected", participant_id)
909
- await self._call_event_handler("on_participant_left", participant_id, "disconnected")
910
-
911
- async def _on_audio_track_subscribed(self, participant_id: str):
912
- """Handle audio track subscribed events."""
913
- await self._call_event_handler("on_audio_track_subscribed", participant_id)
914
- participant = self._client.room.remote_participants.get(participant_id)
915
- if participant:
916
- for publication in participant.audio_tracks.values():
917
- self._client._on_track_subscribed_wrapper(
918
- publication.track, publication, participant
919
- )
920
-
921
- async def _on_audio_track_unsubscribed(self, participant_id: str):
922
- """Handle audio track unsubscribed events."""
923
- await self._call_event_handler("on_audio_track_unsubscribed", participant_id)
924
-
925
- async def _on_data_received(self, data: bytes, participant_id: str):
926
- """Handle data received events."""
927
- if self._input:
928
- await self._input.push_app_message(data.decode(), participant_id)
929
- await self._call_event_handler("on_data_received", data, participant_id)
930
-
931
- async def send_message(self, message: str, participant_id: Optional[str] = None):
932
- """Send a message to participants in the room.
933
-
934
- Args:
935
- message: The message string to send.
936
- participant_id: Optional specific participant to send to.
937
- """
938
- if self._output:
939
- frame = LiveKitTransportMessageFrame(message=message, participant_id=participant_id)
940
- await self._output.send_message(frame)
941
-
942
- async def send_message_urgent(self, message: str, participant_id: Optional[str] = None):
943
- """Send an urgent message to participants in the room.
944
-
945
- Args:
946
- message: The urgent message string to send.
947
- participant_id: Optional specific participant to send to.
948
- """
949
- if self._output:
950
- frame = LiveKitTransportMessageUrgentFrame(
951
- message=message, participant_id=participant_id
952
- )
953
- await self._output.send_message(frame)
954
-
955
- async def on_room_event(self, event):
956
- """Handle room events.
957
-
958
- Args:
959
- event: The room event to handle.
960
- """
961
- # Handle room events
962
- pass
963
-
964
- async def on_participant_event(self, event):
965
- """Handle participant events.
966
-
967
- Args:
968
- event: The participant event to handle.
969
- """
970
- # Handle participant events
971
- pass
972
-
973
- async def on_track_event(self, event):
974
- """Handle track events.
975
-
976
- Args:
977
- event: The track event to handle.
978
- """
979
- # Handle track events
980
- pass
981
-
982
- async def _on_call_state_updated(self, state: str):
983
- """Handle call state update events."""
984
- await self._call_event_handler("on_call_state_updated", self, state)
985
-
986
- async def _on_first_participant_joined(self, participant_id: str):
987
- """Handle first participant joined events."""
988
- await self._call_event_handler("on_first_participant_joined", participant_id)
14
+ import warnings
15
+
16
+ from pipecat.transports.livekit.transport import *
17
+
18
+ with warnings.catch_warnings():
19
+ warnings.simplefilter("always")
20
+ warnings.warn(
21
+ "Module `pipecat.transports.services.livekit` is deprecated, "
22
+ "use `pipecat.transports.livekit.transport` instead.",
23
+ DeprecationWarning,
24
+ stacklevel=2,
25
+ )