dv-pipecat-ai 0.0.74.dev770__py3-none-any.whl → 0.0.82.dev776__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dv-pipecat-ai might be problematic. Click here for more details.

Files changed (244) hide show
  1. {dv_pipecat_ai-0.0.74.dev770.dist-info → dv_pipecat_ai-0.0.82.dev776.dist-info}/METADATA +137 -93
  2. dv_pipecat_ai-0.0.82.dev776.dist-info/RECORD +340 -0
  3. pipecat/__init__.py +17 -0
  4. pipecat/adapters/base_llm_adapter.py +36 -1
  5. pipecat/adapters/schemas/direct_function.py +296 -0
  6. pipecat/adapters/schemas/function_schema.py +15 -6
  7. pipecat/adapters/schemas/tools_schema.py +55 -7
  8. pipecat/adapters/services/anthropic_adapter.py +22 -3
  9. pipecat/adapters/services/aws_nova_sonic_adapter.py +23 -3
  10. pipecat/adapters/services/bedrock_adapter.py +22 -3
  11. pipecat/adapters/services/gemini_adapter.py +16 -3
  12. pipecat/adapters/services/open_ai_adapter.py +17 -2
  13. pipecat/adapters/services/open_ai_realtime_adapter.py +23 -3
  14. pipecat/audio/filters/base_audio_filter.py +30 -6
  15. pipecat/audio/filters/koala_filter.py +37 -2
  16. pipecat/audio/filters/krisp_filter.py +59 -6
  17. pipecat/audio/filters/noisereduce_filter.py +37 -0
  18. pipecat/audio/interruptions/base_interruption_strategy.py +25 -5
  19. pipecat/audio/interruptions/min_words_interruption_strategy.py +21 -4
  20. pipecat/audio/mixers/base_audio_mixer.py +30 -7
  21. pipecat/audio/mixers/soundfile_mixer.py +53 -6
  22. pipecat/audio/resamplers/base_audio_resampler.py +17 -9
  23. pipecat/audio/resamplers/resampy_resampler.py +26 -1
  24. pipecat/audio/resamplers/soxr_resampler.py +32 -1
  25. pipecat/audio/resamplers/soxr_stream_resampler.py +101 -0
  26. pipecat/audio/utils.py +194 -1
  27. pipecat/audio/vad/silero.py +60 -3
  28. pipecat/audio/vad/vad_analyzer.py +114 -30
  29. pipecat/clocks/base_clock.py +19 -0
  30. pipecat/clocks/system_clock.py +25 -0
  31. pipecat/extensions/voicemail/__init__.py +0 -0
  32. pipecat/extensions/voicemail/voicemail_detector.py +707 -0
  33. pipecat/frames/frames.py +590 -156
  34. pipecat/metrics/metrics.py +64 -1
  35. pipecat/observers/base_observer.py +58 -19
  36. pipecat/observers/loggers/debug_log_observer.py +56 -64
  37. pipecat/observers/loggers/llm_log_observer.py +8 -1
  38. pipecat/observers/loggers/transcription_log_observer.py +19 -7
  39. pipecat/observers/loggers/user_bot_latency_log_observer.py +32 -5
  40. pipecat/observers/turn_tracking_observer.py +26 -1
  41. pipecat/pipeline/base_pipeline.py +5 -7
  42. pipecat/pipeline/base_task.py +52 -9
  43. pipecat/pipeline/parallel_pipeline.py +121 -177
  44. pipecat/pipeline/pipeline.py +129 -20
  45. pipecat/pipeline/runner.py +50 -1
  46. pipecat/pipeline/sync_parallel_pipeline.py +132 -32
  47. pipecat/pipeline/task.py +263 -280
  48. pipecat/pipeline/task_observer.py +85 -34
  49. pipecat/pipeline/to_be_updated/merge_pipeline.py +32 -2
  50. pipecat/processors/aggregators/dtmf_aggregator.py +29 -22
  51. pipecat/processors/aggregators/gated.py +25 -24
  52. pipecat/processors/aggregators/gated_openai_llm_context.py +22 -2
  53. pipecat/processors/aggregators/llm_response.py +398 -89
  54. pipecat/processors/aggregators/openai_llm_context.py +161 -13
  55. pipecat/processors/aggregators/sentence.py +25 -14
  56. pipecat/processors/aggregators/user_response.py +28 -3
  57. pipecat/processors/aggregators/vision_image_frame.py +24 -14
  58. pipecat/processors/async_generator.py +28 -0
  59. pipecat/processors/audio/audio_buffer_processor.py +78 -37
  60. pipecat/processors/consumer_processor.py +25 -6
  61. pipecat/processors/filters/frame_filter.py +23 -0
  62. pipecat/processors/filters/function_filter.py +30 -0
  63. pipecat/processors/filters/identity_filter.py +17 -2
  64. pipecat/processors/filters/null_filter.py +24 -1
  65. pipecat/processors/filters/stt_mute_filter.py +56 -21
  66. pipecat/processors/filters/wake_check_filter.py +46 -3
  67. pipecat/processors/filters/wake_notifier_filter.py +21 -3
  68. pipecat/processors/frame_processor.py +488 -131
  69. pipecat/processors/frameworks/langchain.py +38 -3
  70. pipecat/processors/frameworks/rtvi.py +719 -34
  71. pipecat/processors/gstreamer/pipeline_source.py +41 -0
  72. pipecat/processors/idle_frame_processor.py +26 -3
  73. pipecat/processors/logger.py +23 -0
  74. pipecat/processors/metrics/frame_processor_metrics.py +77 -4
  75. pipecat/processors/metrics/sentry.py +42 -4
  76. pipecat/processors/producer_processor.py +34 -14
  77. pipecat/processors/text_transformer.py +22 -10
  78. pipecat/processors/transcript_processor.py +48 -29
  79. pipecat/processors/user_idle_processor.py +31 -21
  80. pipecat/runner/__init__.py +1 -0
  81. pipecat/runner/daily.py +132 -0
  82. pipecat/runner/livekit.py +148 -0
  83. pipecat/runner/run.py +543 -0
  84. pipecat/runner/types.py +67 -0
  85. pipecat/runner/utils.py +515 -0
  86. pipecat/serializers/base_serializer.py +42 -0
  87. pipecat/serializers/exotel.py +17 -6
  88. pipecat/serializers/genesys.py +95 -0
  89. pipecat/serializers/livekit.py +33 -0
  90. pipecat/serializers/plivo.py +16 -15
  91. pipecat/serializers/protobuf.py +37 -1
  92. pipecat/serializers/telnyx.py +18 -17
  93. pipecat/serializers/twilio.py +32 -16
  94. pipecat/services/ai_service.py +5 -3
  95. pipecat/services/anthropic/llm.py +113 -43
  96. pipecat/services/assemblyai/models.py +63 -5
  97. pipecat/services/assemblyai/stt.py +64 -11
  98. pipecat/services/asyncai/__init__.py +0 -0
  99. pipecat/services/asyncai/tts.py +501 -0
  100. pipecat/services/aws/llm.py +185 -111
  101. pipecat/services/aws/stt.py +217 -23
  102. pipecat/services/aws/tts.py +118 -52
  103. pipecat/services/aws/utils.py +101 -5
  104. pipecat/services/aws_nova_sonic/aws.py +82 -64
  105. pipecat/services/aws_nova_sonic/context.py +15 -6
  106. pipecat/services/azure/common.py +10 -2
  107. pipecat/services/azure/image.py +32 -0
  108. pipecat/services/azure/llm.py +9 -7
  109. pipecat/services/azure/stt.py +65 -2
  110. pipecat/services/azure/tts.py +154 -23
  111. pipecat/services/cartesia/stt.py +125 -8
  112. pipecat/services/cartesia/tts.py +102 -38
  113. pipecat/services/cerebras/llm.py +15 -23
  114. pipecat/services/deepgram/stt.py +19 -11
  115. pipecat/services/deepgram/tts.py +36 -0
  116. pipecat/services/deepseek/llm.py +14 -23
  117. pipecat/services/elevenlabs/tts.py +330 -64
  118. pipecat/services/fal/image.py +43 -0
  119. pipecat/services/fal/stt.py +48 -10
  120. pipecat/services/fireworks/llm.py +14 -21
  121. pipecat/services/fish/tts.py +109 -9
  122. pipecat/services/gemini_multimodal_live/__init__.py +1 -0
  123. pipecat/services/gemini_multimodal_live/events.py +83 -2
  124. pipecat/services/gemini_multimodal_live/file_api.py +189 -0
  125. pipecat/services/gemini_multimodal_live/gemini.py +218 -21
  126. pipecat/services/gladia/config.py +17 -10
  127. pipecat/services/gladia/stt.py +82 -36
  128. pipecat/services/google/frames.py +40 -0
  129. pipecat/services/google/google.py +2 -0
  130. pipecat/services/google/image.py +39 -2
  131. pipecat/services/google/llm.py +176 -58
  132. pipecat/services/google/llm_openai.py +26 -4
  133. pipecat/services/google/llm_vertex.py +37 -15
  134. pipecat/services/google/rtvi.py +41 -0
  135. pipecat/services/google/stt.py +65 -17
  136. pipecat/services/google/test-google-chirp.py +45 -0
  137. pipecat/services/google/tts.py +390 -19
  138. pipecat/services/grok/llm.py +8 -6
  139. pipecat/services/groq/llm.py +8 -6
  140. pipecat/services/groq/stt.py +13 -9
  141. pipecat/services/groq/tts.py +40 -0
  142. pipecat/services/hamsa/__init__.py +9 -0
  143. pipecat/services/hamsa/stt.py +241 -0
  144. pipecat/services/heygen/__init__.py +5 -0
  145. pipecat/services/heygen/api.py +281 -0
  146. pipecat/services/heygen/client.py +620 -0
  147. pipecat/services/heygen/video.py +338 -0
  148. pipecat/services/image_service.py +5 -3
  149. pipecat/services/inworld/__init__.py +1 -0
  150. pipecat/services/inworld/tts.py +592 -0
  151. pipecat/services/llm_service.py +127 -45
  152. pipecat/services/lmnt/tts.py +80 -7
  153. pipecat/services/mcp_service.py +85 -44
  154. pipecat/services/mem0/memory.py +42 -13
  155. pipecat/services/minimax/tts.py +74 -15
  156. pipecat/services/mistral/__init__.py +0 -0
  157. pipecat/services/mistral/llm.py +185 -0
  158. pipecat/services/moondream/vision.py +55 -10
  159. pipecat/services/neuphonic/tts.py +275 -48
  160. pipecat/services/nim/llm.py +8 -6
  161. pipecat/services/ollama/llm.py +27 -7
  162. pipecat/services/openai/base_llm.py +54 -16
  163. pipecat/services/openai/image.py +30 -0
  164. pipecat/services/openai/llm.py +7 -5
  165. pipecat/services/openai/stt.py +13 -9
  166. pipecat/services/openai/tts.py +42 -10
  167. pipecat/services/openai_realtime_beta/azure.py +11 -9
  168. pipecat/services/openai_realtime_beta/context.py +7 -5
  169. pipecat/services/openai_realtime_beta/events.py +10 -7
  170. pipecat/services/openai_realtime_beta/openai.py +37 -18
  171. pipecat/services/openpipe/llm.py +30 -24
  172. pipecat/services/openrouter/llm.py +9 -7
  173. pipecat/services/perplexity/llm.py +15 -19
  174. pipecat/services/piper/tts.py +26 -12
  175. pipecat/services/playht/tts.py +227 -65
  176. pipecat/services/qwen/llm.py +8 -6
  177. pipecat/services/rime/tts.py +128 -17
  178. pipecat/services/riva/stt.py +160 -22
  179. pipecat/services/riva/tts.py +67 -2
  180. pipecat/services/sambanova/llm.py +19 -17
  181. pipecat/services/sambanova/stt.py +14 -8
  182. pipecat/services/sarvam/tts.py +60 -13
  183. pipecat/services/simli/video.py +82 -21
  184. pipecat/services/soniox/__init__.py +0 -0
  185. pipecat/services/soniox/stt.py +398 -0
  186. pipecat/services/speechmatics/stt.py +29 -17
  187. pipecat/services/stt_service.py +47 -11
  188. pipecat/services/tavus/video.py +94 -25
  189. pipecat/services/together/llm.py +8 -6
  190. pipecat/services/tts_service.py +77 -53
  191. pipecat/services/ultravox/stt.py +46 -43
  192. pipecat/services/vision_service.py +5 -3
  193. pipecat/services/websocket_service.py +12 -11
  194. pipecat/services/whisper/base_stt.py +58 -12
  195. pipecat/services/whisper/stt.py +69 -58
  196. pipecat/services/xtts/tts.py +59 -2
  197. pipecat/sync/base_notifier.py +19 -0
  198. pipecat/sync/event_notifier.py +24 -0
  199. pipecat/tests/utils.py +73 -5
  200. pipecat/transcriptions/language.py +24 -0
  201. pipecat/transports/base_input.py +112 -8
  202. pipecat/transports/base_output.py +235 -13
  203. pipecat/transports/base_transport.py +119 -0
  204. pipecat/transports/local/audio.py +76 -0
  205. pipecat/transports/local/tk.py +84 -0
  206. pipecat/transports/network/fastapi_websocket.py +174 -15
  207. pipecat/transports/network/small_webrtc.py +383 -39
  208. pipecat/transports/network/webrtc_connection.py +214 -8
  209. pipecat/transports/network/websocket_client.py +171 -1
  210. pipecat/transports/network/websocket_server.py +147 -9
  211. pipecat/transports/services/daily.py +792 -70
  212. pipecat/transports/services/helpers/daily_rest.py +122 -129
  213. pipecat/transports/services/livekit.py +339 -4
  214. pipecat/transports/services/tavus.py +273 -38
  215. pipecat/utils/asyncio/task_manager.py +92 -186
  216. pipecat/utils/base_object.py +83 -1
  217. pipecat/utils/network.py +2 -0
  218. pipecat/utils/string.py +114 -58
  219. pipecat/utils/text/base_text_aggregator.py +44 -13
  220. pipecat/utils/text/base_text_filter.py +46 -0
  221. pipecat/utils/text/markdown_text_filter.py +70 -14
  222. pipecat/utils/text/pattern_pair_aggregator.py +18 -14
  223. pipecat/utils/text/simple_text_aggregator.py +43 -2
  224. pipecat/utils/text/skip_tags_aggregator.py +21 -13
  225. pipecat/utils/time.py +36 -0
  226. pipecat/utils/tracing/class_decorators.py +32 -7
  227. pipecat/utils/tracing/conversation_context_provider.py +12 -2
  228. pipecat/utils/tracing/service_attributes.py +80 -64
  229. pipecat/utils/tracing/service_decorators.py +48 -21
  230. pipecat/utils/tracing/setup.py +13 -7
  231. pipecat/utils/tracing/turn_context_provider.py +12 -2
  232. pipecat/utils/tracing/turn_trace_observer.py +27 -0
  233. pipecat/utils/utils.py +14 -14
  234. dv_pipecat_ai-0.0.74.dev770.dist-info/RECORD +0 -319
  235. pipecat/examples/daily_runner.py +0 -64
  236. pipecat/examples/run.py +0 -265
  237. pipecat/utils/asyncio/watchdog_async_iterator.py +0 -72
  238. pipecat/utils/asyncio/watchdog_event.py +0 -42
  239. pipecat/utils/asyncio/watchdog_priority_queue.py +0 -48
  240. pipecat/utils/asyncio/watchdog_queue.py +0 -48
  241. {dv_pipecat_ai-0.0.74.dev770.dist-info → dv_pipecat_ai-0.0.82.dev776.dist-info}/WHEEL +0 -0
  242. {dv_pipecat_ai-0.0.74.dev770.dist-info → dv_pipecat_ai-0.0.82.dev776.dist-info}/licenses/LICENSE +0 -0
  243. {dv_pipecat_ai-0.0.74.dev770.dist-info → dv_pipecat_ai-0.0.82.dev776.dist-info}/top_level.txt +0 -0
  244. /pipecat/{examples → extensions}/__init__.py +0 -0
@@ -4,6 +4,13 @@
4
4
  # SPDX-License-Identifier: BSD 2-Clause License
5
5
  #
6
6
 
7
+ """LiveKit transport implementation for Pipecat.
8
+
9
+ This module provides comprehensive LiveKit real-time communication integration
10
+ including audio streaming, data messaging, participant management, and room
11
+ event handling for conversational AI applications.
12
+ """
13
+
7
14
  import asyncio
8
15
  from dataclasses import dataclass
9
16
  from typing import Any, Awaitable, Callable, List, Optional
@@ -11,7 +18,7 @@ from typing import Any, Awaitable, Callable, List, Optional
11
18
  from loguru import logger
12
19
  from pydantic import BaseModel
13
20
 
14
- from pipecat.audio.utils import create_default_resampler
21
+ from pipecat.audio.utils import create_stream_resampler
15
22
  from pipecat.audio.vad.vad_analyzer import VADAnalyzer
16
23
  from pipecat.frames.frames import (
17
24
  AudioRawFrame,
@@ -28,7 +35,6 @@ from pipecat.transports.base_input import BaseInputTransport
28
35
  from pipecat.transports.base_output import BaseOutputTransport
29
36
  from pipecat.transports.base_transport import BaseTransport, TransportParams
30
37
  from pipecat.utils.asyncio.task_manager import BaseTaskManager
31
- from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator
32
38
 
33
39
  try:
34
40
  from livekit import rtc
@@ -41,19 +47,49 @@ except ModuleNotFoundError as e:
41
47
 
42
48
  @dataclass
43
49
  class LiveKitTransportMessageFrame(TransportMessageFrame):
50
+ """Frame for transport messages in LiveKit rooms.
51
+
52
+ Parameters:
53
+ participant_id: Optional ID of the participant this message is for/from.
54
+ """
55
+
44
56
  participant_id: Optional[str] = None
45
57
 
46
58
 
47
59
  @dataclass
48
60
  class LiveKitTransportMessageUrgentFrame(TransportMessageUrgentFrame):
61
+ """Frame for urgent transport messages in LiveKit rooms.
62
+
63
+ Parameters:
64
+ participant_id: Optional ID of the participant this message is for/from.
65
+ """
66
+
49
67
  participant_id: Optional[str] = None
50
68
 
51
69
 
52
70
  class LiveKitParams(TransportParams):
71
+ """Configuration parameters for LiveKit transport.
72
+
73
+ Inherits all parameters from TransportParams without additional configuration.
74
+ """
75
+
53
76
  pass
54
77
 
55
78
 
56
79
  class LiveKitCallbacks(BaseModel):
80
+ """Callback handlers for LiveKit events.
81
+
82
+ Parameters:
83
+ on_connected: Called when connected to the LiveKit room.
84
+ on_disconnected: Called when disconnected from the LiveKit room.
85
+ on_participant_connected: Called when a participant joins the room.
86
+ on_participant_disconnected: Called when a participant leaves the room.
87
+ on_audio_track_subscribed: Called when an audio track is subscribed.
88
+ on_audio_track_unsubscribed: Called when an audio track is unsubscribed.
89
+ on_data_received: Called when data is received from a participant.
90
+ on_first_participant_joined: Called when the first participant joins.
91
+ """
92
+
57
93
  on_connected: Callable[[], Awaitable[None]]
58
94
  on_disconnected: Callable[[], Awaitable[None]]
59
95
  on_participant_connected: Callable[[str], Awaitable[None]]
@@ -65,6 +101,12 @@ class LiveKitCallbacks(BaseModel):
65
101
 
66
102
 
67
103
  class LiveKitTransportClient:
104
+ """Core client for interacting with LiveKit rooms.
105
+
106
+ Manages the connection to LiveKit rooms and handles all low-level API interactions
107
+ including room management, audio streaming, data messaging, and event handling.
108
+ """
109
+
68
110
  def __init__(
69
111
  self,
70
112
  url: str,
@@ -74,6 +116,16 @@ class LiveKitTransportClient:
74
116
  callbacks: LiveKitCallbacks,
75
117
  transport_name: str,
76
118
  ):
119
+ """Initialize the LiveKit transport client.
120
+
121
+ Args:
122
+ url: LiveKit server URL to connect to.
123
+ token: Authentication token for the room.
124
+ room_name: Name of the LiveKit room to join.
125
+ params: Configuration parameters for the transport.
126
+ callbacks: Event callback handlers.
127
+ transport_name: Name identifier for the transport.
128
+ """
77
129
  self._url = url
78
130
  self._token = token
79
131
  self._room_name = room_name
@@ -93,15 +145,33 @@ class LiveKitTransportClient:
93
145
 
94
146
  @property
95
147
  def participant_id(self) -> str:
148
+ """Get the participant ID for this client.
149
+
150
+ Returns:
151
+ The participant ID assigned by LiveKit.
152
+ """
96
153
  return self._participant_id
97
154
 
98
155
  @property
99
156
  def room(self) -> rtc.Room:
157
+ """Get the LiveKit room instance.
158
+
159
+ Returns:
160
+ The LiveKit room object.
161
+
162
+ Raises:
163
+ Exception: If room object is not available.
164
+ """
100
165
  if not self._room:
101
166
  raise Exception(f"{self}: missing room object (pipeline not started?)")
102
167
  return self._room
103
168
 
104
169
  async def setup(self, setup: FrameProcessorSetup):
170
+ """Setup the client with task manager and room initialization.
171
+
172
+ Args:
173
+ setup: The frame processor setup configuration.
174
+ """
105
175
  if self._task_manager:
106
176
  return
107
177
 
@@ -118,13 +188,20 @@ class LiveKitTransportClient:
118
188
  self.room.on("disconnected")(self._on_disconnected_wrapper)
119
189
 
120
190
  async def cleanup(self):
191
+ """Cleanup client resources."""
121
192
  await self.disconnect()
122
193
 
123
194
  async def start(self, frame: StartFrame):
195
+ """Start the client and initialize audio components.
196
+
197
+ Args:
198
+ frame: The start frame containing initialization parameters.
199
+ """
124
200
  self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
125
201
 
126
202
  @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
127
203
  async def connect(self):
204
+ """Connect to the LiveKit room with retry logic."""
128
205
  if self._connected:
129
206
  # Increment disconnect counter if already connected.
130
207
  self._disconnect_counter += 1
@@ -168,6 +245,7 @@ class LiveKitTransportClient:
168
245
  raise
169
246
 
170
247
  async def disconnect(self):
248
+ """Disconnect from the LiveKit room."""
171
249
  # Decrement leave counter when leaving.
172
250
  self._disconnect_counter -= 1
173
251
 
@@ -181,6 +259,12 @@ class LiveKitTransportClient:
181
259
  await self._callbacks.on_disconnected()
182
260
 
183
261
  async def send_data(self, data: bytes, participant_id: Optional[str] = None):
262
+ """Send data to participants in the room.
263
+
264
+ Args:
265
+ data: The data bytes to send.
266
+ participant_id: Optional specific participant to send to.
267
+ """
184
268
  if not self._connected:
185
269
  return
186
270
 
@@ -195,6 +279,11 @@ class LiveKitTransportClient:
195
279
  logger.error(f"Error sending data: {e}")
196
280
 
197
281
  async def publish_audio(self, audio_frame: rtc.AudioFrame):
282
+ """Publish an audio frame to the room.
283
+
284
+ Args:
285
+ audio_frame: The LiveKit audio frame to publish.
286
+ """
198
287
  if not self._connected or not self._audio_source:
199
288
  return
200
289
 
@@ -204,9 +293,22 @@ class LiveKitTransportClient:
204
293
  logger.error(f"Error publishing audio: {e}")
205
294
 
206
295
  def get_participants(self) -> List[str]:
296
+ """Get list of participant IDs in the room.
297
+
298
+ Returns:
299
+ List of participant IDs.
300
+ """
207
301
  return [p.sid for p in self.room.remote_participants.values()]
208
302
 
209
303
  async def get_participant_metadata(self, participant_id: str) -> dict:
304
+ """Get metadata for a specific participant.
305
+
306
+ Args:
307
+ participant_id: ID of the participant to get metadata for.
308
+
309
+ Returns:
310
+ Dictionary containing participant metadata.
311
+ """
210
312
  participant = self.room.remote_participants.get(participant_id)
211
313
  if participant:
212
314
  return {
@@ -218,9 +320,19 @@ class LiveKitTransportClient:
218
320
  return {}
219
321
 
220
322
  async def set_participant_metadata(self, metadata: str):
323
+ """Set metadata for the local participant.
324
+
325
+ Args:
326
+ metadata: Metadata string to set.
327
+ """
221
328
  await self.room.local_participant.set_metadata(metadata)
222
329
 
223
330
  async def mute_participant(self, participant_id: str):
331
+ """Mute a specific participant's audio tracks.
332
+
333
+ Args:
334
+ participant_id: ID of the participant to mute.
335
+ """
224
336
  participant = self.room.remote_participants.get(participant_id)
225
337
  if participant:
226
338
  for track in participant.tracks.values():
@@ -228,6 +340,11 @@ class LiveKitTransportClient:
228
340
  await track.set_enabled(False)
229
341
 
230
342
  async def unmute_participant(self, participant_id: str):
343
+ """Unmute a specific participant's audio tracks.
344
+
345
+ Args:
346
+ participant_id: ID of the participant to unmute.
347
+ """
231
348
  participant = self.room.remote_participants.get(participant_id)
232
349
  if participant:
233
350
  for track in participant.tracks.values():
@@ -236,12 +353,14 @@ class LiveKitTransportClient:
236
353
 
237
354
  # Wrapper methods for event handlers
238
355
  def _on_participant_connected_wrapper(self, participant: rtc.RemoteParticipant):
356
+ """Wrapper for participant connected events."""
239
357
  self._task_manager.create_task(
240
358
  self._async_on_participant_connected(participant),
241
359
  f"{self}::_async_on_participant_connected",
242
360
  )
243
361
 
244
362
  def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant):
363
+ """Wrapper for participant disconnected events."""
245
364
  self._task_manager.create_task(
246
365
  self._async_on_participant_disconnected(participant),
247
366
  f"{self}::_async_on_participant_disconnected",
@@ -253,6 +372,7 @@ class LiveKitTransportClient:
253
372
  publication: rtc.RemoteTrackPublication,
254
373
  participant: rtc.RemoteParticipant,
255
374
  ):
375
+ """Wrapper for track subscribed events."""
256
376
  self._task_manager.create_task(
257
377
  self._async_on_track_subscribed(track, publication, participant),
258
378
  f"{self}::_async_on_track_subscribed",
@@ -264,27 +384,32 @@ class LiveKitTransportClient:
264
384
  publication: rtc.RemoteTrackPublication,
265
385
  participant: rtc.RemoteParticipant,
266
386
  ):
387
+ """Wrapper for track unsubscribed events."""
267
388
  self._task_manager.create_task(
268
389
  self._async_on_track_unsubscribed(track, publication, participant),
269
390
  f"{self}::_async_on_track_unsubscribed",
270
391
  )
271
392
 
272
393
  def _on_data_received_wrapper(self, data: rtc.DataPacket):
394
+ """Wrapper for data received events."""
273
395
  self._task_manager.create_task(
274
396
  self._async_on_data_received(data),
275
397
  f"{self}::_async_on_data_received",
276
398
  )
277
399
 
278
400
  def _on_connected_wrapper(self):
401
+ """Wrapper for connected events."""
279
402
  self._task_manager.create_task(self._async_on_connected(), f"{self}::_async_on_connected")
280
403
 
281
404
  def _on_disconnected_wrapper(self):
405
+ """Wrapper for disconnected events."""
282
406
  self._task_manager.create_task(
283
407
  self._async_on_disconnected(), f"{self}::_async_on_disconnected"
284
408
  )
285
409
 
286
410
  # Async methods for event handling
287
411
  async def _async_on_participant_connected(self, participant: rtc.RemoteParticipant):
412
+ """Handle participant connected events."""
288
413
  logger.info(f"Participant connected: {participant.identity}")
289
414
  await self._callbacks.on_participant_connected(participant.sid)
290
415
  if not self._other_participant_has_joined:
@@ -292,6 +417,7 @@ class LiveKitTransportClient:
292
417
  await self._callbacks.on_first_participant_joined(participant.sid)
293
418
 
294
419
  async def _async_on_participant_disconnected(self, participant: rtc.RemoteParticipant):
420
+ """Handle participant disconnected events."""
295
421
  logger.info(f"Participant disconnected: {participant.identity}")
296
422
  await self._callbacks.on_participant_disconnected(participant.sid)
297
423
  if len(self.get_participants()) == 0:
@@ -303,6 +429,7 @@ class LiveKitTransportClient:
303
429
  publication: rtc.RemoteTrackPublication,
304
430
  participant: rtc.RemoteParticipant,
305
431
  ):
432
+ """Handle track subscribed events."""
306
433
  if track.kind == rtc.TrackKind.KIND_AUDIO:
307
434
  logger.info(f"Audio track subscribed: {track.sid} from participant {participant.sid}")
308
435
  self._audio_tracks[participant.sid] = track
@@ -311,6 +438,7 @@ class LiveKitTransportClient:
311
438
  self._process_audio_stream(audio_stream, participant.sid),
312
439
  f"{self}::_process_audio_stream",
313
440
  )
441
+ await self._callbacks.on_audio_track_subscribed(participant.sid)
314
442
 
315
443
  async def _async_on_track_unsubscribed(
316
444
  self,
@@ -318,22 +446,27 @@ class LiveKitTransportClient:
318
446
  publication: rtc.RemoteTrackPublication,
319
447
  participant: rtc.RemoteParticipant,
320
448
  ):
449
+ """Handle track unsubscribed events."""
321
450
  logger.info(f"Track unsubscribed: {publication.sid} from {participant.identity}")
322
451
  if track.kind == rtc.TrackKind.KIND_AUDIO:
323
452
  await self._callbacks.on_audio_track_unsubscribed(participant.sid)
324
453
 
325
454
  async def _async_on_data_received(self, data: rtc.DataPacket):
455
+ """Handle data received events."""
326
456
  await self._callbacks.on_data_received(data.data, data.participant.sid)
327
457
 
328
458
  async def _async_on_connected(self):
459
+ """Handle connected events."""
329
460
  await self._callbacks.on_connected()
330
461
 
331
462
  async def _async_on_disconnected(self, reason=None):
463
+ """Handle disconnected events."""
332
464
  self._connected = False
333
465
  logger.info(f"Disconnected from {self._room_name}. Reason: {reason}")
334
466
  await self._callbacks.on_disconnected()
335
467
 
336
468
  async def _process_audio_stream(self, audio_stream: rtc.AudioStream, participant_id: str):
469
+ """Process incoming audio stream from a participant."""
337
470
  logger.info(f"Started processing audio stream for participant {participant_id}")
338
471
  async for event in audio_stream:
339
472
  if isinstance(event, rtc.AudioFrameEvent):
@@ -342,15 +475,23 @@ class LiveKitTransportClient:
342
475
  logger.warning(f"Received unexpected event type: {type(event)}")
343
476
 
344
477
  async def get_next_audio_frame(self):
478
+ """Get the next audio frame from the queue."""
345
479
  while True:
346
480
  frame, participant_id = await self._audio_queue.get()
347
481
  yield frame, participant_id
348
482
 
349
483
  def __str__(self):
484
+ """String representation of the LiveKit transport client."""
350
485
  return f"{self._transport_name}::LiveKitTransportClient"
351
486
 
352
487
 
353
488
  class LiveKitInputTransport(BaseInputTransport):
489
+ """Handles incoming media streams and events from LiveKit rooms.
490
+
491
+ Processes incoming audio streams from room participants and forwards them
492
+ as Pipecat frames, including audio resampling and VAD integration.
493
+ """
494
+
354
495
  def __init__(
355
496
  self,
356
497
  transport: BaseTransport,
@@ -358,22 +499,40 @@ class LiveKitInputTransport(BaseInputTransport):
358
499
  params: LiveKitParams,
359
500
  **kwargs,
360
501
  ):
502
+ """Initialize the LiveKit input transport.
503
+
504
+ Args:
505
+ transport: The parent transport instance.
506
+ client: LiveKitTransportClient instance.
507
+ params: Configuration parameters.
508
+ **kwargs: Additional arguments passed to parent class.
509
+ """
361
510
  super().__init__(params, **kwargs)
362
511
  self._transport = transport
363
512
  self._client = client
364
513
 
365
514
  self._audio_in_task = None
366
515
  self._vad_analyzer: Optional[VADAnalyzer] = params.vad_analyzer
367
- self._resampler = create_default_resampler()
516
+ self._resampler = create_stream_resampler()
368
517
 
369
518
  # Whether we have seen a StartFrame already.
370
519
  self._initialized = False
371
520
 
372
521
  @property
373
522
  def vad_analyzer(self) -> Optional[VADAnalyzer]:
523
+ """Get the Voice Activity Detection analyzer.
524
+
525
+ Returns:
526
+ The VAD analyzer instance if configured.
527
+ """
374
528
  return self._vad_analyzer
375
529
 
376
530
  async def start(self, frame: StartFrame):
531
+ """Start the input transport and connect to LiveKit room.
532
+
533
+ Args:
534
+ frame: The start frame containing initialization parameters.
535
+ """
377
536
  await super().start(frame)
378
537
 
379
538
  if self._initialized:
@@ -389,6 +548,11 @@ class LiveKitInputTransport(BaseInputTransport):
389
548
  logger.info("LiveKitInputTransport started")
390
549
 
391
550
  async def stop(self, frame: EndFrame):
551
+ """Stop the input transport and disconnect from LiveKit room.
552
+
553
+ Args:
554
+ frame: The end frame signaling transport shutdown.
555
+ """
392
556
  await super().stop(frame)
393
557
  await self._client.disconnect()
394
558
  if self._audio_in_task:
@@ -396,32 +560,55 @@ class LiveKitInputTransport(BaseInputTransport):
396
560
  logger.info("LiveKitInputTransport stopped")
397
561
 
398
562
  async def cancel(self, frame: CancelFrame):
563
+ """Cancel the input transport and disconnect from LiveKit room.
564
+
565
+ Args:
566
+ frame: The cancel frame signaling immediate cancellation.
567
+ """
399
568
  await super().cancel(frame)
400
569
  await self._client.disconnect()
401
570
  if self._audio_in_task and self._params.audio_in_enabled:
402
571
  await self.cancel_task(self._audio_in_task)
403
572
 
404
573
  async def setup(self, setup: FrameProcessorSetup):
574
+ """Setup the input transport with shared client setup.
575
+
576
+ Args:
577
+ setup: The frame processor setup configuration.
578
+ """
405
579
  await super().setup(setup)
406
580
  await self._client.setup(setup)
407
581
 
408
582
  async def cleanup(self):
583
+ """Cleanup input transport and shared resources."""
409
584
  await super().cleanup()
410
585
  await self._transport.cleanup()
411
586
 
412
587
  async def push_app_message(self, message: Any, sender: str):
588
+ """Push an application message as an urgent transport frame.
589
+
590
+ Args:
591
+ message: The message data to send.
592
+ sender: ID of the message sender.
593
+ """
413
594
  frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender)
414
595
  await self.push_frame(frame)
415
596
 
416
597
  async def _audio_in_task_handler(self):
598
+ """Handle incoming audio frames from participants."""
417
599
  logger.info("Audio input task started")
418
600
  audio_iterator = self._client.get_next_audio_frame()
419
- async for audio_data in WatchdogAsyncIterator(audio_iterator, manager=self.task_manager):
601
+ async for audio_data in audio_iterator:
420
602
  if audio_data:
421
603
  audio_frame_event, participant_id = audio_data
422
604
  pipecat_audio_frame = await self._convert_livekit_audio_to_pipecat(
423
605
  audio_frame_event
424
606
  )
607
+
608
+ # Skip frames with no audio data
609
+ if len(pipecat_audio_frame.audio) == 0:
610
+ continue
611
+
425
612
  input_audio_frame = UserAudioRawFrame(
426
613
  user_id=participant_id,
427
614
  audio=pipecat_audio_frame.audio,
@@ -433,6 +620,7 @@ class LiveKitInputTransport(BaseInputTransport):
433
620
  async def _convert_livekit_audio_to_pipecat(
434
621
  self, audio_frame_event: rtc.AudioFrameEvent
435
622
  ) -> AudioRawFrame:
623
+ """Convert LiveKit audio frame to Pipecat audio frame."""
436
624
  audio_frame = audio_frame_event.frame
437
625
 
438
626
  audio_data = await self._resampler.resample(
@@ -447,6 +635,12 @@ class LiveKitInputTransport(BaseInputTransport):
447
635
 
448
636
 
449
637
  class LiveKitOutputTransport(BaseOutputTransport):
638
+ """Handles outgoing media streams and events to LiveKit rooms.
639
+
640
+ Manages sending audio frames and data messages to LiveKit room participants,
641
+ including audio format conversion for LiveKit compatibility.
642
+ """
643
+
450
644
  def __init__(
451
645
  self,
452
646
  transport: BaseTransport,
@@ -454,6 +648,14 @@ class LiveKitOutputTransport(BaseOutputTransport):
454
648
  params: LiveKitParams,
455
649
  **kwargs,
456
650
  ):
651
+ """Initialize the LiveKit output transport.
652
+
653
+ Args:
654
+ transport: The parent transport instance.
655
+ client: LiveKitTransportClient instance.
656
+ params: Configuration parameters.
657
+ **kwargs: Additional arguments passed to parent class.
658
+ """
457
659
  super().__init__(params, **kwargs)
458
660
  self._transport = transport
459
661
  self._client = client
@@ -462,6 +664,11 @@ class LiveKitOutputTransport(BaseOutputTransport):
462
664
  self._initialized = False
463
665
 
464
666
  async def start(self, frame: StartFrame):
667
+ """Start the output transport and connect to LiveKit room.
668
+
669
+ Args:
670
+ frame: The start frame containing initialization parameters.
671
+ """
465
672
  await super().start(frame)
466
673
 
467
674
  if self._initialized:
@@ -475,33 +682,60 @@ class LiveKitOutputTransport(BaseOutputTransport):
475
682
  logger.info("LiveKitOutputTransport started")
476
683
 
477
684
  async def stop(self, frame: EndFrame):
685
+ """Stop the output transport and disconnect from LiveKit room.
686
+
687
+ Args:
688
+ frame: The end frame signaling transport shutdown.
689
+ """
478
690
  await super().stop(frame)
479
691
  await self._client.disconnect()
480
692
  logger.info("LiveKitOutputTransport stopped")
481
693
 
482
694
  async def cancel(self, frame: CancelFrame):
695
+ """Cancel the output transport and disconnect from LiveKit room.
696
+
697
+ Args:
698
+ frame: The cancel frame signaling immediate cancellation.
699
+ """
483
700
  await super().cancel(frame)
484
701
  await self._client.disconnect()
485
702
 
486
703
  async def setup(self, setup: FrameProcessorSetup):
704
+ """Setup the output transport with shared client setup.
705
+
706
+ Args:
707
+ setup: The frame processor setup configuration.
708
+ """
487
709
  await super().setup(setup)
488
710
  await self._client.setup(setup)
489
711
 
490
712
  async def cleanup(self):
713
+ """Cleanup output transport and shared resources."""
491
714
  await super().cleanup()
492
715
  await self._transport.cleanup()
493
716
 
494
717
  async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
718
+ """Send a transport message to participants.
719
+
720
+ Args:
721
+ frame: The transport message frame to send.
722
+ """
495
723
  if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)):
496
724
  await self._client.send_data(frame.message.encode(), frame.participant_id)
497
725
  else:
498
726
  await self._client.send_data(frame.message.encode())
499
727
 
500
728
  async def write_audio_frame(self, frame: OutputAudioRawFrame):
729
+ """Write an audio frame to the LiveKit room.
730
+
731
+ Args:
732
+ frame: The audio frame to write.
733
+ """
501
734
  livekit_audio = self._convert_pipecat_audio_to_livekit(frame.audio)
502
735
  await self._client.publish_audio(livekit_audio)
503
736
 
504
737
  def _convert_pipecat_audio_to_livekit(self, pipecat_audio: bytes) -> rtc.AudioFrame:
738
+ """Convert Pipecat audio data to LiveKit audio frame."""
505
739
  bytes_per_sample = 2 # Assuming 16-bit audio
506
740
  total_samples = len(pipecat_audio) // bytes_per_sample
507
741
  samples_per_channel = total_samples // self._params.audio_out_channels
@@ -515,6 +749,13 @@ class LiveKitOutputTransport(BaseOutputTransport):
515
749
 
516
750
 
517
751
  class LiveKitTransport(BaseTransport):
752
+ """Transport implementation for LiveKit real-time communication.
753
+
754
+ Provides comprehensive LiveKit integration including audio streaming, data
755
+ messaging, participant management, and room event handling for conversational
756
+ AI applications.
757
+ """
758
+
518
759
  def __init__(
519
760
  self,
520
761
  url: str,
@@ -524,6 +765,16 @@ class LiveKitTransport(BaseTransport):
524
765
  input_name: Optional[str] = None,
525
766
  output_name: Optional[str] = None,
526
767
  ):
768
+ """Initialize the LiveKit transport.
769
+
770
+ Args:
771
+ url: LiveKit server URL to connect to.
772
+ token: Authentication token for the room.
773
+ room_name: Name of the LiveKit room to join.
774
+ params: Configuration parameters for the transport.
775
+ input_name: Optional name for the input transport.
776
+ output_name: Optional name for the output transport.
777
+ """
527
778
  super().__init__(input_name=input_name, output_name=output_name)
528
779
 
529
780
  callbacks = LiveKitCallbacks(
@@ -556,6 +807,11 @@ class LiveKitTransport(BaseTransport):
556
807
  self._register_event_handler("on_call_state_updated")
557
808
 
558
809
  def input(self) -> LiveKitInputTransport:
810
+ """Get the input transport for receiving media and events.
811
+
812
+ Returns:
813
+ The LiveKit input transport instance.
814
+ """
559
815
  if not self._input:
560
816
  self._input = LiveKitInputTransport(
561
817
  self, self._client, self._params, name=self._input_name
@@ -563,6 +819,11 @@ class LiveKitTransport(BaseTransport):
563
819
  return self._input
564
820
 
565
821
  def output(self) -> LiveKitOutputTransport:
822
+ """Get the output transport for sending media and events.
823
+
824
+ Returns:
825
+ The LiveKit output transport instance.
826
+ """
566
827
  if not self._output:
567
828
  self._output = LiveKitOutputTransport(
568
829
  self, self._client, self._params, name=self._output_name
@@ -571,41 +832,84 @@ class LiveKitTransport(BaseTransport):
571
832
 
572
833
  @property
573
834
  def participant_id(self) -> str:
835
+ """Get the participant ID for this transport.
836
+
837
+ Returns:
838
+ The participant ID assigned by LiveKit.
839
+ """
574
840
  return self._client.participant_id
575
841
 
576
842
  async def send_audio(self, frame: OutputAudioRawFrame):
843
+ """Send an audio frame to the LiveKit room.
844
+
845
+ Args:
846
+ frame: The audio frame to send.
847
+ """
577
848
  if self._output:
578
849
  await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
579
850
 
580
851
  def get_participants(self) -> List[str]:
852
+ """Get list of participant IDs in the room.
853
+
854
+ Returns:
855
+ List of participant IDs.
856
+ """
581
857
  return self._client.get_participants()
582
858
 
583
859
  async def get_participant_metadata(self, participant_id: str) -> dict:
860
+ """Get metadata for a specific participant.
861
+
862
+ Args:
863
+ participant_id: ID of the participant to get metadata for.
864
+
865
+ Returns:
866
+ Dictionary containing participant metadata.
867
+ """
584
868
  return await self._client.get_participant_metadata(participant_id)
585
869
 
586
870
  async def set_metadata(self, metadata: str):
871
+ """Set metadata for the local participant.
872
+
873
+ Args:
874
+ metadata: Metadata string to set.
875
+ """
587
876
  await self._client.set_participant_metadata(metadata)
588
877
 
589
878
  async def mute_participant(self, participant_id: str):
879
+ """Mute a specific participant's audio tracks.
880
+
881
+ Args:
882
+ participant_id: ID of the participant to mute.
883
+ """
590
884
  await self._client.mute_participant(participant_id)
591
885
 
592
886
  async def unmute_participant(self, participant_id: str):
887
+ """Unmute a specific participant's audio tracks.
888
+
889
+ Args:
890
+ participant_id: ID of the participant to unmute.
891
+ """
593
892
  await self._client.unmute_participant(participant_id)
594
893
 
595
894
  async def _on_connected(self):
895
+ """Handle room connected events."""
596
896
  await self._call_event_handler("on_connected")
597
897
 
598
898
  async def _on_disconnected(self):
899
+ """Handle room disconnected events."""
599
900
  await self._call_event_handler("on_disconnected")
600
901
 
601
902
  async def _on_participant_connected(self, participant_id: str):
903
+ """Handle participant connected events."""
602
904
  await self._call_event_handler("on_participant_connected", participant_id)
603
905
 
604
906
  async def _on_participant_disconnected(self, participant_id: str):
907
+ """Handle participant disconnected events."""
605
908
  await self._call_event_handler("on_participant_disconnected", participant_id)
606
909
  await self._call_event_handler("on_participant_left", participant_id, "disconnected")
607
910
 
608
911
  async def _on_audio_track_subscribed(self, participant_id: str):
912
+ """Handle audio track subscribed events."""
609
913
  await self._call_event_handler("on_audio_track_subscribed", participant_id)
610
914
  participant = self._client.room.remote_participants.get(participant_id)
611
915
  if participant:
@@ -615,19 +919,33 @@ class LiveKitTransport(BaseTransport):
615
919
  )
616
920
 
617
921
  async def _on_audio_track_unsubscribed(self, participant_id: str):
922
+ """Handle audio track unsubscribed events."""
618
923
  await self._call_event_handler("on_audio_track_unsubscribed", participant_id)
619
924
 
620
925
  async def _on_data_received(self, data: bytes, participant_id: str):
926
+ """Handle data received events."""
621
927
  if self._input:
622
928
  await self._input.push_app_message(data.decode(), participant_id)
623
929
  await self._call_event_handler("on_data_received", data, participant_id)
624
930
 
625
931
  async def send_message(self, message: str, participant_id: Optional[str] = None):
932
+ """Send a message to participants in the room.
933
+
934
+ Args:
935
+ message: The message string to send.
936
+ participant_id: Optional specific participant to send to.
937
+ """
626
938
  if self._output:
627
939
  frame = LiveKitTransportMessageFrame(message=message, participant_id=participant_id)
628
940
  await self._output.send_message(frame)
629
941
 
630
942
  async def send_message_urgent(self, message: str, participant_id: Optional[str] = None):
943
+ """Send an urgent message to participants in the room.
944
+
945
+ Args:
946
+ message: The urgent message string to send.
947
+ participant_id: Optional specific participant to send to.
948
+ """
631
949
  if self._output:
632
950
  frame = LiveKitTransportMessageUrgentFrame(
633
951
  message=message, participant_id=participant_id
@@ -635,19 +953,36 @@ class LiveKitTransport(BaseTransport):
635
953
  await self._output.send_message(frame)
636
954
 
637
955
  async def on_room_event(self, event):
956
+ """Handle room events.
957
+
958
+ Args:
959
+ event: The room event to handle.
960
+ """
638
961
  # Handle room events
639
962
  pass
640
963
 
641
964
  async def on_participant_event(self, event):
965
+ """Handle participant events.
966
+
967
+ Args:
968
+ event: The participant event to handle.
969
+ """
642
970
  # Handle participant events
643
971
  pass
644
972
 
645
973
  async def on_track_event(self, event):
974
+ """Handle track events.
975
+
976
+ Args:
977
+ event: The track event to handle.
978
+ """
646
979
  # Handle track events
647
980
  pass
648
981
 
649
982
  async def _on_call_state_updated(self, state: str):
983
+ """Handle call state update events."""
650
984
  await self._call_event_handler("on_call_state_updated", self, state)
651
985
 
652
986
  async def _on_first_participant_joined(self, participant_id: str):
987
+ """Handle first participant joined events."""
653
988
  await self._call_event_handler("on_first_participant_joined", participant_id)