dv-pipecat-ai 0.0.85.dev5__py3-none-any.whl → 0.0.85.dev698__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dv-pipecat-ai might be problematic. Click here for more details.
- {dv_pipecat_ai-0.0.85.dev5.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/METADATA +78 -117
- {dv_pipecat_ai-0.0.85.dev5.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/RECORD +157 -123
- pipecat/adapters/base_llm_adapter.py +38 -1
- pipecat/adapters/services/anthropic_adapter.py +9 -14
- pipecat/adapters/services/aws_nova_sonic_adapter.py +5 -0
- pipecat/adapters/services/bedrock_adapter.py +236 -13
- pipecat/adapters/services/gemini_adapter.py +12 -8
- pipecat/adapters/services/open_ai_adapter.py +19 -7
- pipecat/adapters/services/open_ai_realtime_adapter.py +5 -0
- pipecat/audio/filters/krisp_viva_filter.py +193 -0
- pipecat/audio/filters/noisereduce_filter.py +15 -0
- pipecat/audio/turn/base_turn_analyzer.py +9 -1
- pipecat/audio/turn/smart_turn/base_smart_turn.py +14 -8
- pipecat/audio/turn/smart_turn/data/__init__.py +0 -0
- pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx +0 -0
- pipecat/audio/turn/smart_turn/http_smart_turn.py +6 -2
- pipecat/audio/turn/smart_turn/local_smart_turn.py +1 -1
- pipecat/audio/turn/smart_turn/local_smart_turn_v2.py +1 -1
- pipecat/audio/turn/smart_turn/local_smart_turn_v3.py +124 -0
- pipecat/audio/vad/data/README.md +10 -0
- pipecat/audio/vad/vad_analyzer.py +13 -1
- pipecat/extensions/voicemail/voicemail_detector.py +5 -5
- pipecat/frames/frames.py +120 -87
- pipecat/observers/loggers/debug_log_observer.py +3 -3
- pipecat/observers/loggers/llm_log_observer.py +7 -3
- pipecat/observers/loggers/user_bot_latency_log_observer.py +22 -10
- pipecat/pipeline/runner.py +12 -4
- pipecat/pipeline/service_switcher.py +64 -36
- pipecat/pipeline/task.py +85 -24
- pipecat/processors/aggregators/dtmf_aggregator.py +28 -22
- pipecat/processors/aggregators/{gated_openai_llm_context.py → gated_llm_context.py} +9 -9
- pipecat/processors/aggregators/gated_open_ai_llm_context.py +12 -0
- pipecat/processors/aggregators/llm_response.py +6 -7
- pipecat/processors/aggregators/llm_response_universal.py +19 -15
- pipecat/processors/aggregators/user_response.py +6 -6
- pipecat/processors/aggregators/vision_image_frame.py +24 -2
- pipecat/processors/audio/audio_buffer_processor.py +43 -8
- pipecat/processors/filters/stt_mute_filter.py +2 -0
- pipecat/processors/frame_processor.py +103 -17
- pipecat/processors/frameworks/langchain.py +8 -2
- pipecat/processors/frameworks/rtvi.py +209 -68
- pipecat/processors/frameworks/strands_agents.py +170 -0
- pipecat/processors/logger.py +2 -2
- pipecat/processors/transcript_processor.py +4 -4
- pipecat/processors/user_idle_processor.py +3 -6
- pipecat/runner/run.py +270 -50
- pipecat/runner/types.py +2 -0
- pipecat/runner/utils.py +51 -10
- pipecat/serializers/exotel.py +5 -5
- pipecat/serializers/livekit.py +20 -0
- pipecat/serializers/plivo.py +6 -9
- pipecat/serializers/protobuf.py +6 -5
- pipecat/serializers/telnyx.py +2 -2
- pipecat/serializers/twilio.py +43 -23
- pipecat/services/ai_service.py +2 -6
- pipecat/services/anthropic/llm.py +2 -25
- pipecat/services/asyncai/tts.py +2 -3
- pipecat/services/aws/__init__.py +1 -0
- pipecat/services/aws/llm.py +122 -97
- pipecat/services/aws/nova_sonic/__init__.py +0 -0
- pipecat/services/aws/nova_sonic/context.py +367 -0
- pipecat/services/aws/nova_sonic/frames.py +25 -0
- pipecat/services/aws/nova_sonic/llm.py +1155 -0
- pipecat/services/aws/stt.py +1 -3
- pipecat/services/aws_nova_sonic/__init__.py +19 -1
- pipecat/services/aws_nova_sonic/aws.py +11 -1151
- pipecat/services/aws_nova_sonic/context.py +13 -355
- pipecat/services/aws_nova_sonic/frames.py +13 -17
- pipecat/services/azure/realtime/__init__.py +0 -0
- pipecat/services/azure/realtime/llm.py +65 -0
- pipecat/services/azure/stt.py +15 -0
- pipecat/services/cartesia/tts.py +2 -2
- pipecat/services/deepgram/__init__.py +1 -0
- pipecat/services/deepgram/flux/__init__.py +0 -0
- pipecat/services/deepgram/flux/stt.py +636 -0
- pipecat/services/elevenlabs/__init__.py +2 -1
- pipecat/services/elevenlabs/stt.py +254 -276
- pipecat/services/elevenlabs/tts.py +5 -5
- pipecat/services/fish/tts.py +2 -2
- pipecat/services/gemini_multimodal_live/events.py +38 -524
- pipecat/services/gemini_multimodal_live/file_api.py +23 -173
- pipecat/services/gemini_multimodal_live/gemini.py +41 -1403
- pipecat/services/gladia/stt.py +56 -72
- pipecat/services/google/__init__.py +1 -0
- pipecat/services/google/gemini_live/__init__.py +3 -0
- pipecat/services/google/gemini_live/file_api.py +189 -0
- pipecat/services/google/gemini_live/llm.py +1582 -0
- pipecat/services/google/gemini_live/llm_vertex.py +184 -0
- pipecat/services/google/llm.py +15 -11
- pipecat/services/google/llm_openai.py +3 -3
- pipecat/services/google/llm_vertex.py +86 -16
- pipecat/services/google/tts.py +7 -3
- pipecat/services/heygen/api.py +2 -0
- pipecat/services/heygen/client.py +8 -4
- pipecat/services/heygen/video.py +2 -0
- pipecat/services/hume/__init__.py +5 -0
- pipecat/services/hume/tts.py +220 -0
- pipecat/services/inworld/tts.py +6 -6
- pipecat/services/llm_service.py +15 -5
- pipecat/services/lmnt/tts.py +2 -2
- pipecat/services/mcp_service.py +4 -2
- pipecat/services/mem0/memory.py +6 -5
- pipecat/services/mistral/llm.py +29 -8
- pipecat/services/moondream/vision.py +42 -16
- pipecat/services/neuphonic/tts.py +2 -2
- pipecat/services/openai/__init__.py +1 -0
- pipecat/services/openai/base_llm.py +27 -20
- pipecat/services/openai/realtime/__init__.py +0 -0
- pipecat/services/openai/realtime/context.py +272 -0
- pipecat/services/openai/realtime/events.py +1106 -0
- pipecat/services/openai/realtime/frames.py +37 -0
- pipecat/services/openai/realtime/llm.py +829 -0
- pipecat/services/openai/tts.py +16 -8
- pipecat/services/openai_realtime/__init__.py +27 -0
- pipecat/services/openai_realtime/azure.py +21 -0
- pipecat/services/openai_realtime/context.py +21 -0
- pipecat/services/openai_realtime/events.py +21 -0
- pipecat/services/openai_realtime/frames.py +21 -0
- pipecat/services/openai_realtime_beta/azure.py +16 -0
- pipecat/services/openai_realtime_beta/openai.py +17 -5
- pipecat/services/playht/tts.py +31 -4
- pipecat/services/rime/tts.py +3 -4
- pipecat/services/sarvam/tts.py +2 -6
- pipecat/services/simli/video.py +2 -2
- pipecat/services/speechmatics/stt.py +1 -7
- pipecat/services/stt_service.py +34 -0
- pipecat/services/tavus/video.py +2 -2
- pipecat/services/tts_service.py +9 -9
- pipecat/services/vision_service.py +7 -6
- pipecat/services/vistaar/llm.py +4 -0
- pipecat/tests/utils.py +4 -4
- pipecat/transcriptions/language.py +41 -1
- pipecat/transports/base_input.py +17 -42
- pipecat/transports/base_output.py +42 -26
- pipecat/transports/daily/transport.py +199 -26
- pipecat/transports/heygen/__init__.py +0 -0
- pipecat/transports/heygen/transport.py +381 -0
- pipecat/transports/livekit/transport.py +228 -63
- pipecat/transports/local/audio.py +6 -1
- pipecat/transports/local/tk.py +11 -2
- pipecat/transports/network/fastapi_websocket.py +1 -1
- pipecat/transports/smallwebrtc/connection.py +98 -19
- pipecat/transports/smallwebrtc/request_handler.py +204 -0
- pipecat/transports/smallwebrtc/transport.py +65 -23
- pipecat/transports/tavus/transport.py +23 -12
- pipecat/transports/websocket/client.py +41 -5
- pipecat/transports/websocket/fastapi.py +21 -11
- pipecat/transports/websocket/server.py +14 -7
- pipecat/transports/whatsapp/api.py +8 -0
- pipecat/transports/whatsapp/client.py +47 -0
- pipecat/utils/base_object.py +54 -22
- pipecat/utils/string.py +12 -1
- pipecat/utils/tracing/service_decorators.py +21 -21
- {dv_pipecat_ai-0.0.85.dev5.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/WHEEL +0 -0
- {dv_pipecat_ai-0.0.85.dev5.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/licenses/LICENSE +0 -0
- {dv_pipecat_ai-0.0.85.dev5.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/top_level.txt +0 -0
- /pipecat/services/{aws_nova_sonic → aws/nova_sonic}/ready.wav +0 -0
|
@@ -95,15 +95,20 @@ class SmallWebRTCTrack:
|
|
|
95
95
|
enable/disable control and frame discarding for audio and video streams.
|
|
96
96
|
"""
|
|
97
97
|
|
|
98
|
-
def __init__(self,
|
|
98
|
+
def __init__(self, receiver):
|
|
99
99
|
"""Initialize the WebRTC track wrapper.
|
|
100
100
|
|
|
101
101
|
Args:
|
|
102
|
-
|
|
103
|
-
index: The index of the track in the transceiver (0 for mic, 1 for cam, 2 for screen)
|
|
102
|
+
receiver: The RemoteStreamTrack receiver instance.
|
|
104
103
|
"""
|
|
105
|
-
self.
|
|
104
|
+
self._receiver = receiver
|
|
105
|
+
# Configuring the receiver for not consuming the track by default to prevent memory grow
|
|
106
|
+
self._receiver._enabled = False
|
|
107
|
+
self._track = receiver.track
|
|
106
108
|
self._enabled = True
|
|
109
|
+
self._last_recv_time: float = 0.0
|
|
110
|
+
self._idle_task: Optional[asyncio.Task] = None
|
|
111
|
+
self._idle_timeout: float = 2.0 # seconds before discarding old frames
|
|
107
112
|
|
|
108
113
|
def set_enabled(self, enabled: bool) -> None:
|
|
109
114
|
"""Enable or disable the track.
|
|
@@ -138,13 +143,44 @@ class SmallWebRTCTrack:
|
|
|
138
143
|
async def recv(self) -> Optional[Frame]:
|
|
139
144
|
"""Receive the next frame from the track.
|
|
140
145
|
|
|
146
|
+
Enables the internal receiving state and starts idle watcher.
|
|
147
|
+
|
|
141
148
|
Returns:
|
|
142
149
|
The next frame, except for video tracks, where it returns the frame only if the track is enabled, otherwise, returns None.
|
|
143
150
|
"""
|
|
151
|
+
self._receiver._enabled = True
|
|
152
|
+
self._last_recv_time = time.time()
|
|
153
|
+
|
|
154
|
+
# start idle watcher if not already running
|
|
155
|
+
if not self._idle_task or self._idle_task.done():
|
|
156
|
+
self._idle_task = asyncio.create_task(self._idle_watcher())
|
|
157
|
+
|
|
144
158
|
if not self._enabled and self._track.kind == "video":
|
|
145
159
|
return None
|
|
146
160
|
return await self._track.recv()
|
|
147
161
|
|
|
162
|
+
async def _idle_watcher(self):
|
|
163
|
+
"""Disable receiving if idle for more than _idle_timeout and monitor queue size."""
|
|
164
|
+
while self._receiver._enabled:
|
|
165
|
+
await asyncio.sleep(self._idle_timeout)
|
|
166
|
+
idle_duration = time.time() - self._last_recv_time
|
|
167
|
+
if idle_duration >= self._idle_timeout:
|
|
168
|
+
# discard old frames to prevent memory growth
|
|
169
|
+
logger.debug(
|
|
170
|
+
f"Disabling receiver for {self._track.kind} track after {idle_duration:.2f}s idle"
|
|
171
|
+
)
|
|
172
|
+
await self.discard_old_frames()
|
|
173
|
+
self._receiver._enabled = False
|
|
174
|
+
|
|
175
|
+
def stop(self):
|
|
176
|
+
"""Stop receiving frames from the track."""
|
|
177
|
+
self._receiver._enabled = False
|
|
178
|
+
if self._idle_task:
|
|
179
|
+
self._idle_task.cancel()
|
|
180
|
+
self._idle_task = None
|
|
181
|
+
if self._track:
|
|
182
|
+
self._track.stop()
|
|
183
|
+
|
|
148
184
|
def __getattr__(self, name):
|
|
149
185
|
"""Forward attribute access to the underlying track.
|
|
150
186
|
|
|
@@ -170,11 +206,16 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
170
206
|
for real-time audio/video communication.
|
|
171
207
|
"""
|
|
172
208
|
|
|
173
|
-
def __init__(
|
|
209
|
+
def __init__(
|
|
210
|
+
self,
|
|
211
|
+
ice_servers: Optional[Union[List[str], List[IceServer]]] = None,
|
|
212
|
+
connection_timeout_secs: int = 60,
|
|
213
|
+
):
|
|
174
214
|
"""Initialize the WebRTC connection.
|
|
175
215
|
|
|
176
216
|
Args:
|
|
177
217
|
ice_servers: List of ICE servers as URLs or IceServer objects.
|
|
218
|
+
connection_timeout_secs: Timeout in seconds for connecting to the peer.
|
|
178
219
|
|
|
179
220
|
Raises:
|
|
180
221
|
TypeError: If ice_servers contains mixed types or unsupported types.
|
|
@@ -195,6 +236,7 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
195
236
|
VIDEO_TRANSCEIVER_INDEX: self.video_input_track,
|
|
196
237
|
SCREEN_VIDEO_TRANSCEIVER_INDEX: self.screen_video_input_track,
|
|
197
238
|
}
|
|
239
|
+
self.connection_timeout_secs = connection_timeout_secs
|
|
198
240
|
|
|
199
241
|
self._initialize()
|
|
200
242
|
|
|
@@ -241,8 +283,8 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
241
283
|
self._data_channel = None
|
|
242
284
|
self._renegotiation_in_progress = False
|
|
243
285
|
self._last_received_time = None
|
|
244
|
-
self._message_queue = []
|
|
245
286
|
self._pending_app_messages = []
|
|
287
|
+
self._connecting_timeout_task = None
|
|
246
288
|
|
|
247
289
|
def _setup_listeners(self):
|
|
248
290
|
"""Set up event listeners for the peer connection."""
|
|
@@ -254,10 +296,7 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
254
296
|
# Flush queued messages once the data channel is open
|
|
255
297
|
@channel.on("open")
|
|
256
298
|
async def on_open():
|
|
257
|
-
logger.debug("Data channel is open
|
|
258
|
-
while self._message_queue:
|
|
259
|
-
message = self._message_queue.pop(0)
|
|
260
|
-
self._data_channel.send(message)
|
|
299
|
+
logger.debug("Data channel is open!")
|
|
261
300
|
|
|
262
301
|
@channel.on("message")
|
|
263
302
|
async def on_message(message):
|
|
@@ -454,11 +493,15 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
454
493
|
|
|
455
494
|
async def _close(self):
|
|
456
495
|
"""Close the peer connection and cleanup resources."""
|
|
496
|
+
for track in self._track_map.values():
|
|
497
|
+
if track:
|
|
498
|
+
track.stop()
|
|
499
|
+
self._track_map.clear()
|
|
457
500
|
if self._pc:
|
|
458
501
|
await self._pc.close()
|
|
459
|
-
self._message_queue.clear()
|
|
460
502
|
self._pending_app_messages.clear()
|
|
461
503
|
self._track_map = {}
|
|
504
|
+
self._cancel_monitoring_connecting_state()
|
|
462
505
|
|
|
463
506
|
def get_answer(self):
|
|
464
507
|
"""Get the SDP answer for the current connection.
|
|
@@ -476,9 +519,45 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
476
519
|
"pc_id": self._pc_id,
|
|
477
520
|
}
|
|
478
521
|
|
|
522
|
+
def _monitoring_connecting_state(self) -> None:
|
|
523
|
+
"""Start monitoring the peer connection while it is in the *connecting* state.
|
|
524
|
+
|
|
525
|
+
This method schedules a timeout task that will automatically close the
|
|
526
|
+
connection if it remains in the connecting state for more than the specified
|
|
527
|
+
timeout, default to 60 seconds.
|
|
528
|
+
"""
|
|
529
|
+
logger.debug("Monitoring connecting state")
|
|
530
|
+
|
|
531
|
+
async def timeout_handler():
|
|
532
|
+
# We will close the connection in case we have remained in the connecting state for over 1 minute
|
|
533
|
+
await asyncio.sleep(self.connection_timeout_secs)
|
|
534
|
+
logger.warning("Timeout establishing the connection to the remote peer. Closing.")
|
|
535
|
+
|
|
536
|
+
await self._close()
|
|
537
|
+
|
|
538
|
+
# Create and store the timeout task
|
|
539
|
+
self._connecting_timeout_task = asyncio.create_task(timeout_handler())
|
|
540
|
+
|
|
541
|
+
def _cancel_monitoring_connecting_state(self) -> None:
|
|
542
|
+
"""Cancel the ongoing connecting-state timeout task, if any.
|
|
543
|
+
|
|
544
|
+
This method should be called once the connection has either succeeded or
|
|
545
|
+
transitioned out of the connecting state. If the timeout task is still
|
|
546
|
+
pending, it will be canceled and the reference cleared.
|
|
547
|
+
"""
|
|
548
|
+
if self._connecting_timeout_task and not self._connecting_timeout_task.done():
|
|
549
|
+
logger.debug("Cancelling the connecting timeout task")
|
|
550
|
+
self._connecting_timeout_task.cancel()
|
|
551
|
+
self._connecting_timeout_task = None
|
|
552
|
+
|
|
479
553
|
async def _handle_new_connection_state(self):
|
|
480
554
|
"""Handle changes in the peer connection state."""
|
|
481
555
|
state = self._pc.connectionState
|
|
556
|
+
if state == "connecting":
|
|
557
|
+
self._monitoring_connecting_state()
|
|
558
|
+
else:
|
|
559
|
+
self._cancel_monitoring_connecting_state()
|
|
560
|
+
|
|
482
561
|
if state == "connected" and not self._connect_invoked:
|
|
483
562
|
# We are going to wait until the pipeline is ready before triggering the event
|
|
484
563
|
return
|
|
@@ -526,8 +605,8 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
526
605
|
logger.warning("No audio transceiver is available")
|
|
527
606
|
return None
|
|
528
607
|
|
|
529
|
-
|
|
530
|
-
audio_track = SmallWebRTCTrack(
|
|
608
|
+
receiver = transceivers[AUDIO_TRANSCEIVER_INDEX].receiver
|
|
609
|
+
audio_track = SmallWebRTCTrack(receiver) if receiver else None
|
|
531
610
|
self._track_map[AUDIO_TRANSCEIVER_INDEX] = audio_track
|
|
532
611
|
return audio_track
|
|
533
612
|
|
|
@@ -548,8 +627,8 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
548
627
|
logger.warning("No video transceiver is available")
|
|
549
628
|
return None
|
|
550
629
|
|
|
551
|
-
|
|
552
|
-
video_track = SmallWebRTCTrack(
|
|
630
|
+
receiver = transceivers[VIDEO_TRANSCEIVER_INDEX].receiver
|
|
631
|
+
video_track = SmallWebRTCTrack(receiver) if receiver else None
|
|
553
632
|
self._track_map[VIDEO_TRANSCEIVER_INDEX] = video_track
|
|
554
633
|
return video_track
|
|
555
634
|
|
|
@@ -570,8 +649,8 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
570
649
|
logger.warning("No screen video transceiver is available")
|
|
571
650
|
return None
|
|
572
651
|
|
|
573
|
-
|
|
574
|
-
video_track = SmallWebRTCTrack(
|
|
652
|
+
receiver = transceivers[SCREEN_VIDEO_TRANSCEIVER_INDEX].receiver
|
|
653
|
+
video_track = SmallWebRTCTrack(receiver) if receiver else None
|
|
575
654
|
self._track_map[SCREEN_VIDEO_TRANSCEIVER_INDEX] = video_track
|
|
576
655
|
return video_track
|
|
577
656
|
|
|
@@ -585,8 +664,8 @@ class SmallWebRTCConnection(BaseObject):
|
|
|
585
664
|
if self._data_channel and self._data_channel.readyState == "open":
|
|
586
665
|
self._data_channel.send(json_message)
|
|
587
666
|
else:
|
|
588
|
-
|
|
589
|
-
|
|
667
|
+
# The client might choose never to create a data channel.
|
|
668
|
+
logger.trace("Data channel not ready, discarding message!")
|
|
590
669
|
|
|
591
670
|
def ask_to_renegotiate(self):
|
|
592
671
|
"""Request renegotiation of the WebRTC connection."""
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2024–2025, Daily
|
|
3
|
+
#
|
|
4
|
+
# SPDX-License-Identifier: BSD 2-Clause License
|
|
5
|
+
#
|
|
6
|
+
|
|
7
|
+
"""SmallWebRTC request handler for managing peer connections.
|
|
8
|
+
|
|
9
|
+
This module provides a client for handling web requests and managing WebRTC connections.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from enum import Enum
|
|
15
|
+
from typing import Any, Awaitable, Callable, Dict, List, Optional
|
|
16
|
+
|
|
17
|
+
from fastapi import HTTPException
|
|
18
|
+
from loguru import logger
|
|
19
|
+
|
|
20
|
+
from pipecat.transports.smallwebrtc.connection import IceServer, SmallWebRTCConnection
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class SmallWebRTCRequest:
|
|
25
|
+
"""Small WebRTC transport session arguments for the runner.
|
|
26
|
+
|
|
27
|
+
Parameters:
|
|
28
|
+
sdp: The SDP string (Session Description Protocol).
|
|
29
|
+
type: The type of the SDP, either "offer" or "answer".
|
|
30
|
+
pc_id: Optional identifier for the peer connection.
|
|
31
|
+
restart_pc: Optional whether to restart the peer connection.
|
|
32
|
+
request_data: Optional custom data sent by the customer.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
sdp: str
|
|
36
|
+
type: str
|
|
37
|
+
pc_id: Optional[str] = None
|
|
38
|
+
restart_pc: Optional[bool] = None
|
|
39
|
+
request_data: Optional[Any] = None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class ConnectionMode(Enum):
|
|
43
|
+
"""Enum defining the connection handling modes."""
|
|
44
|
+
|
|
45
|
+
SINGLE = "single" # Only one active connection allowed
|
|
46
|
+
MULTIPLE = "multiple" # Multiple simultaneous connections allowed
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class SmallWebRTCRequestHandler:
|
|
50
|
+
"""SmallWebRTC request handler for managing peer connections.
|
|
51
|
+
|
|
52
|
+
This class is responsible for:
|
|
53
|
+
- Handling incoming SmallWebRTC requests.
|
|
54
|
+
- Creating and managing WebRTC peer connections.
|
|
55
|
+
- Supporting ESP32-specific SDP munging if enabled.
|
|
56
|
+
- Invoking callbacks for newly initialized connections.
|
|
57
|
+
- Supporting both single and multiple connection modes.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def __init__(
|
|
61
|
+
self,
|
|
62
|
+
ice_servers: Optional[List[IceServer]] = None,
|
|
63
|
+
esp32_mode: bool = False,
|
|
64
|
+
host: Optional[str] = None,
|
|
65
|
+
connection_mode: ConnectionMode = ConnectionMode.MULTIPLE,
|
|
66
|
+
) -> None:
|
|
67
|
+
"""Initialize a SmallWebRTC request handler.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
ice_servers (Optional[List[IceServer]]): List of ICE servers to use for WebRTC
|
|
71
|
+
connections.
|
|
72
|
+
esp32_mode (bool): If True, enables ESP32-specific SDP munging.
|
|
73
|
+
host (Optional[str]): Host address used for SDP munging in ESP32 mode.
|
|
74
|
+
Ignored if `esp32_mode` is False.
|
|
75
|
+
connection_mode (ConnectionMode): Mode of operation for handling connections.
|
|
76
|
+
SINGLE allows only one active connection, MULTIPLE allows several.
|
|
77
|
+
"""
|
|
78
|
+
self._ice_servers = ice_servers
|
|
79
|
+
self._esp32_mode = esp32_mode
|
|
80
|
+
self._host = host
|
|
81
|
+
self._connection_mode = connection_mode
|
|
82
|
+
|
|
83
|
+
# Store connections by pc_id
|
|
84
|
+
self._pcs_map: Dict[str, SmallWebRTCConnection] = {}
|
|
85
|
+
|
|
86
|
+
def _check_single_connection_constraints(self, pc_id: Optional[str]) -> None:
|
|
87
|
+
"""Check if the connection request satisfies single connection mode constraints.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
pc_id: The peer connection ID from the request
|
|
91
|
+
|
|
92
|
+
Raises:
|
|
93
|
+
HTTPException: If constraints are violated in single connection mode
|
|
94
|
+
"""
|
|
95
|
+
if self._connection_mode != ConnectionMode.SINGLE:
|
|
96
|
+
return
|
|
97
|
+
|
|
98
|
+
if not self._pcs_map: # No existing connections
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
# Get the existing connection (should be only one in single mode)
|
|
102
|
+
existing_connection = next(iter(self._pcs_map.values()))
|
|
103
|
+
|
|
104
|
+
if existing_connection.pc_id != pc_id and pc_id:
|
|
105
|
+
logger.warning(
|
|
106
|
+
f"Connection pc_id mismatch: existing={existing_connection.pc_id}, received={pc_id}"
|
|
107
|
+
)
|
|
108
|
+
raise HTTPException(status_code=400, detail="PC ID mismatch with existing connection")
|
|
109
|
+
|
|
110
|
+
if not pc_id:
|
|
111
|
+
logger.warning(
|
|
112
|
+
"Cannot create new connection: existing connection found but no pc_id received"
|
|
113
|
+
)
|
|
114
|
+
raise HTTPException(
|
|
115
|
+
status_code=400,
|
|
116
|
+
detail="Cannot create new connection with existing connection active",
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
def update_ice_servers(self, ice_servers: Optional[List[IceServer]] = None):
|
|
120
|
+
"""Update the list of ICE servers used for WebRTC connections."""
|
|
121
|
+
self._ice_servers = ice_servers
|
|
122
|
+
|
|
123
|
+
async def handle_web_request(
|
|
124
|
+
self,
|
|
125
|
+
request: SmallWebRTCRequest,
|
|
126
|
+
webrtc_connection_callback: Callable[[Any], Awaitable[None]],
|
|
127
|
+
) -> None:
|
|
128
|
+
"""Handle a SmallWebRTC request and resolve the pending answer.
|
|
129
|
+
|
|
130
|
+
This method will:
|
|
131
|
+
- Reuse an existing WebRTC connection if `pc_id` exists.
|
|
132
|
+
- Otherwise, create a new `SmallWebRTCConnection`.
|
|
133
|
+
- Invoke the provided callback with the connection.
|
|
134
|
+
- Manage ESP32-specific munging if enabled.
|
|
135
|
+
- Enforce single/multiple connection mode constraints.
|
|
136
|
+
|
|
137
|
+
Args:
|
|
138
|
+
request (SmallWebRTCRequest): The incoming WebRTC request, containing
|
|
139
|
+
SDP, type, and optionally a `pc_id`.
|
|
140
|
+
webrtc_connection_callback (Callable[[Any], Awaitable[None]]): An
|
|
141
|
+
asynchronous callback function that is invoked with the WebRTC connection.
|
|
142
|
+
|
|
143
|
+
Raises:
|
|
144
|
+
HTTPException: If connection mode constraints are violated
|
|
145
|
+
Exception: Any exception raised during request handling or callback execution
|
|
146
|
+
will be logged and propagated.
|
|
147
|
+
"""
|
|
148
|
+
try:
|
|
149
|
+
pc_id = request.pc_id
|
|
150
|
+
|
|
151
|
+
# Check connection mode constraints first
|
|
152
|
+
self._check_single_connection_constraints(pc_id)
|
|
153
|
+
|
|
154
|
+
# After constraints are satisfied, get the existing connection if any
|
|
155
|
+
existing_connection = self._pcs_map.get(pc_id) if pc_id else None
|
|
156
|
+
|
|
157
|
+
if existing_connection:
|
|
158
|
+
pipecat_connection = existing_connection
|
|
159
|
+
logger.info(f"Reusing existing connection for pc_id: {pc_id}")
|
|
160
|
+
await pipecat_connection.renegotiate(
|
|
161
|
+
sdp=request.sdp,
|
|
162
|
+
type=request.type,
|
|
163
|
+
restart_pc=request.restart_pc or False,
|
|
164
|
+
)
|
|
165
|
+
else:
|
|
166
|
+
pipecat_connection = SmallWebRTCConnection(ice_servers=self._ice_servers)
|
|
167
|
+
await pipecat_connection.initialize(sdp=request.sdp, type=request.type)
|
|
168
|
+
|
|
169
|
+
@pipecat_connection.event_handler("closed")
|
|
170
|
+
async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
|
|
171
|
+
logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
|
|
172
|
+
self._pcs_map.pop(webrtc_connection.pc_id, None)
|
|
173
|
+
|
|
174
|
+
# Invoke callback provided in runner arguments
|
|
175
|
+
try:
|
|
176
|
+
await webrtc_connection_callback(pipecat_connection)
|
|
177
|
+
logger.debug(
|
|
178
|
+
f"webrtc_connection_callback executed successfully for peer: {pipecat_connection.pc_id}"
|
|
179
|
+
)
|
|
180
|
+
except Exception as callback_error:
|
|
181
|
+
logger.error(
|
|
182
|
+
f"webrtc_connection_callback failed for peer {pipecat_connection.pc_id}: {callback_error}"
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
answer = pipecat_connection.get_answer()
|
|
186
|
+
|
|
187
|
+
if self._esp32_mode:
|
|
188
|
+
from pipecat.runner.utils import smallwebrtc_sdp_munging
|
|
189
|
+
|
|
190
|
+
answer["sdp"] = smallwebrtc_sdp_munging(answer["sdp"], self._host)
|
|
191
|
+
|
|
192
|
+
self._pcs_map[answer["pc_id"]] = pipecat_connection
|
|
193
|
+
|
|
194
|
+
return answer
|
|
195
|
+
except Exception as e:
|
|
196
|
+
logger.error(f"Error processing SmallWebRTC request: {e}")
|
|
197
|
+
logger.debug(f"SmallWebRTC request details: {request}")
|
|
198
|
+
raise
|
|
199
|
+
|
|
200
|
+
async def close(self):
|
|
201
|
+
"""Clear the connection map."""
|
|
202
|
+
coros = [pc.disconnect() for pc in self._pcs_map.values()]
|
|
203
|
+
await asyncio.gather(*coros)
|
|
204
|
+
self._pcs_map.clear()
|
|
@@ -26,13 +26,13 @@ from pipecat.frames.frames import (
|
|
|
26
26
|
EndFrame,
|
|
27
27
|
Frame,
|
|
28
28
|
InputAudioRawFrame,
|
|
29
|
-
|
|
29
|
+
InputTransportMessageFrame,
|
|
30
30
|
OutputAudioRawFrame,
|
|
31
31
|
OutputImageRawFrame,
|
|
32
|
+
OutputTransportMessageFrame,
|
|
33
|
+
OutputTransportMessageUrgentFrame,
|
|
32
34
|
SpriteFrame,
|
|
33
35
|
StartFrame,
|
|
34
|
-
TransportMessageFrame,
|
|
35
|
-
TransportMessageUrgentFrame,
|
|
36
36
|
UserImageRawFrame,
|
|
37
37
|
UserImageRequestFrame,
|
|
38
38
|
)
|
|
@@ -66,7 +66,7 @@ class SmallWebRTCCallbacks(BaseModel):
|
|
|
66
66
|
on_client_disconnected: Called when a client disconnects.
|
|
67
67
|
"""
|
|
68
68
|
|
|
69
|
-
on_app_message: Callable[[Any], Awaitable[None]]
|
|
69
|
+
on_app_message: Callable[[Any, str], Awaitable[None]]
|
|
70
70
|
on_client_connected: Callable[[SmallWebRTCConnection], Awaitable[None]]
|
|
71
71
|
on_client_disconnected: Callable[[SmallWebRTCConnection], Awaitable[None]]
|
|
72
72
|
|
|
@@ -254,7 +254,7 @@ class SmallWebRTCClient:
|
|
|
254
254
|
|
|
255
255
|
@self._webrtc_connection.event_handler("app-message")
|
|
256
256
|
async def on_app_message(connection: SmallWebRTCConnection, message: Any):
|
|
257
|
-
await self._handle_app_message(message)
|
|
257
|
+
await self._handle_app_message(message, connection.pc_id)
|
|
258
258
|
|
|
259
259
|
def _convert_frame(self, frame_array: np.ndarray, format_name: str) -> np.ndarray:
|
|
260
260
|
"""Convert a video frame to RGB format based on the input format.
|
|
@@ -309,7 +309,7 @@ class SmallWebRTCClient:
|
|
|
309
309
|
# self._webrtc_connection.ask_to_renegotiate()
|
|
310
310
|
frame = None
|
|
311
311
|
except MediaStreamError:
|
|
312
|
-
logger.warning("Received an unexpected media stream error while reading the
|
|
312
|
+
logger.warning("Received an unexpected media stream error while reading the video.")
|
|
313
313
|
frame = None
|
|
314
314
|
|
|
315
315
|
if frame is None or not isinstance(frame, VideoFrame):
|
|
@@ -321,15 +321,21 @@ class SmallWebRTCClient:
|
|
|
321
321
|
# Convert frame to NumPy array in its native format
|
|
322
322
|
frame_array = frame.to_ndarray(format=format_name)
|
|
323
323
|
frame_rgb = self._convert_frame(frame_array, format_name)
|
|
324
|
+
del frame_array # free intermediate array immediately
|
|
325
|
+
image_bytes = frame_rgb.tobytes()
|
|
326
|
+
del frame_rgb # free RGB array immediately
|
|
324
327
|
|
|
325
328
|
image_frame = UserImageRawFrame(
|
|
326
329
|
user_id=self._webrtc_connection.pc_id,
|
|
327
|
-
image=
|
|
330
|
+
image=image_bytes,
|
|
328
331
|
size=(frame.width, frame.height),
|
|
329
332
|
format="RGB",
|
|
330
333
|
)
|
|
331
334
|
image_frame.transport_source = video_source
|
|
332
335
|
|
|
336
|
+
del frame # free original VideoFrame
|
|
337
|
+
del image_bytes # reference kept in image_frame
|
|
338
|
+
|
|
333
339
|
yield image_frame
|
|
334
340
|
|
|
335
341
|
async def read_audio_frame(self):
|
|
@@ -364,40 +370,62 @@ class SmallWebRTCClient:
|
|
|
364
370
|
resampled_frames = self._pipecat_resampler.resample(frame)
|
|
365
371
|
for resampled_frame in resampled_frames:
|
|
366
372
|
# 16-bit PCM bytes
|
|
367
|
-
|
|
373
|
+
pcm_array = resampled_frame.to_ndarray().astype(np.int16)
|
|
374
|
+
pcm_bytes = pcm_array.tobytes()
|
|
375
|
+
del pcm_array # free NumPy array immediately
|
|
376
|
+
|
|
368
377
|
audio_frame = InputAudioRawFrame(
|
|
369
378
|
audio=pcm_bytes,
|
|
370
379
|
sample_rate=resampled_frame.sample_rate,
|
|
371
380
|
num_channels=self._audio_in_channels,
|
|
372
381
|
)
|
|
382
|
+
del pcm_bytes # reference kept in audio_frame
|
|
383
|
+
|
|
373
384
|
yield audio_frame
|
|
374
385
|
else:
|
|
375
386
|
# 16-bit PCM bytes
|
|
376
|
-
|
|
387
|
+
pcm_array = frame.to_ndarray().astype(np.int16)
|
|
388
|
+
pcm_bytes = pcm_array.tobytes()
|
|
389
|
+
del pcm_array # free NumPy array immediately
|
|
390
|
+
|
|
377
391
|
audio_frame = InputAudioRawFrame(
|
|
378
392
|
audio=pcm_bytes,
|
|
379
393
|
sample_rate=frame.sample_rate,
|
|
380
394
|
num_channels=self._audio_in_channels,
|
|
381
395
|
)
|
|
396
|
+
del pcm_bytes # reference kept in audio_frame
|
|
397
|
+
|
|
382
398
|
yield audio_frame
|
|
383
399
|
|
|
384
|
-
|
|
400
|
+
del frame # free original AudioFrame
|
|
401
|
+
|
|
402
|
+
async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
|
|
385
403
|
"""Write an audio frame to the WebRTC connection.
|
|
386
404
|
|
|
387
405
|
Args:
|
|
388
406
|
frame: The audio frame to transmit.
|
|
407
|
+
|
|
408
|
+
Returns:
|
|
409
|
+
True if the audio frame was written successfully, False otherwise.
|
|
389
410
|
"""
|
|
390
411
|
if self._can_send() and self._audio_output_track:
|
|
391
412
|
await self._audio_output_track.add_audio_bytes(frame.audio)
|
|
413
|
+
return True
|
|
414
|
+
return False
|
|
392
415
|
|
|
393
|
-
async def write_video_frame(self, frame: OutputImageRawFrame):
|
|
416
|
+
async def write_video_frame(self, frame: OutputImageRawFrame) -> bool:
|
|
394
417
|
"""Write a video frame to the WebRTC connection.
|
|
395
418
|
|
|
396
419
|
Args:
|
|
397
420
|
frame: The video frame to transmit.
|
|
421
|
+
|
|
422
|
+
Returns:
|
|
423
|
+
True if the video frame was written successfully, False otherwise.
|
|
398
424
|
"""
|
|
399
425
|
if self._can_send() and self._video_output_track:
|
|
400
426
|
self._video_output_track.add_video_frame(frame)
|
|
427
|
+
return True
|
|
428
|
+
return False
|
|
401
429
|
|
|
402
430
|
async def setup(self, _params: TransportParams, frame):
|
|
403
431
|
"""Set up the client with transport parameters.
|
|
@@ -433,7 +461,9 @@ class SmallWebRTCClient:
|
|
|
433
461
|
await self._webrtc_connection.disconnect()
|
|
434
462
|
await self._handle_peer_disconnected()
|
|
435
463
|
|
|
436
|
-
async def send_message(
|
|
464
|
+
async def send_message(
|
|
465
|
+
self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
|
|
466
|
+
):
|
|
437
467
|
"""Send an application message through the WebRTC connection.
|
|
438
468
|
|
|
439
469
|
Args:
|
|
@@ -478,11 +508,15 @@ class SmallWebRTCClient:
|
|
|
478
508
|
self._screen_video_track = None
|
|
479
509
|
self._audio_output_track = None
|
|
480
510
|
self._video_output_track = None
|
|
481
|
-
await self._callbacks.on_client_disconnected(self._webrtc_connection)
|
|
482
511
|
|
|
483
|
-
|
|
512
|
+
# Trigger `on_client_disconnected` if the client actually disconnects,
|
|
513
|
+
# that is, we are not the ones disconnecting.
|
|
514
|
+
if not self._closing:
|
|
515
|
+
await self._callbacks.on_client_disconnected(self._webrtc_connection)
|
|
516
|
+
|
|
517
|
+
async def _handle_app_message(self, message: Any, sender: str):
|
|
484
518
|
"""Handle incoming application messages."""
|
|
485
|
-
await self._callbacks.on_app_message(message)
|
|
519
|
+
await self._callbacks.on_app_message(message, sender)
|
|
486
520
|
|
|
487
521
|
def _can_send(self):
|
|
488
522
|
"""Check if the connection is ready for sending data."""
|
|
@@ -651,7 +685,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
|
|
|
651
685
|
message: The application message to process.
|
|
652
686
|
"""
|
|
653
687
|
logger.debug(f"Received app message inside SmallWebRTCInputTransport {message}")
|
|
654
|
-
frame =
|
|
688
|
+
frame = InputTransportMessageFrame(message=message)
|
|
655
689
|
await self.push_frame(frame)
|
|
656
690
|
|
|
657
691
|
# Add this method similar to DailyInputTransport.request_participant_image
|
|
@@ -788,7 +822,9 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
|
|
|
788
822
|
await super().cancel(frame)
|
|
789
823
|
await self._client.disconnect()
|
|
790
824
|
|
|
791
|
-
async def send_message(
|
|
825
|
+
async def send_message(
|
|
826
|
+
self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
|
|
827
|
+
):
|
|
792
828
|
"""Send a transport message through the WebRTC connection.
|
|
793
829
|
|
|
794
830
|
Args:
|
|
@@ -796,21 +832,27 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
|
|
|
796
832
|
"""
|
|
797
833
|
await self._client.send_message(frame)
|
|
798
834
|
|
|
799
|
-
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
|
835
|
+
async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
|
|
800
836
|
"""Write an audio frame to the WebRTC connection.
|
|
801
837
|
|
|
802
838
|
Args:
|
|
803
839
|
frame: The output audio frame to transmit.
|
|
840
|
+
|
|
841
|
+
Returns:
|
|
842
|
+
True if the audio frame was written successfully, False otherwise.
|
|
804
843
|
"""
|
|
805
|
-
await self._client.write_audio_frame(frame)
|
|
844
|
+
return await self._client.write_audio_frame(frame)
|
|
806
845
|
|
|
807
|
-
async def write_video_frame(self, frame: OutputImageRawFrame):
|
|
846
|
+
async def write_video_frame(self, frame: OutputImageRawFrame) -> bool:
|
|
808
847
|
"""Write a video frame to the WebRTC connection.
|
|
809
848
|
|
|
810
849
|
Args:
|
|
811
850
|
frame: The output video frame to transmit.
|
|
851
|
+
|
|
852
|
+
Returns:
|
|
853
|
+
True if the video frame was written successfully, False otherwise.
|
|
812
854
|
"""
|
|
813
|
-
await self._client.write_video_frame(frame)
|
|
855
|
+
return await self._client.write_video_frame(frame)
|
|
814
856
|
|
|
815
857
|
|
|
816
858
|
class SmallWebRTCTransport(BaseTransport):
|
|
@@ -897,11 +939,11 @@ class SmallWebRTCTransport(BaseTransport):
|
|
|
897
939
|
if self._output:
|
|
898
940
|
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
|
899
941
|
|
|
900
|
-
async def _on_app_message(self, message: Any):
|
|
942
|
+
async def _on_app_message(self, message: Any, sender: str):
|
|
901
943
|
"""Handle incoming application messages."""
|
|
902
944
|
if self._input:
|
|
903
945
|
await self._input.push_app_message(message)
|
|
904
|
-
await self._call_event_handler("on_app_message", message)
|
|
946
|
+
await self._call_event_handler("on_app_message", message, sender)
|
|
905
947
|
|
|
906
948
|
async def _on_client_connected(self, webrtc_connection):
|
|
907
949
|
"""Handle client connection events."""
|