dv-pipecat-ai 0.0.85.dev7__py3-none-any.whl → 0.0.85.dev699__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dv-pipecat-ai might be problematic. Click here for more details.

Files changed (158) hide show
  1. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev699.dist-info}/METADATA +78 -117
  2. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev699.dist-info}/RECORD +158 -122
  3. pipecat/adapters/base_llm_adapter.py +38 -1
  4. pipecat/adapters/services/anthropic_adapter.py +9 -14
  5. pipecat/adapters/services/aws_nova_sonic_adapter.py +5 -0
  6. pipecat/adapters/services/bedrock_adapter.py +236 -13
  7. pipecat/adapters/services/gemini_adapter.py +12 -8
  8. pipecat/adapters/services/open_ai_adapter.py +19 -7
  9. pipecat/adapters/services/open_ai_realtime_adapter.py +5 -0
  10. pipecat/audio/filters/krisp_viva_filter.py +193 -0
  11. pipecat/audio/filters/noisereduce_filter.py +15 -0
  12. pipecat/audio/turn/base_turn_analyzer.py +9 -1
  13. pipecat/audio/turn/smart_turn/base_smart_turn.py +14 -8
  14. pipecat/audio/turn/smart_turn/data/__init__.py +0 -0
  15. pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx +0 -0
  16. pipecat/audio/turn/smart_turn/http_smart_turn.py +6 -2
  17. pipecat/audio/turn/smart_turn/local_smart_turn.py +1 -1
  18. pipecat/audio/turn/smart_turn/local_smart_turn_v2.py +1 -1
  19. pipecat/audio/turn/smart_turn/local_smart_turn_v3.py +124 -0
  20. pipecat/audio/vad/data/README.md +10 -0
  21. pipecat/audio/vad/vad_analyzer.py +13 -1
  22. pipecat/extensions/voicemail/voicemail_detector.py +5 -5
  23. pipecat/frames/frames.py +120 -87
  24. pipecat/observers/loggers/debug_log_observer.py +3 -3
  25. pipecat/observers/loggers/llm_log_observer.py +7 -3
  26. pipecat/observers/loggers/user_bot_latency_log_observer.py +22 -10
  27. pipecat/pipeline/runner.py +12 -4
  28. pipecat/pipeline/service_switcher.py +64 -36
  29. pipecat/pipeline/task.py +85 -24
  30. pipecat/processors/aggregators/dtmf_aggregator.py +28 -22
  31. pipecat/processors/aggregators/{gated_openai_llm_context.py → gated_llm_context.py} +9 -9
  32. pipecat/processors/aggregators/gated_open_ai_llm_context.py +12 -0
  33. pipecat/processors/aggregators/llm_response.py +6 -7
  34. pipecat/processors/aggregators/llm_response_universal.py +19 -15
  35. pipecat/processors/aggregators/user_response.py +6 -6
  36. pipecat/processors/aggregators/vision_image_frame.py +24 -2
  37. pipecat/processors/audio/audio_buffer_processor.py +43 -8
  38. pipecat/processors/filters/stt_mute_filter.py +2 -0
  39. pipecat/processors/frame_processor.py +103 -17
  40. pipecat/processors/frameworks/langchain.py +8 -2
  41. pipecat/processors/frameworks/rtvi.py +209 -68
  42. pipecat/processors/frameworks/strands_agents.py +170 -0
  43. pipecat/processors/logger.py +2 -2
  44. pipecat/processors/transcript_processor.py +4 -4
  45. pipecat/processors/user_idle_processor.py +3 -6
  46. pipecat/runner/run.py +270 -50
  47. pipecat/runner/types.py +2 -0
  48. pipecat/runner/utils.py +51 -10
  49. pipecat/serializers/exotel.py +5 -5
  50. pipecat/serializers/livekit.py +20 -0
  51. pipecat/serializers/plivo.py +6 -9
  52. pipecat/serializers/protobuf.py +6 -5
  53. pipecat/serializers/telnyx.py +2 -2
  54. pipecat/serializers/twilio.py +43 -23
  55. pipecat/services/ai_service.py +2 -6
  56. pipecat/services/anthropic/llm.py +2 -25
  57. pipecat/services/asyncai/tts.py +2 -3
  58. pipecat/services/aws/__init__.py +1 -0
  59. pipecat/services/aws/llm.py +122 -97
  60. pipecat/services/aws/nova_sonic/__init__.py +0 -0
  61. pipecat/services/aws/nova_sonic/context.py +367 -0
  62. pipecat/services/aws/nova_sonic/frames.py +25 -0
  63. pipecat/services/aws/nova_sonic/llm.py +1155 -0
  64. pipecat/services/aws/stt.py +1 -3
  65. pipecat/services/aws_nova_sonic/__init__.py +19 -1
  66. pipecat/services/aws_nova_sonic/aws.py +11 -1151
  67. pipecat/services/aws_nova_sonic/context.py +13 -355
  68. pipecat/services/aws_nova_sonic/frames.py +13 -17
  69. pipecat/services/azure/realtime/__init__.py +0 -0
  70. pipecat/services/azure/realtime/llm.py +65 -0
  71. pipecat/services/azure/stt.py +15 -0
  72. pipecat/services/cartesia/tts.py +2 -2
  73. pipecat/services/deepgram/__init__.py +1 -0
  74. pipecat/services/deepgram/flux/__init__.py +0 -0
  75. pipecat/services/deepgram/flux/stt.py +636 -0
  76. pipecat/services/elevenlabs/__init__.py +2 -1
  77. pipecat/services/elevenlabs/stt.py +254 -276
  78. pipecat/services/elevenlabs/tts.py +5 -5
  79. pipecat/services/fish/tts.py +2 -2
  80. pipecat/services/gemini_multimodal_live/events.py +38 -524
  81. pipecat/services/gemini_multimodal_live/file_api.py +23 -173
  82. pipecat/services/gemini_multimodal_live/gemini.py +41 -1403
  83. pipecat/services/gladia/stt.py +56 -72
  84. pipecat/services/google/__init__.py +1 -0
  85. pipecat/services/google/gemini_live/__init__.py +3 -0
  86. pipecat/services/google/gemini_live/file_api.py +189 -0
  87. pipecat/services/google/gemini_live/llm.py +1582 -0
  88. pipecat/services/google/gemini_live/llm_vertex.py +184 -0
  89. pipecat/services/google/llm.py +15 -11
  90. pipecat/services/google/llm_openai.py +3 -3
  91. pipecat/services/google/llm_vertex.py +86 -16
  92. pipecat/services/google/tts.py +7 -3
  93. pipecat/services/heygen/api.py +2 -0
  94. pipecat/services/heygen/client.py +8 -4
  95. pipecat/services/heygen/video.py +2 -0
  96. pipecat/services/hume/__init__.py +5 -0
  97. pipecat/services/hume/tts.py +220 -0
  98. pipecat/services/inworld/tts.py +6 -6
  99. pipecat/services/llm_service.py +15 -5
  100. pipecat/services/lmnt/tts.py +2 -2
  101. pipecat/services/mcp_service.py +4 -2
  102. pipecat/services/mem0/memory.py +6 -5
  103. pipecat/services/mistral/llm.py +29 -8
  104. pipecat/services/moondream/vision.py +42 -16
  105. pipecat/services/neuphonic/tts.py +2 -2
  106. pipecat/services/openai/__init__.py +1 -0
  107. pipecat/services/openai/base_llm.py +27 -20
  108. pipecat/services/openai/realtime/__init__.py +0 -0
  109. pipecat/services/openai/realtime/context.py +272 -0
  110. pipecat/services/openai/realtime/events.py +1106 -0
  111. pipecat/services/openai/realtime/frames.py +37 -0
  112. pipecat/services/openai/realtime/llm.py +829 -0
  113. pipecat/services/openai/tts.py +16 -8
  114. pipecat/services/openai_realtime/__init__.py +27 -0
  115. pipecat/services/openai_realtime/azure.py +21 -0
  116. pipecat/services/openai_realtime/context.py +21 -0
  117. pipecat/services/openai_realtime/events.py +21 -0
  118. pipecat/services/openai_realtime/frames.py +21 -0
  119. pipecat/services/openai_realtime_beta/azure.py +16 -0
  120. pipecat/services/openai_realtime_beta/openai.py +17 -5
  121. pipecat/services/playht/tts.py +31 -4
  122. pipecat/services/rime/tts.py +3 -4
  123. pipecat/services/salesforce/__init__.py +9 -0
  124. pipecat/services/salesforce/llm.py +465 -0
  125. pipecat/services/sarvam/tts.py +2 -6
  126. pipecat/services/simli/video.py +2 -2
  127. pipecat/services/speechmatics/stt.py +1 -7
  128. pipecat/services/stt_service.py +34 -0
  129. pipecat/services/tavus/video.py +2 -2
  130. pipecat/services/tts_service.py +9 -9
  131. pipecat/services/vision_service.py +7 -6
  132. pipecat/tests/utils.py +4 -4
  133. pipecat/transcriptions/language.py +41 -1
  134. pipecat/transports/base_input.py +17 -42
  135. pipecat/transports/base_output.py +42 -26
  136. pipecat/transports/daily/transport.py +199 -26
  137. pipecat/transports/heygen/__init__.py +0 -0
  138. pipecat/transports/heygen/transport.py +381 -0
  139. pipecat/transports/livekit/transport.py +228 -63
  140. pipecat/transports/local/audio.py +6 -1
  141. pipecat/transports/local/tk.py +11 -2
  142. pipecat/transports/network/fastapi_websocket.py +1 -1
  143. pipecat/transports/smallwebrtc/connection.py +98 -19
  144. pipecat/transports/smallwebrtc/request_handler.py +204 -0
  145. pipecat/transports/smallwebrtc/transport.py +65 -23
  146. pipecat/transports/tavus/transport.py +23 -12
  147. pipecat/transports/websocket/client.py +41 -5
  148. pipecat/transports/websocket/fastapi.py +21 -11
  149. pipecat/transports/websocket/server.py +14 -7
  150. pipecat/transports/whatsapp/api.py +8 -0
  151. pipecat/transports/whatsapp/client.py +47 -0
  152. pipecat/utils/base_object.py +54 -22
  153. pipecat/utils/string.py +12 -1
  154. pipecat/utils/tracing/service_decorators.py +21 -21
  155. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev699.dist-info}/WHEEL +0 -0
  156. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev699.dist-info}/licenses/LICENSE +0 -0
  157. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev699.dist-info}/top_level.txt +0 -0
  158. /pipecat/services/{aws_nova_sonic → aws/nova_sonic}/ready.wav +0 -0
@@ -95,15 +95,20 @@ class SmallWebRTCTrack:
95
95
  enable/disable control and frame discarding for audio and video streams.
96
96
  """
97
97
 
98
- def __init__(self, track: MediaStreamTrack):
98
+ def __init__(self, receiver):
99
99
  """Initialize the WebRTC track wrapper.
100
100
 
101
101
  Args:
102
- track: The underlying MediaStreamTrack to wrap.
103
- index: The index of the track in the transceiver (0 for mic, 1 for cam, 2 for screen)
102
+ receiver: The RemoteStreamTrack receiver instance.
104
103
  """
105
- self._track = track
104
+ self._receiver = receiver
105
+ # Configuring the receiver for not consuming the track by default to prevent memory grow
106
+ self._receiver._enabled = False
107
+ self._track = receiver.track
106
108
  self._enabled = True
109
+ self._last_recv_time: float = 0.0
110
+ self._idle_task: Optional[asyncio.Task] = None
111
+ self._idle_timeout: float = 2.0 # seconds before discarding old frames
107
112
 
108
113
  def set_enabled(self, enabled: bool) -> None:
109
114
  """Enable or disable the track.
@@ -138,13 +143,44 @@ class SmallWebRTCTrack:
138
143
  async def recv(self) -> Optional[Frame]:
139
144
  """Receive the next frame from the track.
140
145
 
146
+ Enables the internal receiving state and starts idle watcher.
147
+
141
148
  Returns:
142
149
  The next frame, except for video tracks, where it returns the frame only if the track is enabled, otherwise, returns None.
143
150
  """
151
+ self._receiver._enabled = True
152
+ self._last_recv_time = time.time()
153
+
154
+ # start idle watcher if not already running
155
+ if not self._idle_task or self._idle_task.done():
156
+ self._idle_task = asyncio.create_task(self._idle_watcher())
157
+
144
158
  if not self._enabled and self._track.kind == "video":
145
159
  return None
146
160
  return await self._track.recv()
147
161
 
162
+ async def _idle_watcher(self):
163
+ """Disable receiving if idle for more than _idle_timeout and monitor queue size."""
164
+ while self._receiver._enabled:
165
+ await asyncio.sleep(self._idle_timeout)
166
+ idle_duration = time.time() - self._last_recv_time
167
+ if idle_duration >= self._idle_timeout:
168
+ # discard old frames to prevent memory growth
169
+ logger.debug(
170
+ f"Disabling receiver for {self._track.kind} track after {idle_duration:.2f}s idle"
171
+ )
172
+ await self.discard_old_frames()
173
+ self._receiver._enabled = False
174
+
175
+ def stop(self):
176
+ """Stop receiving frames from the track."""
177
+ self._receiver._enabled = False
178
+ if self._idle_task:
179
+ self._idle_task.cancel()
180
+ self._idle_task = None
181
+ if self._track:
182
+ self._track.stop()
183
+
148
184
  def __getattr__(self, name):
149
185
  """Forward attribute access to the underlying track.
150
186
 
@@ -170,11 +206,16 @@ class SmallWebRTCConnection(BaseObject):
170
206
  for real-time audio/video communication.
171
207
  """
172
208
 
173
- def __init__(self, ice_servers: Optional[Union[List[str], List[IceServer]]] = None):
209
+ def __init__(
210
+ self,
211
+ ice_servers: Optional[Union[List[str], List[IceServer]]] = None,
212
+ connection_timeout_secs: int = 60,
213
+ ):
174
214
  """Initialize the WebRTC connection.
175
215
 
176
216
  Args:
177
217
  ice_servers: List of ICE servers as URLs or IceServer objects.
218
+ connection_timeout_secs: Timeout in seconds for connecting to the peer.
178
219
 
179
220
  Raises:
180
221
  TypeError: If ice_servers contains mixed types or unsupported types.
@@ -195,6 +236,7 @@ class SmallWebRTCConnection(BaseObject):
195
236
  VIDEO_TRANSCEIVER_INDEX: self.video_input_track,
196
237
  SCREEN_VIDEO_TRANSCEIVER_INDEX: self.screen_video_input_track,
197
238
  }
239
+ self.connection_timeout_secs = connection_timeout_secs
198
240
 
199
241
  self._initialize()
200
242
 
@@ -241,8 +283,8 @@ class SmallWebRTCConnection(BaseObject):
241
283
  self._data_channel = None
242
284
  self._renegotiation_in_progress = False
243
285
  self._last_received_time = None
244
- self._message_queue = []
245
286
  self._pending_app_messages = []
287
+ self._connecting_timeout_task = None
246
288
 
247
289
  def _setup_listeners(self):
248
290
  """Set up event listeners for the peer connection."""
@@ -254,10 +296,7 @@ class SmallWebRTCConnection(BaseObject):
254
296
  # Flush queued messages once the data channel is open
255
297
  @channel.on("open")
256
298
  async def on_open():
257
- logger.debug("Data channel is open, flushing queued messages")
258
- while self._message_queue:
259
- message = self._message_queue.pop(0)
260
- self._data_channel.send(message)
299
+ logger.debug("Data channel is open!")
261
300
 
262
301
  @channel.on("message")
263
302
  async def on_message(message):
@@ -454,11 +493,15 @@ class SmallWebRTCConnection(BaseObject):
454
493
 
455
494
  async def _close(self):
456
495
  """Close the peer connection and cleanup resources."""
496
+ for track in self._track_map.values():
497
+ if track:
498
+ track.stop()
499
+ self._track_map.clear()
457
500
  if self._pc:
458
501
  await self._pc.close()
459
- self._message_queue.clear()
460
502
  self._pending_app_messages.clear()
461
503
  self._track_map = {}
504
+ self._cancel_monitoring_connecting_state()
462
505
 
463
506
  def get_answer(self):
464
507
  """Get the SDP answer for the current connection.
@@ -476,9 +519,45 @@ class SmallWebRTCConnection(BaseObject):
476
519
  "pc_id": self._pc_id,
477
520
  }
478
521
 
522
+ def _monitoring_connecting_state(self) -> None:
523
+ """Start monitoring the peer connection while it is in the *connecting* state.
524
+
525
+ This method schedules a timeout task that will automatically close the
526
+ connection if it remains in the connecting state for more than the specified
527
+ timeout, default to 60 seconds.
528
+ """
529
+ logger.debug("Monitoring connecting state")
530
+
531
+ async def timeout_handler():
532
+ # We will close the connection in case we have remained in the connecting state for over 1 minute
533
+ await asyncio.sleep(self.connection_timeout_secs)
534
+ logger.warning("Timeout establishing the connection to the remote peer. Closing.")
535
+
536
+ await self._close()
537
+
538
+ # Create and store the timeout task
539
+ self._connecting_timeout_task = asyncio.create_task(timeout_handler())
540
+
541
+ def _cancel_monitoring_connecting_state(self) -> None:
542
+ """Cancel the ongoing connecting-state timeout task, if any.
543
+
544
+ This method should be called once the connection has either succeeded or
545
+ transitioned out of the connecting state. If the timeout task is still
546
+ pending, it will be canceled and the reference cleared.
547
+ """
548
+ if self._connecting_timeout_task and not self._connecting_timeout_task.done():
549
+ logger.debug("Cancelling the connecting timeout task")
550
+ self._connecting_timeout_task.cancel()
551
+ self._connecting_timeout_task = None
552
+
479
553
  async def _handle_new_connection_state(self):
480
554
  """Handle changes in the peer connection state."""
481
555
  state = self._pc.connectionState
556
+ if state == "connecting":
557
+ self._monitoring_connecting_state()
558
+ else:
559
+ self._cancel_monitoring_connecting_state()
560
+
482
561
  if state == "connected" and not self._connect_invoked:
483
562
  # We are going to wait until the pipeline is ready before triggering the event
484
563
  return
@@ -526,8 +605,8 @@ class SmallWebRTCConnection(BaseObject):
526
605
  logger.warning("No audio transceiver is available")
527
606
  return None
528
607
 
529
- track = transceivers[AUDIO_TRANSCEIVER_INDEX].receiver.track
530
- audio_track = SmallWebRTCTrack(track) if track else None
608
+ receiver = transceivers[AUDIO_TRANSCEIVER_INDEX].receiver
609
+ audio_track = SmallWebRTCTrack(receiver) if receiver else None
531
610
  self._track_map[AUDIO_TRANSCEIVER_INDEX] = audio_track
532
611
  return audio_track
533
612
 
@@ -548,8 +627,8 @@ class SmallWebRTCConnection(BaseObject):
548
627
  logger.warning("No video transceiver is available")
549
628
  return None
550
629
 
551
- track = transceivers[VIDEO_TRANSCEIVER_INDEX].receiver.track
552
- video_track = SmallWebRTCTrack(track) if track else None
630
+ receiver = transceivers[VIDEO_TRANSCEIVER_INDEX].receiver
631
+ video_track = SmallWebRTCTrack(receiver) if receiver else None
553
632
  self._track_map[VIDEO_TRANSCEIVER_INDEX] = video_track
554
633
  return video_track
555
634
 
@@ -570,8 +649,8 @@ class SmallWebRTCConnection(BaseObject):
570
649
  logger.warning("No screen video transceiver is available")
571
650
  return None
572
651
 
573
- track = transceivers[SCREEN_VIDEO_TRANSCEIVER_INDEX].receiver.track
574
- video_track = SmallWebRTCTrack(track) if track else None
652
+ receiver = transceivers[SCREEN_VIDEO_TRANSCEIVER_INDEX].receiver
653
+ video_track = SmallWebRTCTrack(receiver) if receiver else None
575
654
  self._track_map[SCREEN_VIDEO_TRANSCEIVER_INDEX] = video_track
576
655
  return video_track
577
656
 
@@ -585,8 +664,8 @@ class SmallWebRTCConnection(BaseObject):
585
664
  if self._data_channel and self._data_channel.readyState == "open":
586
665
  self._data_channel.send(json_message)
587
666
  else:
588
- logger.debug("Data channel not ready, queuing message")
589
- self._message_queue.append(json_message)
667
+ # The client might choose never to create a data channel.
668
+ logger.trace("Data channel not ready, discarding message!")
590
669
 
591
670
  def ask_to_renegotiate(self):
592
671
  """Request renegotiation of the WebRTC connection."""
@@ -0,0 +1,204 @@
1
+ #
2
+ # Copyright (c) 2024–2025, Daily
3
+ #
4
+ # SPDX-License-Identifier: BSD 2-Clause License
5
+ #
6
+
7
+ """SmallWebRTC request handler for managing peer connections.
8
+
9
+ This module provides a client for handling web requests and managing WebRTC connections.
10
+ """
11
+
12
+ import asyncio
13
+ from dataclasses import dataclass
14
+ from enum import Enum
15
+ from typing import Any, Awaitable, Callable, Dict, List, Optional
16
+
17
+ from fastapi import HTTPException
18
+ from loguru import logger
19
+
20
+ from pipecat.transports.smallwebrtc.connection import IceServer, SmallWebRTCConnection
21
+
22
+
23
+ @dataclass
24
+ class SmallWebRTCRequest:
25
+ """Small WebRTC transport session arguments for the runner.
26
+
27
+ Parameters:
28
+ sdp: The SDP string (Session Description Protocol).
29
+ type: The type of the SDP, either "offer" or "answer".
30
+ pc_id: Optional identifier for the peer connection.
31
+ restart_pc: Optional whether to restart the peer connection.
32
+ request_data: Optional custom data sent by the customer.
33
+ """
34
+
35
+ sdp: str
36
+ type: str
37
+ pc_id: Optional[str] = None
38
+ restart_pc: Optional[bool] = None
39
+ request_data: Optional[Any] = None
40
+
41
+
42
+ class ConnectionMode(Enum):
43
+ """Enum defining the connection handling modes."""
44
+
45
+ SINGLE = "single" # Only one active connection allowed
46
+ MULTIPLE = "multiple" # Multiple simultaneous connections allowed
47
+
48
+
49
+ class SmallWebRTCRequestHandler:
50
+ """SmallWebRTC request handler for managing peer connections.
51
+
52
+ This class is responsible for:
53
+ - Handling incoming SmallWebRTC requests.
54
+ - Creating and managing WebRTC peer connections.
55
+ - Supporting ESP32-specific SDP munging if enabled.
56
+ - Invoking callbacks for newly initialized connections.
57
+ - Supporting both single and multiple connection modes.
58
+ """
59
+
60
+ def __init__(
61
+ self,
62
+ ice_servers: Optional[List[IceServer]] = None,
63
+ esp32_mode: bool = False,
64
+ host: Optional[str] = None,
65
+ connection_mode: ConnectionMode = ConnectionMode.MULTIPLE,
66
+ ) -> None:
67
+ """Initialize a SmallWebRTC request handler.
68
+
69
+ Args:
70
+ ice_servers (Optional[List[IceServer]]): List of ICE servers to use for WebRTC
71
+ connections.
72
+ esp32_mode (bool): If True, enables ESP32-specific SDP munging.
73
+ host (Optional[str]): Host address used for SDP munging in ESP32 mode.
74
+ Ignored if `esp32_mode` is False.
75
+ connection_mode (ConnectionMode): Mode of operation for handling connections.
76
+ SINGLE allows only one active connection, MULTIPLE allows several.
77
+ """
78
+ self._ice_servers = ice_servers
79
+ self._esp32_mode = esp32_mode
80
+ self._host = host
81
+ self._connection_mode = connection_mode
82
+
83
+ # Store connections by pc_id
84
+ self._pcs_map: Dict[str, SmallWebRTCConnection] = {}
85
+
86
+ def _check_single_connection_constraints(self, pc_id: Optional[str]) -> None:
87
+ """Check if the connection request satisfies single connection mode constraints.
88
+
89
+ Args:
90
+ pc_id: The peer connection ID from the request
91
+
92
+ Raises:
93
+ HTTPException: If constraints are violated in single connection mode
94
+ """
95
+ if self._connection_mode != ConnectionMode.SINGLE:
96
+ return
97
+
98
+ if not self._pcs_map: # No existing connections
99
+ return
100
+
101
+ # Get the existing connection (should be only one in single mode)
102
+ existing_connection = next(iter(self._pcs_map.values()))
103
+
104
+ if existing_connection.pc_id != pc_id and pc_id:
105
+ logger.warning(
106
+ f"Connection pc_id mismatch: existing={existing_connection.pc_id}, received={pc_id}"
107
+ )
108
+ raise HTTPException(status_code=400, detail="PC ID mismatch with existing connection")
109
+
110
+ if not pc_id:
111
+ logger.warning(
112
+ "Cannot create new connection: existing connection found but no pc_id received"
113
+ )
114
+ raise HTTPException(
115
+ status_code=400,
116
+ detail="Cannot create new connection with existing connection active",
117
+ )
118
+
119
+ def update_ice_servers(self, ice_servers: Optional[List[IceServer]] = None):
120
+ """Update the list of ICE servers used for WebRTC connections."""
121
+ self._ice_servers = ice_servers
122
+
123
+ async def handle_web_request(
124
+ self,
125
+ request: SmallWebRTCRequest,
126
+ webrtc_connection_callback: Callable[[Any], Awaitable[None]],
127
+ ) -> None:
128
+ """Handle a SmallWebRTC request and resolve the pending answer.
129
+
130
+ This method will:
131
+ - Reuse an existing WebRTC connection if `pc_id` exists.
132
+ - Otherwise, create a new `SmallWebRTCConnection`.
133
+ - Invoke the provided callback with the connection.
134
+ - Manage ESP32-specific munging if enabled.
135
+ - Enforce single/multiple connection mode constraints.
136
+
137
+ Args:
138
+ request (SmallWebRTCRequest): The incoming WebRTC request, containing
139
+ SDP, type, and optionally a `pc_id`.
140
+ webrtc_connection_callback (Callable[[Any], Awaitable[None]]): An
141
+ asynchronous callback function that is invoked with the WebRTC connection.
142
+
143
+ Raises:
144
+ HTTPException: If connection mode constraints are violated
145
+ Exception: Any exception raised during request handling or callback execution
146
+ will be logged and propagated.
147
+ """
148
+ try:
149
+ pc_id = request.pc_id
150
+
151
+ # Check connection mode constraints first
152
+ self._check_single_connection_constraints(pc_id)
153
+
154
+ # After constraints are satisfied, get the existing connection if any
155
+ existing_connection = self._pcs_map.get(pc_id) if pc_id else None
156
+
157
+ if existing_connection:
158
+ pipecat_connection = existing_connection
159
+ logger.info(f"Reusing existing connection for pc_id: {pc_id}")
160
+ await pipecat_connection.renegotiate(
161
+ sdp=request.sdp,
162
+ type=request.type,
163
+ restart_pc=request.restart_pc or False,
164
+ )
165
+ else:
166
+ pipecat_connection = SmallWebRTCConnection(ice_servers=self._ice_servers)
167
+ await pipecat_connection.initialize(sdp=request.sdp, type=request.type)
168
+
169
+ @pipecat_connection.event_handler("closed")
170
+ async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
171
+ logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
172
+ self._pcs_map.pop(webrtc_connection.pc_id, None)
173
+
174
+ # Invoke callback provided in runner arguments
175
+ try:
176
+ await webrtc_connection_callback(pipecat_connection)
177
+ logger.debug(
178
+ f"webrtc_connection_callback executed successfully for peer: {pipecat_connection.pc_id}"
179
+ )
180
+ except Exception as callback_error:
181
+ logger.error(
182
+ f"webrtc_connection_callback failed for peer {pipecat_connection.pc_id}: {callback_error}"
183
+ )
184
+
185
+ answer = pipecat_connection.get_answer()
186
+
187
+ if self._esp32_mode:
188
+ from pipecat.runner.utils import smallwebrtc_sdp_munging
189
+
190
+ answer["sdp"] = smallwebrtc_sdp_munging(answer["sdp"], self._host)
191
+
192
+ self._pcs_map[answer["pc_id"]] = pipecat_connection
193
+
194
+ return answer
195
+ except Exception as e:
196
+ logger.error(f"Error processing SmallWebRTC request: {e}")
197
+ logger.debug(f"SmallWebRTC request details: {request}")
198
+ raise
199
+
200
+ async def close(self):
201
+ """Clear the connection map."""
202
+ coros = [pc.disconnect() for pc in self._pcs_map.values()]
203
+ await asyncio.gather(*coros)
204
+ self._pcs_map.clear()
@@ -26,13 +26,13 @@ from pipecat.frames.frames import (
26
26
  EndFrame,
27
27
  Frame,
28
28
  InputAudioRawFrame,
29
- InputTransportMessageUrgentFrame,
29
+ InputTransportMessageFrame,
30
30
  OutputAudioRawFrame,
31
31
  OutputImageRawFrame,
32
+ OutputTransportMessageFrame,
33
+ OutputTransportMessageUrgentFrame,
32
34
  SpriteFrame,
33
35
  StartFrame,
34
- TransportMessageFrame,
35
- TransportMessageUrgentFrame,
36
36
  UserImageRawFrame,
37
37
  UserImageRequestFrame,
38
38
  )
@@ -66,7 +66,7 @@ class SmallWebRTCCallbacks(BaseModel):
66
66
  on_client_disconnected: Called when a client disconnects.
67
67
  """
68
68
 
69
- on_app_message: Callable[[Any], Awaitable[None]]
69
+ on_app_message: Callable[[Any, str], Awaitable[None]]
70
70
  on_client_connected: Callable[[SmallWebRTCConnection], Awaitable[None]]
71
71
  on_client_disconnected: Callable[[SmallWebRTCConnection], Awaitable[None]]
72
72
 
@@ -254,7 +254,7 @@ class SmallWebRTCClient:
254
254
 
255
255
  @self._webrtc_connection.event_handler("app-message")
256
256
  async def on_app_message(connection: SmallWebRTCConnection, message: Any):
257
- await self._handle_app_message(message)
257
+ await self._handle_app_message(message, connection.pc_id)
258
258
 
259
259
  def _convert_frame(self, frame_array: np.ndarray, format_name: str) -> np.ndarray:
260
260
  """Convert a video frame to RGB format based on the input format.
@@ -309,7 +309,7 @@ class SmallWebRTCClient:
309
309
  # self._webrtc_connection.ask_to_renegotiate()
310
310
  frame = None
311
311
  except MediaStreamError:
312
- logger.warning("Received an unexpected media stream error while reading the audio.")
312
+ logger.warning("Received an unexpected media stream error while reading the video.")
313
313
  frame = None
314
314
 
315
315
  if frame is None or not isinstance(frame, VideoFrame):
@@ -321,15 +321,21 @@ class SmallWebRTCClient:
321
321
  # Convert frame to NumPy array in its native format
322
322
  frame_array = frame.to_ndarray(format=format_name)
323
323
  frame_rgb = self._convert_frame(frame_array, format_name)
324
+ del frame_array # free intermediate array immediately
325
+ image_bytes = frame_rgb.tobytes()
326
+ del frame_rgb # free RGB array immediately
324
327
 
325
328
  image_frame = UserImageRawFrame(
326
329
  user_id=self._webrtc_connection.pc_id,
327
- image=frame_rgb.tobytes(),
330
+ image=image_bytes,
328
331
  size=(frame.width, frame.height),
329
332
  format="RGB",
330
333
  )
331
334
  image_frame.transport_source = video_source
332
335
 
336
+ del frame # free original VideoFrame
337
+ del image_bytes # reference kept in image_frame
338
+
333
339
  yield image_frame
334
340
 
335
341
  async def read_audio_frame(self):
@@ -364,40 +370,62 @@ class SmallWebRTCClient:
364
370
  resampled_frames = self._pipecat_resampler.resample(frame)
365
371
  for resampled_frame in resampled_frames:
366
372
  # 16-bit PCM bytes
367
- pcm_bytes = resampled_frame.to_ndarray().astype(np.int16).tobytes()
373
+ pcm_array = resampled_frame.to_ndarray().astype(np.int16)
374
+ pcm_bytes = pcm_array.tobytes()
375
+ del pcm_array # free NumPy array immediately
376
+
368
377
  audio_frame = InputAudioRawFrame(
369
378
  audio=pcm_bytes,
370
379
  sample_rate=resampled_frame.sample_rate,
371
380
  num_channels=self._audio_in_channels,
372
381
  )
382
+ del pcm_bytes # reference kept in audio_frame
383
+
373
384
  yield audio_frame
374
385
  else:
375
386
  # 16-bit PCM bytes
376
- pcm_bytes = frame.to_ndarray().astype(np.int16).tobytes()
387
+ pcm_array = frame.to_ndarray().astype(np.int16)
388
+ pcm_bytes = pcm_array.tobytes()
389
+ del pcm_array # free NumPy array immediately
390
+
377
391
  audio_frame = InputAudioRawFrame(
378
392
  audio=pcm_bytes,
379
393
  sample_rate=frame.sample_rate,
380
394
  num_channels=self._audio_in_channels,
381
395
  )
396
+ del pcm_bytes # reference kept in audio_frame
397
+
382
398
  yield audio_frame
383
399
 
384
- async def write_audio_frame(self, frame: OutputAudioRawFrame):
400
+ del frame # free original AudioFrame
401
+
402
+ async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
385
403
  """Write an audio frame to the WebRTC connection.
386
404
 
387
405
  Args:
388
406
  frame: The audio frame to transmit.
407
+
408
+ Returns:
409
+ True if the audio frame was written successfully, False otherwise.
389
410
  """
390
411
  if self._can_send() and self._audio_output_track:
391
412
  await self._audio_output_track.add_audio_bytes(frame.audio)
413
+ return True
414
+ return False
392
415
 
393
- async def write_video_frame(self, frame: OutputImageRawFrame):
416
+ async def write_video_frame(self, frame: OutputImageRawFrame) -> bool:
394
417
  """Write a video frame to the WebRTC connection.
395
418
 
396
419
  Args:
397
420
  frame: The video frame to transmit.
421
+
422
+ Returns:
423
+ True if the video frame was written successfully, False otherwise.
398
424
  """
399
425
  if self._can_send() and self._video_output_track:
400
426
  self._video_output_track.add_video_frame(frame)
427
+ return True
428
+ return False
401
429
 
402
430
  async def setup(self, _params: TransportParams, frame):
403
431
  """Set up the client with transport parameters.
@@ -433,7 +461,9 @@ class SmallWebRTCClient:
433
461
  await self._webrtc_connection.disconnect()
434
462
  await self._handle_peer_disconnected()
435
463
 
436
- async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
464
+ async def send_message(
465
+ self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
466
+ ):
437
467
  """Send an application message through the WebRTC connection.
438
468
 
439
469
  Args:
@@ -478,11 +508,15 @@ class SmallWebRTCClient:
478
508
  self._screen_video_track = None
479
509
  self._audio_output_track = None
480
510
  self._video_output_track = None
481
- await self._callbacks.on_client_disconnected(self._webrtc_connection)
482
511
 
483
- async def _handle_app_message(self, message: Any):
512
+ # Trigger `on_client_disconnected` if the client actually disconnects,
513
+ # that is, we are not the ones disconnecting.
514
+ if not self._closing:
515
+ await self._callbacks.on_client_disconnected(self._webrtc_connection)
516
+
517
+ async def _handle_app_message(self, message: Any, sender: str):
484
518
  """Handle incoming application messages."""
485
- await self._callbacks.on_app_message(message)
519
+ await self._callbacks.on_app_message(message, sender)
486
520
 
487
521
  def _can_send(self):
488
522
  """Check if the connection is ready for sending data."""
@@ -651,7 +685,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
651
685
  message: The application message to process.
652
686
  """
653
687
  logger.debug(f"Received app message inside SmallWebRTCInputTransport {message}")
654
- frame = InputTransportMessageUrgentFrame(message=message)
688
+ frame = InputTransportMessageFrame(message=message)
655
689
  await self.push_frame(frame)
656
690
 
657
691
  # Add this method similar to DailyInputTransport.request_participant_image
@@ -788,7 +822,9 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
788
822
  await super().cancel(frame)
789
823
  await self._client.disconnect()
790
824
 
791
- async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
825
+ async def send_message(
826
+ self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
827
+ ):
792
828
  """Send a transport message through the WebRTC connection.
793
829
 
794
830
  Args:
@@ -796,21 +832,27 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
796
832
  """
797
833
  await self._client.send_message(frame)
798
834
 
799
- async def write_audio_frame(self, frame: OutputAudioRawFrame):
835
+ async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
800
836
  """Write an audio frame to the WebRTC connection.
801
837
 
802
838
  Args:
803
839
  frame: The output audio frame to transmit.
840
+
841
+ Returns:
842
+ True if the audio frame was written successfully, False otherwise.
804
843
  """
805
- await self._client.write_audio_frame(frame)
844
+ return await self._client.write_audio_frame(frame)
806
845
 
807
- async def write_video_frame(self, frame: OutputImageRawFrame):
846
+ async def write_video_frame(self, frame: OutputImageRawFrame) -> bool:
808
847
  """Write a video frame to the WebRTC connection.
809
848
 
810
849
  Args:
811
850
  frame: The output video frame to transmit.
851
+
852
+ Returns:
853
+ True if the video frame was written successfully, False otherwise.
812
854
  """
813
- await self._client.write_video_frame(frame)
855
+ return await self._client.write_video_frame(frame)
814
856
 
815
857
 
816
858
  class SmallWebRTCTransport(BaseTransport):
@@ -897,11 +939,11 @@ class SmallWebRTCTransport(BaseTransport):
897
939
  if self._output:
898
940
  await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
899
941
 
900
- async def _on_app_message(self, message: Any):
942
+ async def _on_app_message(self, message: Any, sender: str):
901
943
  """Handle incoming application messages."""
902
944
  if self._input:
903
945
  await self._input.push_app_message(message)
904
- await self._call_event_handler("on_app_message", message)
946
+ await self._call_event_handler("on_app_message", message, sender)
905
947
 
906
948
  async def _on_client_connected(self, webrtc_connection):
907
949
  """Handle client connection events."""