dv-pipecat-ai 0.0.82.dev857__py3-none-any.whl → 0.0.85.dev837__py3-none-any.whl

pipecat/transports/smallwebrtc/transport.py

@@ -26,13 +26,13 @@ from pipecat.frames.frames import (
     EndFrame,
     Frame,
     InputAudioRawFrame,
-    InputTransportMessageUrgentFrame,
+    InputTransportMessageFrame,
     OutputAudioRawFrame,
     OutputImageRawFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
     SpriteFrame,
     StartFrame,
-    TransportMessageFrame,
-    TransportMessageUrgentFrame,
     UserImageRawFrame,
     UserImageRequestFrame,
 )
@@ -66,7 +66,7 @@ class SmallWebRTCCallbacks(BaseModel):
         on_client_disconnected: Called when a client disconnects.
     """
 
-    on_app_message: Callable[[Any], Awaitable[None]]
+    on_app_message: Callable[[Any, str], Awaitable[None]]
     on_client_connected: Callable[[SmallWebRTCConnection], Awaitable[None]]
     on_client_disconnected: Callable[[SmallWebRTCConnection], Awaitable[None]]
 
@@ -254,7 +254,7 @@ class SmallWebRTCClient:
 
         @self._webrtc_connection.event_handler("app-message")
         async def on_app_message(connection: SmallWebRTCConnection, message: Any):
-            await self._handle_app_message(message)
+            await self._handle_app_message(message, connection.pc_id)
 
     def _convert_frame(self, frame_array: np.ndarray, format_name: str) -> np.ndarray:
         """Convert a video frame to RGB format based on the input format.
@@ -309,7 +309,7 @@ class SmallWebRTCClient:
                     # self._webrtc_connection.ask_to_renegotiate()
                 frame = None
             except MediaStreamError:
-                logger.warning("Received an unexpected media stream error while reading the audio.")
+                logger.warning("Received an unexpected media stream error while reading the video.")
                 frame = None
 
             if frame is None or not isinstance(frame, VideoFrame):
@@ -321,15 +321,21 @@ class SmallWebRTCClient:
             # Convert frame to NumPy array in its native format
             frame_array = frame.to_ndarray(format=format_name)
             frame_rgb = self._convert_frame(frame_array, format_name)
+            del frame_array  # free intermediate array immediately
+            image_bytes = frame_rgb.tobytes()
+            del frame_rgb  # free RGB array immediately
 
             image_frame = UserImageRawFrame(
                 user_id=self._webrtc_connection.pc_id,
-                image=frame_rgb.tobytes(),
+                image=image_bytes,
                 size=(frame.width, frame.height),
                 format="RGB",
             )
             image_frame.transport_source = video_source
 
+            del frame  # free original VideoFrame
+            del image_bytes  # reference kept in image_frame
+
             yield image_frame
 
     async def read_audio_frame(self):
@@ -364,40 +370,62 @@ class SmallWebRTCClient:
                 resampled_frames = self._pipecat_resampler.resample(frame)
                 for resampled_frame in resampled_frames:
                     # 16-bit PCM bytes
-                    pcm_bytes = resampled_frame.to_ndarray().astype(np.int16).tobytes()
+                    pcm_array = resampled_frame.to_ndarray().astype(np.int16)
+                    pcm_bytes = pcm_array.tobytes()
+                    del pcm_array  # free NumPy array immediately
+
                     audio_frame = InputAudioRawFrame(
                         audio=pcm_bytes,
                         sample_rate=resampled_frame.sample_rate,
                         num_channels=self._audio_in_channels,
                     )
+                    del pcm_bytes  # reference kept in audio_frame
+
                     yield audio_frame
             else:
                 # 16-bit PCM bytes
-                pcm_bytes = frame.to_ndarray().astype(np.int16).tobytes()
+                pcm_array = frame.to_ndarray().astype(np.int16)
+                pcm_bytes = pcm_array.tobytes()
+                del pcm_array  # free NumPy array immediately
+
                 audio_frame = InputAudioRawFrame(
                     audio=pcm_bytes,
                     sample_rate=frame.sample_rate,
                     num_channels=self._audio_in_channels,
                 )
+                del pcm_bytes  # reference kept in audio_frame
+
                 yield audio_frame
 
-    async def write_audio_frame(self, frame: OutputAudioRawFrame):
+            del frame  # free original AudioFrame
+
+    async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
         """Write an audio frame to the WebRTC connection.
 
         Args:
             frame: The audio frame to transmit.
+
+        Returns:
+            True if the audio frame was written successfully, False otherwise.
         """
         if self._can_send() and self._audio_output_track:
             await self._audio_output_track.add_audio_bytes(frame.audio)
+            return True
+        return False
 
-    async def write_video_frame(self, frame: OutputImageRawFrame):
+    async def write_video_frame(self, frame: OutputImageRawFrame) -> bool:
         """Write a video frame to the WebRTC connection.
 
         Args:
             frame: The video frame to transmit.
+
+        Returns:
+            True if the video frame was written successfully, False otherwise.
         """
         if self._can_send() and self._video_output_track:
             self._video_output_track.add_video_frame(frame)
+            return True
+        return False
 
     async def setup(self, _params: TransportParams, frame):
         """Set up the client with transport parameters.
@@ -433,7 +461,9 @@ class SmallWebRTCClient:
             await self._webrtc_connection.disconnect()
             await self._handle_peer_disconnected()
 
-    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+    async def send_message(
+        self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
+    ):
         """Send an application message through the WebRTC connection.
 
         Args:
@@ -478,11 +508,15 @@ class SmallWebRTCClient:
         self._screen_video_track = None
         self._audio_output_track = None
         self._video_output_track = None
-        await self._callbacks.on_client_disconnected(self._webrtc_connection)
 
-    async def _handle_app_message(self, message: Any):
+        # Trigger `on_client_disconnected` if the client actually disconnects,
+        # that is, we are not the ones disconnecting.
+        if not self._closing:
+            await self._callbacks.on_client_disconnected(self._webrtc_connection)
+
+    async def _handle_app_message(self, message: Any, sender: str):
         """Handle incoming application messages."""
-        await self._callbacks.on_app_message(message)
+        await self._callbacks.on_app_message(message, sender)
 
     def _can_send(self):
         """Check if the connection is ready for sending data."""
@@ -651,7 +685,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
             message: The application message to process.
         """
         logger.debug(f"Received app message inside SmallWebRTCInputTransport  {message}")
-        frame = InputTransportMessageUrgentFrame(message=message)
+        frame = InputTransportMessageFrame(message=message)
         await self.push_frame(frame)
 
     # Add this method similar to DailyInputTransport.request_participant_image
@@ -788,7 +822,9 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
         await super().cancel(frame)
         await self._client.disconnect()
 
-    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+    async def send_message(
+        self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
+    ):
         """Send a transport message through the WebRTC connection.
 
         Args:
@@ -796,21 +832,27 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
         """
         await self._client.send_message(frame)
 
-    async def write_audio_frame(self, frame: OutputAudioRawFrame):
+    async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
         """Write an audio frame to the WebRTC connection.
 
         Args:
             frame: The output audio frame to transmit.
+
+        Returns:
+            True if the audio frame was written successfully, False otherwise.
         """
-        await self._client.write_audio_frame(frame)
+        return await self._client.write_audio_frame(frame)
 
-    async def write_video_frame(self, frame: OutputImageRawFrame):
+    async def write_video_frame(self, frame: OutputImageRawFrame) -> bool:
         """Write a video frame to the WebRTC connection.
 
         Args:
             frame: The output video frame to transmit.
+
+        Returns:
+            True if the video frame was written successfully, False otherwise.
         """
-        await self._client.write_video_frame(frame)
+        return await self._client.write_video_frame(frame)
 
 
 class SmallWebRTCTransport(BaseTransport):
@@ -897,11 +939,11 @@ class SmallWebRTCTransport(BaseTransport):
         if self._output:
             await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
 
-    async def _on_app_message(self, message: Any):
+    async def _on_app_message(self, message: Any, sender: str):
         """Handle incoming application messages."""
         if self._input:
             await self._input.push_app_message(message)
-        await self._call_event_handler("on_app_message", message)
+        await self._call_event_handler("on_app_message", message, sender)
 
     async def _on_client_connected(self, webrtc_connection):
         """Handle client connection events."""

pipecat/transports/tavus/transport.py

@@ -25,11 +25,11 @@ from pipecat.frames.frames import (
     EndFrame,
     Frame,
     InputAudioRawFrame,
+    InterruptionFrame,
     OutputAudioRawFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
     StartFrame,
-    StartInterruptionFrame,
-    TransportMessageFrame,
-    TransportMessageUrgentFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
 from pipecat.transports.base_input import BaseInputTransport
@@ -221,6 +221,7 @@ class TavusTransportClient:
                 ),
                 on_joined=self._on_joined,
                 on_left=self._on_left,
+                on_before_leave=partial(self._on_handle_callback, "on_before_leave"),
                 on_error=partial(self._on_handle_callback, "on_error"),
                 on_app_message=partial(self._on_handle_callback, "on_app_message"),
                 on_call_state_updated=partial(self._on_handle_callback, "on_call_state_updated"),
@@ -344,7 +345,9 @@ class TavusTransportClient:
             participant_id, callback, audio_source, sample_rate, callback_interval_ms
         )
 
-    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+    async def send_message(
+        self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
+    ):
         """Send a message to participants.
 
         Args:
@@ -372,7 +375,7 @@ class TavusTransportClient:
 
     async def send_interrupt_message(self) -> None:
         """Send an interrupt message to the conversation."""
-        transport_frame = TransportMessageUrgentFrame(
+        transport_frame = OutputTransportMessageUrgentFrame(
             message={
                 "message_type": "conversation",
                 "event_type": "conversation.interrupt",
@@ -395,15 +398,18 @@ class TavusTransportClient:
             participant_settings=participant_settings, profile_settings=profile_settings
         )
 
-    async def write_audio_frame(self, frame: OutputAudioRawFrame):
+    async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
         """Write an audio frame to the transport.
 
         Args:
             frame: The audio frame to write.
+
+        Returns:
+            True if the audio frame was written successfully, False otherwise.
         """
         if not self._client:
-            return
-        await self._client.write_audio_frame(frame)
+            return False
+        return await self._client.write_audio_frame(frame)
 
     async def register_audio_destination(self, destination: str):
         """Register an audio destination for output.
@@ -601,7 +607,9 @@ class TavusOutputTransport(BaseOutputTransport):
         await super().cancel(frame)
         await self._client.stop()
 
-    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+    async def send_message(
+        self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
+    ):
         """Send a message to participants.
 
         Args:
@@ -618,22 +626,25 @@ class TavusOutputTransport(BaseOutputTransport):
             direction: The direction of frame flow in the pipeline.
         """
         await super().process_frame(frame, direction)
-        if isinstance(frame, StartInterruptionFrame):
+        if isinstance(frame, InterruptionFrame):
             await self._handle_interruptions()
 
     async def _handle_interruptions(self):
         """Handle interruption events by sending interrupt message."""
         await self._client.send_interrupt_message()
 
-    async def write_audio_frame(self, frame: OutputAudioRawFrame):
+    async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
         """Write an audio frame to the Tavus transport.
 
         Args:
             frame: The audio frame to write.
+
+        Returns:
+            True if the audio frame was written successfully, False otherwise.
         """
         # This is the custom track destination expected by Tavus
         frame.transport_destination = self._transport_destination
-        await self._client.write_audio_frame(frame)
+        return await self._client.write_audio_frame(frame)
 
     async def register_audio_destination(self, destination: str):
         """Register an audio destination.

pipecat/transports/websocket/client.py

@@ -28,9 +28,9 @@ from pipecat.frames.frames import (
     Frame,
     InputAudioRawFrame,
     OutputAudioRawFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
     StartFrame,
-    TransportMessageFrame,
-    TransportMessageUrgentFrame,
 )
 from pipecat.processors.frame_processor import FrameProcessorSetup
 from pipecat.serializers.base_serializer import FrameSerializer
@@ -150,17 +150,39 @@ class WebsocketClientSession:
         await self._websocket.close()
         self._websocket = None
 
-    async def send(self, message: websockets.Data):
+    async def send(self, message: websockets.Data) -> bool:
         """Send a message through the WebSocket connection.
 
         Args:
             message: The message data to send.
         """
+        result = False
         try:
             if self._websocket:
                 await self._websocket.send(message)
+                result = True
         except Exception as e:
             logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
+        finally:
+            return result
+
+    @property
+    def is_connected(self) -> bool:
+        """Check if the WebSocket is currently connected.
+
+        Returns:
+            True if the WebSocket is in connected state.
+        """
+        return self._websocket.state == websockets.State.OPEN if self._websocket else False
+
+    @property
+    def is_closing(self) -> bool:
+        """Check if the WebSocket is currently closing.
+
+        Returns:
+            True if the WebSocket is in the process of closing.
+        """
+        return self._websocket.state == websockets.State.CLOSING if self._websocket else False
 
     async def _client_task_handler(self):
         """Handle incoming messages from the WebSocket connection."""
@@ -363,7 +385,9 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
         await super().cleanup()
         await self._transport.cleanup()
 
-    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+    async def send_message(
+        self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
+    ):
         """Send a transport message through the WebSocket.
 
         Args:
@@ -371,12 +395,18 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
         """
         await self._write_frame(frame)
 
-    async def write_audio_frame(self, frame: OutputAudioRawFrame):
+    async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
         """Write an audio frame to the WebSocket with optional WAV header.
 
         Args:
             frame: The output audio frame to write.
+
+        Returns:
+            True if the audio frame was written successfully, False otherwise.
         """
+        if self._session.is_closing or not self._session.is_connected:
+            return False
+
         frame = OutputAudioRawFrame(
             audio=frame.audio,
             sample_rate=self.sample_rate,
@@ -402,10 +432,16 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
         # Simulate audio playback with a sleep.
         await self._write_audio_sleep()
 
+        return True
+
     async def _write_frame(self, frame: Frame):
         """Write a frame to the WebSocket after serialization."""
+        if self._session.is_closing or not self._session.is_connected:
+            return
+
         if not self._params.serializer:
             return
+
         payload = await self._params.serializer.serialize(frame)
         if payload:
             await self._session.send(payload)

pipecat/transports/websocket/fastapi.py

@@ -26,11 +26,11 @@ from pipecat.frames.frames import (
     EndFrame,
     Frame,
     InputAudioRawFrame,
+    InterruptionFrame,
     OutputAudioRawFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
     StartFrame,
-    StartInterruptionFrame,
-    TransportMessageFrame,
-    TransportMessageUrgentFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
@@ -150,7 +150,6 @@ class FastAPIWebsocketClient:
                     "Closing already disconnected websocket!", call_id=self._conversation_id
                 )
                 self._closing = True
-                await self.trigger_client_disconnected()
 
     async def disconnect(self):
         """Disconnect the WebSocket client."""
@@ -164,8 +163,6 @@ class FastAPIWebsocketClient:
                 await self._websocket.close()
             except Exception as e:
                 logger.error(f"{self} exception while closing the websocket: {e}")
-            finally:
-                await self.trigger_client_disconnected()
 
     async def trigger_client_disconnected(self):
         """Trigger the client disconnected callback."""
@@ -310,7 +307,10 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
         except Exception as e:
             logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
 
-        await self._client.trigger_client_disconnected()
+        # Trigger `on_client_disconnected` if the client actually disconnects,
+        # that is, we are not the ones disconnecting.
+        if not self._client.is_closing:
+            await self._client.trigger_client_disconnected()
 
     async def _monitor_websocket(self):
         """Wait for self._params.session_timeout seconds, if the websocket is still open, trigger timeout event."""
@@ -410,11 +410,13 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
         """
         await super().process_frame(frame, direction)
 
-        if isinstance(frame, StartInterruptionFrame):
+        if isinstance(frame, InterruptionFrame):
             await self._write_frame(frame)
             self._next_send_time = 0
 
-    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+    async def send_message(
+        self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
+    ):
         """Send a transport message frame.
 
         Args:
@@ -422,14 +424,17 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
         """
         await self._write_frame(frame)
 
-    async def write_audio_frame(self, frame: OutputAudioRawFrame):
+    async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
         """Write an audio frame to the WebSocket with timing simulation.
 
         Args:
             frame: The output audio frame to write.
+
+        Returns:
+            True if the audio frame was written successfully, False otherwise.
         """
         if self._client.is_closing or not self._client.is_connected:
-            return
+            return False
 
         frame = OutputAudioRawFrame(
             audio=frame.audio,
@@ -456,8 +461,13 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
         # Simulate audio playback with a sleep.
         await self._write_audio_sleep()
 
+        return True
+
     async def _write_frame(self, frame: Frame):
         """Serialize and send a frame through the WebSocket."""
+        if self._client.is_closing or not self._client.is_connected:
+            return
+
         if not self._params.serializer:
             return
 

pipecat/transports/websocket/server.py

@@ -25,11 +25,11 @@ from pipecat.frames.frames import (
     EndFrame,
     Frame,
     InputAudioRawFrame,
+    InterruptionFrame,
     OutputAudioRawFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
     StartFrame,
-    StartInterruptionFrame,
-    TransportMessageFrame,
-    TransportMessageUrgentFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.serializers.base_serializer import FrameSerializer
@@ -334,11 +334,13 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
         """
         await super().process_frame(frame, direction)
 
-        if isinstance(frame, StartInterruptionFrame):
+        if isinstance(frame, InterruptionFrame):
             await self._write_frame(frame)
             self._next_send_time = 0
 
-    async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
+    async def send_message(
+        self, frame: OutputTransportMessageFrame | OutputTransportMessageUrgentFrame
+    ):
         """Send a transport message frame to the client.
 
         Args:
@@ -346,14 +348,17 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
         """
         await self._write_frame(frame)
 
-    async def write_audio_frame(self, frame: OutputAudioRawFrame):
+    async def write_audio_frame(self, frame: OutputAudioRawFrame) -> bool:
         """Write an audio frame to the WebSocket client with timing control.
 
         Args:
             frame: The output audio frame to write.
+
+        Returns:
+            True if the audio frame was written successfully, False otherwise.
         """
         if not self._websocket:
-            return
+            return False
 
         frame = OutputAudioRawFrame(
             audio=frame.audio,
@@ -380,6 +385,8 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
         # Simulate audio playback with a sleep.
         await self._write_audio_sleep()
 
+        return True
+
     async def _write_frame(self, frame: Frame):
         """Serialize and send a frame to the WebSocket client."""
         if not self._params.serializer:

pipecat/transports/whatsapp/api.py

@@ -241,6 +241,14 @@ class WhatsAppApi:
         self._whatsapp_url = f"{self.BASE_URL}{phone_number_id}/calls"
         self._whatsapp_token = whatsapp_token
 
+    def update_whatsapp_token(self, whatsapp_token: str):
+        """Update the WhatsApp access token for authentication."""
+        self._whatsapp_token = whatsapp_token
+
+    def update_whatsapp_phone_number_id(self, phone_number_id: str):
+        """Update the WhatsApp phone number ID for authentication."""
+        self._phone_number_id = phone_number_id
+
     async def answer_call_to_whatsapp(self, call_id: str, action: str, sdp: str, from_: str):
         """Answer an incoming WhatsApp call.