dv-pipecat-ai 0.0.85.dev7__py3-none-any.whl → 0.0.85.dev698__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of dv-pipecat-ai might be problematic. Click here for more details.

Files changed (156) hide show
  1. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/METADATA +78 -117
  2. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/RECORD +156 -122
  3. pipecat/adapters/base_llm_adapter.py +38 -1
  4. pipecat/adapters/services/anthropic_adapter.py +9 -14
  5. pipecat/adapters/services/aws_nova_sonic_adapter.py +5 -0
  6. pipecat/adapters/services/bedrock_adapter.py +236 -13
  7. pipecat/adapters/services/gemini_adapter.py +12 -8
  8. pipecat/adapters/services/open_ai_adapter.py +19 -7
  9. pipecat/adapters/services/open_ai_realtime_adapter.py +5 -0
  10. pipecat/audio/filters/krisp_viva_filter.py +193 -0
  11. pipecat/audio/filters/noisereduce_filter.py +15 -0
  12. pipecat/audio/turn/base_turn_analyzer.py +9 -1
  13. pipecat/audio/turn/smart_turn/base_smart_turn.py +14 -8
  14. pipecat/audio/turn/smart_turn/data/__init__.py +0 -0
  15. pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx +0 -0
  16. pipecat/audio/turn/smart_turn/http_smart_turn.py +6 -2
  17. pipecat/audio/turn/smart_turn/local_smart_turn.py +1 -1
  18. pipecat/audio/turn/smart_turn/local_smart_turn_v2.py +1 -1
  19. pipecat/audio/turn/smart_turn/local_smart_turn_v3.py +124 -0
  20. pipecat/audio/vad/data/README.md +10 -0
  21. pipecat/audio/vad/vad_analyzer.py +13 -1
  22. pipecat/extensions/voicemail/voicemail_detector.py +5 -5
  23. pipecat/frames/frames.py +120 -87
  24. pipecat/observers/loggers/debug_log_observer.py +3 -3
  25. pipecat/observers/loggers/llm_log_observer.py +7 -3
  26. pipecat/observers/loggers/user_bot_latency_log_observer.py +22 -10
  27. pipecat/pipeline/runner.py +12 -4
  28. pipecat/pipeline/service_switcher.py +64 -36
  29. pipecat/pipeline/task.py +85 -24
  30. pipecat/processors/aggregators/dtmf_aggregator.py +28 -22
  31. pipecat/processors/aggregators/{gated_openai_llm_context.py → gated_llm_context.py} +9 -9
  32. pipecat/processors/aggregators/gated_open_ai_llm_context.py +12 -0
  33. pipecat/processors/aggregators/llm_response.py +6 -7
  34. pipecat/processors/aggregators/llm_response_universal.py +19 -15
  35. pipecat/processors/aggregators/user_response.py +6 -6
  36. pipecat/processors/aggregators/vision_image_frame.py +24 -2
  37. pipecat/processors/audio/audio_buffer_processor.py +43 -8
  38. pipecat/processors/filters/stt_mute_filter.py +2 -0
  39. pipecat/processors/frame_processor.py +103 -17
  40. pipecat/processors/frameworks/langchain.py +8 -2
  41. pipecat/processors/frameworks/rtvi.py +209 -68
  42. pipecat/processors/frameworks/strands_agents.py +170 -0
  43. pipecat/processors/logger.py +2 -2
  44. pipecat/processors/transcript_processor.py +4 -4
  45. pipecat/processors/user_idle_processor.py +3 -6
  46. pipecat/runner/run.py +270 -50
  47. pipecat/runner/types.py +2 -0
  48. pipecat/runner/utils.py +51 -10
  49. pipecat/serializers/exotel.py +5 -5
  50. pipecat/serializers/livekit.py +20 -0
  51. pipecat/serializers/plivo.py +6 -9
  52. pipecat/serializers/protobuf.py +6 -5
  53. pipecat/serializers/telnyx.py +2 -2
  54. pipecat/serializers/twilio.py +43 -23
  55. pipecat/services/ai_service.py +2 -6
  56. pipecat/services/anthropic/llm.py +2 -25
  57. pipecat/services/asyncai/tts.py +2 -3
  58. pipecat/services/aws/__init__.py +1 -0
  59. pipecat/services/aws/llm.py +122 -97
  60. pipecat/services/aws/nova_sonic/__init__.py +0 -0
  61. pipecat/services/aws/nova_sonic/context.py +367 -0
  62. pipecat/services/aws/nova_sonic/frames.py +25 -0
  63. pipecat/services/aws/nova_sonic/llm.py +1155 -0
  64. pipecat/services/aws/stt.py +1 -3
  65. pipecat/services/aws_nova_sonic/__init__.py +19 -1
  66. pipecat/services/aws_nova_sonic/aws.py +11 -1151
  67. pipecat/services/aws_nova_sonic/context.py +13 -355
  68. pipecat/services/aws_nova_sonic/frames.py +13 -17
  69. pipecat/services/azure/realtime/__init__.py +0 -0
  70. pipecat/services/azure/realtime/llm.py +65 -0
  71. pipecat/services/azure/stt.py +15 -0
  72. pipecat/services/cartesia/tts.py +2 -2
  73. pipecat/services/deepgram/__init__.py +1 -0
  74. pipecat/services/deepgram/flux/__init__.py +0 -0
  75. pipecat/services/deepgram/flux/stt.py +636 -0
  76. pipecat/services/elevenlabs/__init__.py +2 -1
  77. pipecat/services/elevenlabs/stt.py +254 -276
  78. pipecat/services/elevenlabs/tts.py +5 -5
  79. pipecat/services/fish/tts.py +2 -2
  80. pipecat/services/gemini_multimodal_live/events.py +38 -524
  81. pipecat/services/gemini_multimodal_live/file_api.py +23 -173
  82. pipecat/services/gemini_multimodal_live/gemini.py +41 -1403
  83. pipecat/services/gladia/stt.py +56 -72
  84. pipecat/services/google/__init__.py +1 -0
  85. pipecat/services/google/gemini_live/__init__.py +3 -0
  86. pipecat/services/google/gemini_live/file_api.py +189 -0
  87. pipecat/services/google/gemini_live/llm.py +1582 -0
  88. pipecat/services/google/gemini_live/llm_vertex.py +184 -0
  89. pipecat/services/google/llm.py +15 -11
  90. pipecat/services/google/llm_openai.py +3 -3
  91. pipecat/services/google/llm_vertex.py +86 -16
  92. pipecat/services/google/tts.py +7 -3
  93. pipecat/services/heygen/api.py +2 -0
  94. pipecat/services/heygen/client.py +8 -4
  95. pipecat/services/heygen/video.py +2 -0
  96. pipecat/services/hume/__init__.py +5 -0
  97. pipecat/services/hume/tts.py +220 -0
  98. pipecat/services/inworld/tts.py +6 -6
  99. pipecat/services/llm_service.py +15 -5
  100. pipecat/services/lmnt/tts.py +2 -2
  101. pipecat/services/mcp_service.py +4 -2
  102. pipecat/services/mem0/memory.py +6 -5
  103. pipecat/services/mistral/llm.py +29 -8
  104. pipecat/services/moondream/vision.py +42 -16
  105. pipecat/services/neuphonic/tts.py +2 -2
  106. pipecat/services/openai/__init__.py +1 -0
  107. pipecat/services/openai/base_llm.py +27 -20
  108. pipecat/services/openai/realtime/__init__.py +0 -0
  109. pipecat/services/openai/realtime/context.py +272 -0
  110. pipecat/services/openai/realtime/events.py +1106 -0
  111. pipecat/services/openai/realtime/frames.py +37 -0
  112. pipecat/services/openai/realtime/llm.py +829 -0
  113. pipecat/services/openai/tts.py +16 -8
  114. pipecat/services/openai_realtime/__init__.py +27 -0
  115. pipecat/services/openai_realtime/azure.py +21 -0
  116. pipecat/services/openai_realtime/context.py +21 -0
  117. pipecat/services/openai_realtime/events.py +21 -0
  118. pipecat/services/openai_realtime/frames.py +21 -0
  119. pipecat/services/openai_realtime_beta/azure.py +16 -0
  120. pipecat/services/openai_realtime_beta/openai.py +17 -5
  121. pipecat/services/playht/tts.py +31 -4
  122. pipecat/services/rime/tts.py +3 -4
  123. pipecat/services/sarvam/tts.py +2 -6
  124. pipecat/services/simli/video.py +2 -2
  125. pipecat/services/speechmatics/stt.py +1 -7
  126. pipecat/services/stt_service.py +34 -0
  127. pipecat/services/tavus/video.py +2 -2
  128. pipecat/services/tts_service.py +9 -9
  129. pipecat/services/vision_service.py +7 -6
  130. pipecat/tests/utils.py +4 -4
  131. pipecat/transcriptions/language.py +41 -1
  132. pipecat/transports/base_input.py +17 -42
  133. pipecat/transports/base_output.py +42 -26
  134. pipecat/transports/daily/transport.py +199 -26
  135. pipecat/transports/heygen/__init__.py +0 -0
  136. pipecat/transports/heygen/transport.py +381 -0
  137. pipecat/transports/livekit/transport.py +228 -63
  138. pipecat/transports/local/audio.py +6 -1
  139. pipecat/transports/local/tk.py +11 -2
  140. pipecat/transports/network/fastapi_websocket.py +1 -1
  141. pipecat/transports/smallwebrtc/connection.py +98 -19
  142. pipecat/transports/smallwebrtc/request_handler.py +204 -0
  143. pipecat/transports/smallwebrtc/transport.py +65 -23
  144. pipecat/transports/tavus/transport.py +23 -12
  145. pipecat/transports/websocket/client.py +41 -5
  146. pipecat/transports/websocket/fastapi.py +21 -11
  147. pipecat/transports/websocket/server.py +14 -7
  148. pipecat/transports/whatsapp/api.py +8 -0
  149. pipecat/transports/whatsapp/client.py +47 -0
  150. pipecat/utils/base_object.py +54 -22
  151. pipecat/utils/string.py +12 -1
  152. pipecat/utils/tracing/service_decorators.py +21 -21
  153. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/WHEEL +0 -0
  154. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/licenses/LICENSE +0 -0
  155. {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/top_level.txt +0 -0
  156. /pipecat/services/{aws_nova_sonic → aws/nova_sonic}/ready.wav +0 -0
pipecat/runner/run.py CHANGED
@@ -67,11 +67,15 @@ To run locally:
67
67
 
68
68
  import argparse
69
69
  import asyncio
70
+ import mimetypes
70
71
  import os
71
72
  import sys
72
73
  from contextlib import asynccontextmanager
73
- from typing import Dict
74
+ from pathlib import Path
75
+ from typing import Optional
74
76
 
77
+ import aiohttp
78
+ from fastapi.responses import FileResponse
75
79
  from loguru import logger
76
80
 
77
81
  from pipecat.runner.types import (
@@ -83,7 +87,7 @@ from pipecat.runner.types import (
83
87
  try:
84
88
  import uvicorn
85
89
  from dotenv import load_dotenv
86
- from fastapi import BackgroundTasks, FastAPI, Request, WebSocket
90
+ from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request, WebSocket
87
91
  from fastapi.middleware.cors import CORSMiddleware
88
92
  from fastapi.responses import HTMLResponse, RedirectResponse
89
93
  except ImportError as e:
@@ -97,6 +101,12 @@ except ImportError as e:
97
101
  load_dotenv(override=True)
98
102
  os.environ["ENV"] = "local"
99
103
 
104
+ TELEPHONY_TRANSPORTS = ["twilio", "telnyx", "plivo", "exotel"]
105
+
106
+ RUNNER_DOWNLOADS_FOLDER: Optional[str] = None
107
+ RUNNER_HOST: str = "localhost"
108
+ RUNNER_PORT: int = 7860
109
+
100
110
 
101
111
  def _get_bot_module():
102
112
  """Get the bot module from the calling script."""
@@ -151,7 +161,12 @@ async def _run_telephony_bot(websocket: WebSocket):
151
161
 
152
162
 
153
163
  def _create_server_app(
154
- transport_type: str, host: str = "localhost", proxy: str = None, esp32_mode: bool = False
164
+ *,
165
+ transport_type: str,
166
+ host: str = "localhost",
167
+ proxy: str,
168
+ esp32_mode: bool = False,
169
+ folder: Optional[str] = None,
155
170
  ):
156
171
  """Create FastAPI app with transport-specific routes."""
157
172
  app = FastAPI()
@@ -166,30 +181,34 @@ def _create_server_app(
166
181
 
167
182
  # Set up transport-specific routes
168
183
  if transport_type == "webrtc":
169
- _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host)
184
+ _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host, folder=folder)
185
+ _setup_whatsapp_routes(app)
170
186
  elif transport_type == "daily":
171
187
  _setup_daily_routes(app)
172
- elif transport_type in ["twilio", "telnyx", "plivo", "exotel"]:
173
- _setup_telephony_routes(app, transport_type, proxy)
188
+ elif transport_type in TELEPHONY_TRANSPORTS:
189
+ _setup_telephony_routes(app, transport_type=transport_type, proxy=proxy)
174
190
  else:
175
191
  logger.warning(f"Unknown transport type: {transport_type}")
176
192
 
177
193
  return app
178
194
 
179
195
 
180
- def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "localhost"):
196
+ def _setup_webrtc_routes(
197
+ app: FastAPI, *, esp32_mode: bool = False, host: str = "localhost", folder: Optional[str] = None
198
+ ):
181
199
  """Set up WebRTC-specific routes."""
182
200
  try:
183
201
  from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
184
202
 
185
203
  from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
204
+ from pipecat.transports.smallwebrtc.request_handler import (
205
+ SmallWebRTCRequest,
206
+ SmallWebRTCRequestHandler,
207
+ )
186
208
  except ImportError as e:
187
209
  logger.error(f"WebRTC transport dependencies not installed: {e}")
188
210
  return
189
211
 
190
- # Store connections by pc_id
191
- pcs_map: Dict[str, SmallWebRTCConnection] = {}
192
-
193
212
  # Mount the frontend
194
213
  app.mount("/client", SmallWebRTCPrebuiltUI)
195
214
 
@@ -198,53 +217,222 @@ def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "lo
198
217
  """Redirect root requests to client interface."""
199
218
  return RedirectResponse(url="/client/")
200
219
 
201
- @app.post("/api/offer")
202
- async def offer(request: dict, background_tasks: BackgroundTasks):
203
- """Handle WebRTC offer requests and manage peer connections."""
204
- pc_id = request.get("pc_id")
205
-
206
- if pc_id and pc_id in pcs_map:
207
- pipecat_connection = pcs_map[pc_id]
208
- logger.info(f"Reusing existing connection for pc_id: {pc_id}")
209
- await pipecat_connection.renegotiate(
210
- sdp=request["sdp"],
211
- type=request["type"],
212
- restart_pc=request.get("restart_pc", False),
213
- )
214
- else:
215
- pipecat_connection = SmallWebRTCConnection()
216
- await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
220
+ @app.get("/files/{filename}")
221
+ async def download_file(filename: str):
222
+ """Handle file downloads."""
223
+ if not folder:
224
+ logger.warning(f"Attempting to dowload {filename}, but downloads folder not setup.")
225
+ return
217
226
 
218
- @pipecat_connection.event_handler("closed")
219
- async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
220
- """Handle WebRTC connection closure and cleanup."""
221
- logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
222
- pcs_map.pop(webrtc_connection.pc_id, None)
227
+ file_path = Path(folder) / filename
228
+ if not os.path.exists(file_path):
229
+ raise HTTPException(404)
223
230
 
224
- bot_module = _get_bot_module()
225
- runner_args = SmallWebRTCRunnerArguments(webrtc_connection=pipecat_connection)
226
- background_tasks.add_task(bot_module.bot, runner_args)
231
+ media_type, _ = mimetypes.guess_type(file_path)
227
232
 
228
- answer = pipecat_connection.get_answer()
233
+ return FileResponse(path=file_path, media_type=media_type, filename=filename)
229
234
 
230
- # Apply ESP32 SDP munging if enabled
231
- if esp32_mode and host != "localhost":
232
- from pipecat.runner.utils import smallwebrtc_sdp_munging
235
+ # Initialize the SmallWebRTC request handler
236
+ small_webrtc_handler: SmallWebRTCRequestHandler = SmallWebRTCRequestHandler(
237
+ esp32_mode=esp32_mode, host=host
238
+ )
239
+
240
+ @app.post("/api/offer")
241
+ async def offer(request: SmallWebRTCRequest, background_tasks: BackgroundTasks):
242
+ """Handle WebRTC offer requests via SmallWebRTCRequestHandler."""
233
243
 
234
- answer["sdp"] = smallwebrtc_sdp_munging(answer["sdp"], host)
244
+ # Prepare runner arguments with the callback to run your bot
245
+ async def webrtc_connection_callback(connection):
246
+ bot_module = _get_bot_module()
247
+ runner_args = SmallWebRTCRunnerArguments(webrtc_connection=connection)
248
+ background_tasks.add_task(bot_module.bot, runner_args)
235
249
 
236
- pcs_map[answer["pc_id"]] = pipecat_connection
250
+ # Delegate handling to SmallWebRTCRequestHandler
251
+ answer = await small_webrtc_handler.handle_web_request(
252
+ request=request,
253
+ webrtc_connection_callback=webrtc_connection_callback,
254
+ )
237
255
  return answer
238
256
 
239
257
  @asynccontextmanager
240
- async def lifespan(app: FastAPI):
258
+ async def smallwebrtc_lifespan(app: FastAPI):
241
259
  """Manage FastAPI application lifecycle and cleanup connections."""
242
260
  yield
243
- coros = [pc.disconnect() for pc in pcs_map.values()]
244
- await asyncio.gather(*coros)
245
- pcs_map.clear()
261
+ await small_webrtc_handler.close()
262
+
263
+ # Add the SmallWebRTC lifespan to the app
264
+ _add_lifespan_to_app(app, smallwebrtc_lifespan)
265
+
266
+
267
+ def _add_lifespan_to_app(app: FastAPI, new_lifespan):
268
+ """Add a new lifespan context manager to the app, combining with existing if present.
269
+
270
+ Args:
271
+ app: The FastAPI application instance
272
+ new_lifespan: The new lifespan context manager to add
273
+ """
274
+ if hasattr(app.router, "lifespan_context") and app.router.lifespan_context is not None:
275
+ # If there's already a lifespan context, combine them
276
+ existing_lifespan = app.router.lifespan_context
277
+
278
+ @asynccontextmanager
279
+ async def combined_lifespan(app: FastAPI):
280
+ async with existing_lifespan(app):
281
+ async with new_lifespan(app):
282
+ yield
283
+
284
+ app.router.lifespan_context = combined_lifespan
285
+ else:
286
+ # No existing lifespan, use the new one
287
+ app.router.lifespan_context = new_lifespan
288
+
289
+
290
+ def _setup_whatsapp_routes(app: FastAPI):
291
+ """Set up WebRTC-specific routes."""
292
+ try:
293
+ from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
246
294
 
247
- app.router.lifespan_context = lifespan
295
+ from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
296
+ from pipecat.transports.smallwebrtc.request_handler import (
297
+ SmallWebRTCRequest,
298
+ SmallWebRTCRequestHandler,
299
+ )
300
+ from pipecat.transports.whatsapp.api import WhatsAppWebhookRequest
301
+ from pipecat.transports.whatsapp.client import WhatsAppClient
302
+ except ImportError as e:
303
+ logger.error(f"WebRTC transport dependencies not installed: {e}")
304
+ return
305
+
306
+ WHATSAPP_TOKEN = os.getenv("WHATSAPP_TOKEN")
307
+ WHATSAPP_PHONE_NUMBER_ID = os.getenv("WHATSAPP_PHONE_NUMBER_ID")
308
+ WHATSAPP_WEBHOOK_VERIFICATION_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFICATION_TOKEN")
309
+ WHATSAPP_APP_SECRET = os.getenv("WHATSAPP_APP_SECRET")
310
+
311
+ if not all(
312
+ [
313
+ WHATSAPP_TOKEN,
314
+ WHATSAPP_PHONE_NUMBER_ID,
315
+ WHATSAPP_WEBHOOK_VERIFICATION_TOKEN,
316
+ ]
317
+ ):
318
+ logger.debug(
319
+ "Missing required environment variables for WhatsApp transport. Keeping it disabled."
320
+ )
321
+ return
322
+
323
+ # Global WhatsApp client instance
324
+ whatsapp_client: Optional[WhatsAppClient] = None
325
+
326
+ @app.get(
327
+ "/whatsapp",
328
+ summary="Verify WhatsApp webhook",
329
+ description="Handles WhatsApp webhook verification requests from Meta",
330
+ )
331
+ async def verify_webhook(request: Request):
332
+ """Verify WhatsApp webhook endpoint.
333
+
334
+ This endpoint is called by Meta's WhatsApp Business API to verify
335
+ the webhook URL during setup. It validates the verification token
336
+ and returns the challenge parameter if successful.
337
+ """
338
+ if whatsapp_client is None:
339
+ logger.error("WhatsApp client is not initialized")
340
+ raise HTTPException(status_code=503, detail="Service unavailable")
341
+
342
+ params = dict(request.query_params)
343
+ logger.debug(f"Webhook verification request received with params: {list(params.keys())}")
344
+
345
+ try:
346
+ result = await whatsapp_client.handle_verify_webhook_request(
347
+ params=params, expected_verification_token=WHATSAPP_WEBHOOK_VERIFICATION_TOKEN
348
+ )
349
+ logger.info("Webhook verification successful")
350
+ return result
351
+ except ValueError as e:
352
+ logger.warning(f"Webhook verification failed: {e}")
353
+ raise HTTPException(status_code=403, detail="Verification failed")
354
+
355
+ @app.post(
356
+ "/whatsapp",
357
+ summary="Handle WhatsApp webhook events",
358
+ description="Processes incoming WhatsApp messages and call events",
359
+ )
360
+ async def whatsapp_webhook(
361
+ body: WhatsAppWebhookRequest,
362
+ background_tasks: BackgroundTasks,
363
+ request: Request,
364
+ x_hub_signature_256: str = Header(None),
365
+ ):
366
+ """Handle incoming WhatsApp webhook events.
367
+
368
+ For call events, establishes WebRTC connections and spawns bot instances
369
+ in the background to handle real-time communication.
370
+ """
371
+ if whatsapp_client is None:
372
+ logger.error("WhatsApp client is not initialized")
373
+ raise HTTPException(status_code=503, detail="Service unavailable")
374
+
375
+ # Validate webhook object type
376
+ if body.object != "whatsapp_business_account":
377
+ logger.warning(f"Invalid webhook object type: {body.object}")
378
+ raise HTTPException(status_code=400, detail="Invalid object type")
379
+
380
+ logger.debug(f"Processing WhatsApp webhook: {body.model_dump()}")
381
+
382
+ async def connection_callback(connection: SmallWebRTCConnection):
383
+ """Handle new WebRTC connections from WhatsApp calls.
384
+
385
+ Called when a WebRTC connection is established for a WhatsApp call.
386
+ Spawns a bot instance to handle the conversation.
387
+
388
+ Args:
389
+ connection: The established WebRTC connection
390
+ """
391
+ bot_module = _get_bot_module()
392
+ runner_args = SmallWebRTCRunnerArguments(webrtc_connection=connection)
393
+ background_tasks.add_task(bot_module.bot, runner_args)
394
+
395
+ try:
396
+ # Process the webhook request
397
+ raw_body = await request.body()
398
+ result = await whatsapp_client.handle_webhook_request(
399
+ body, connection_callback, sha256_signature=x_hub_signature_256, raw_body=raw_body
400
+ )
401
+ logger.debug(f"Webhook processed successfully: {result}")
402
+ return {"status": "success", "message": "Webhook processed successfully"}
403
+ except ValueError as ve:
404
+ logger.warning(f"Invalid webhook request format: {ve}")
405
+ raise HTTPException(status_code=400, detail=f"Invalid request: {str(ve)}")
406
+ except Exception as e:
407
+ logger.error(f"Internal error processing webhook: {e}")
408
+ raise HTTPException(status_code=500, detail="Internal server error processing webhook")
409
+
410
+ @asynccontextmanager
411
+ async def whatsapp_lifespan(app: FastAPI):
412
+ """Manage WhatsApp client lifecycle and cleanup connections."""
413
+ nonlocal whatsapp_client
414
+
415
+ # Initialize WhatsApp client with persistent HTTP session
416
+ async with aiohttp.ClientSession() as session:
417
+ whatsapp_client = WhatsAppClient(
418
+ whatsapp_token=WHATSAPP_TOKEN,
419
+ whatsapp_secret=WHATSAPP_APP_SECRET,
420
+ phone_number_id=WHATSAPP_PHONE_NUMBER_ID,
421
+ session=session,
422
+ )
423
+ logger.info("WhatsApp client initialized successfully")
424
+
425
+ try:
426
+ yield # Run the application
427
+ finally:
428
+ # Cleanup all active calls on shutdown
429
+ logger.info("Cleaning up WhatsApp client resources...")
430
+ if whatsapp_client:
431
+ await whatsapp_client.terminate_all_calls()
432
+ logger.info("WhatsApp cleanup completed")
433
+
434
+ # Add the WhatsApp lifespan to the app
435
+ _add_lifespan_to_app(app, whatsapp_lifespan)
248
436
 
249
437
 
250
438
  def _setup_daily_routes(app: FastAPI):
@@ -332,7 +520,7 @@ def _setup_daily_routes(app: FastAPI):
332
520
  return await _handle_rtvi_request(request)
333
521
 
334
522
 
335
- def _setup_telephony_routes(app: FastAPI, transport_type: str, proxy: str):
523
+ def _setup_telephony_routes(app: FastAPI, *, transport_type: str, proxy: str):
336
524
  """Set up telephony-specific routes."""
337
525
  # XML response templates (Exotel doesn't use XML webhooks)
338
526
  XML_TEMPLATES = {
@@ -435,6 +623,21 @@ def _validate_and_clean_proxy(proxy: str) -> str:
435
623
  return proxy
436
624
 
437
625
 
626
+ def runner_downloads_folder() -> Optional[str]:
627
+ """Returns the folder where files are stored for later download."""
628
+ return RUNNER_DOWNLOADS_FOLDER
629
+
630
+
631
+ def runner_host() -> str:
632
+ """Returns the host name of this runner."""
633
+ return RUNNER_HOST
634
+
635
+
636
+ def runner_port() -> int:
637
+ """Returns the port of this runner."""
638
+ return RUNNER_PORT
639
+
640
+
438
641
  def main():
439
642
  """Start the Pipecat development runner.
440
643
 
@@ -455,14 +658,16 @@ def main():
455
658
 
456
659
  The bot file must contain a `bot(runner_args)` function as the entry point.
457
660
  """
661
+ global RUNNER_DOWNLOADS_FOLDER, RUNNER_HOST, RUNNER_PORT
662
+
458
663
  parser = argparse.ArgumentParser(description="Pipecat Development Runner")
459
- parser.add_argument("--host", type=str, default="localhost", help="Host address")
460
- parser.add_argument("--port", type=int, default=7860, help="Port number")
664
+ parser.add_argument("--host", type=str, default=RUNNER_HOST, help="Host address")
665
+ parser.add_argument("--port", type=int, default=RUNNER_PORT, help="Port number")
461
666
  parser.add_argument(
462
667
  "-t",
463
668
  "--transport",
464
669
  type=str,
465
- choices=["daily", "webrtc", "twilio", "telnyx", "plivo", "exotel"],
670
+ choices=["daily", "webrtc", *TELEPHONY_TRANSPORTS],
466
671
  default="webrtc",
467
672
  help="Transport type",
468
673
  )
@@ -480,6 +685,7 @@ def main():
480
685
  default=False,
481
686
  help="Connect directly to Daily room (automatically sets transport to daily)",
482
687
  )
688
+ parser.add_argument("-f", "--folder", type=str, help="Path to downloads folder")
483
689
  parser.add_argument(
484
690
  "--verbose", "-v", action="count", default=0, help="Increase logging verbosity"
485
691
  )
@@ -502,6 +708,10 @@ def main():
502
708
  logger.error("For ESP32, you need to specify `--host IP` so we can do SDP munging.")
503
709
  return
504
710
 
711
+ if args.transport in TELEPHONY_TRANSPORTS and not args.proxy:
712
+ logger.error(f"For telephony transports, you need to specify `--proxy PROXY`.")
713
+ return
714
+
505
715
  # Log level
506
716
  logger.remove()
507
717
  logger.add(sys.stderr, level="TRACE" if args.verbose else "DEBUG")
@@ -532,8 +742,18 @@ def main():
532
742
  print(f" → Open http://{args.host}:{args.port} in your browser to start a session")
533
743
  print()
534
744
 
745
+ RUNNER_DOWNLOADS_FOLDER = args.folder
746
+ RUNNER_HOST = args.host
747
+ RUNNER_PORT = args.port
748
+
535
749
  # Create the app with transport-specific setup
536
- app = _create_server_app(args.transport, args.host, args.proxy, args.esp32)
750
+ app = _create_server_app(
751
+ transport_type=args.transport,
752
+ host=args.host,
753
+ proxy=args.proxy,
754
+ esp32_mode=args.esp32,
755
+ folder=args.folder,
756
+ )
537
757
 
538
758
  # Run the server
539
759
  uvicorn.run(app, host=args.host, port=args.port)
pipecat/runner/types.py CHANGED
@@ -51,9 +51,11 @@ class WebSocketRunnerArguments(RunnerArguments):
51
51
 
52
52
  Parameters:
53
53
  websocket: WebSocket connection for audio streaming
54
+ body: Additional request data
54
55
  """
55
56
 
56
57
  websocket: WebSocket
58
+ body: Optional[Any] = field(default_factory=dict)
57
59
 
58
60
 
59
61
  @dataclass
pipecat/runner/utils.py CHANGED
@@ -99,16 +99,47 @@ async def parse_telephony_websocket(websocket: WebSocket):
99
99
  tuple: (transport_type: str, call_data: dict)
100
100
 
101
101
  call_data contains provider-specific fields:
102
- - Twilio: {"stream_id": str, "call_id": str}
103
- - Telnyx: {"stream_id": str, "call_control_id": str, "outbound_encoding": str}
104
- - Plivo: {"stream_id": str, "call_id": str}
105
- - Exotel: {"stream_id": str, "call_id": str, "account_sid": str}
102
+
103
+ - Twilio::
104
+
105
+ {
106
+ "stream_id": str,
107
+ "call_id": str,
108
+ "body": dict
109
+ }
110
+
111
+ - Telnyx::
112
+
113
+ {
114
+ "stream_id": str,
115
+ "call_control_id": str,
116
+ "outbound_encoding": str,
117
+ "from": str,
118
+ "to": str,
119
+ }
120
+
121
+ - Plivo::
122
+
123
+ {
124
+ "stream_id": str,
125
+ "call_id": str,
126
+ }
127
+
128
+ - Exotel::
129
+
130
+ {
131
+ "stream_id": str,
132
+ "call_id": str,
133
+ "account_sid": str,
134
+ "from": str,
135
+ "to": str,
136
+ }
106
137
 
107
138
  Example usage::
108
139
 
109
140
  transport_type, call_data = await parse_telephony_websocket(websocket)
110
- if transport_type == "telnyx":
111
- outbound_encoding = call_data["outbound_encoding"]
141
+ if transport_type == "twilio":
142
+ user_id = call_data["body"]["user_id"]
112
143
  """
113
144
  # Read first two messages
114
145
  start_data = websocket.iter_text()
@@ -151,9 +182,12 @@ async def parse_telephony_websocket(websocket: WebSocket):
151
182
  # Extract provider-specific data
152
183
  if transport_type == "twilio":
153
184
  start_data = call_data_raw.get("start", {})
185
+ body_data = start_data.get("customParameters", {})
154
186
  call_data = {
155
187
  "stream_id": start_data.get("streamSid"),
156
188
  "call_id": start_data.get("callSid"),
189
+ # All custom parameters
190
+ "body": body_data,
157
191
  }
158
192
 
159
193
  elif transport_type == "telnyx":
@@ -163,6 +197,8 @@ async def parse_telephony_websocket(websocket: WebSocket):
163
197
  "outbound_encoding": call_data_raw.get("start", {})
164
198
  .get("media_format", {})
165
199
  .get("encoding"),
200
+ "from": call_data_raw.get("start", {}).get("from", ""),
201
+ "to": call_data_raw.get("start", {}).get("to", ""),
166
202
  }
167
203
 
168
204
  elif transport_type == "plivo":
@@ -178,6 +214,8 @@ async def parse_telephony_websocket(websocket: WebSocket):
178
214
  "stream_id": start_data.get("stream_sid"),
179
215
  "call_id": start_data.get("call_sid"),
180
216
  "account_sid": start_data.get("account_sid"),
217
+ "from": start_data.get("from", ""),
218
+ "to": start_data.get("to", ""),
181
219
  }
182
220
 
183
221
  else:
@@ -275,6 +313,7 @@ def _smallwebrtc_sdp_cleanup_ice_candidates(text: str, pattern: str) -> str:
275
313
  Returns:
276
314
  Cleaned SDP text with filtered ICE candidates.
277
315
  """
316
+ logger.debug("Removing unsupported ICE candidates from SDP")
278
317
  result = []
279
318
  lines = text.splitlines()
280
319
  for line in lines:
@@ -283,7 +322,7 @@ def _smallwebrtc_sdp_cleanup_ice_candidates(text: str, pattern: str) -> str:
283
322
  result.append(line)
284
323
  else:
285
324
  result.append(line)
286
- return "\r\n".join(result)
325
+ return "\r\n".join(result) + "\r\n"
287
326
 
288
327
 
289
328
  def _smallwebrtc_sdp_cleanup_fingerprints(text: str) -> str:
@@ -295,15 +334,16 @@ def _smallwebrtc_sdp_cleanup_fingerprints(text: str) -> str:
295
334
  Returns:
296
335
  SDP text with sha-384 and sha-512 fingerprints removed.
297
336
  """
337
+ logger.debug("Removing unsupported fingerprints from SDP")
298
338
  result = []
299
339
  lines = text.splitlines()
300
340
  for line in lines:
301
341
  if not re.search("sha-384", line) and not re.search("sha-512", line):
302
342
  result.append(line)
303
- return "\r\n".join(result)
343
+ return "\r\n".join(result) + "\r\n"
304
344
 
305
345
 
306
- def smallwebrtc_sdp_munging(sdp: str, host: str) -> str:
346
+ def smallwebrtc_sdp_munging(sdp: str, host: Optional[str]) -> str:
307
347
  """Apply SDP modifications for SmallWebRTC compatibility.
308
348
 
309
349
  Args:
@@ -314,7 +354,8 @@ def smallwebrtc_sdp_munging(sdp: str, host: str) -> str:
314
354
  Modified SDP string with fingerprint and ICE candidate cleanup.
315
355
  """
316
356
  sdp = _smallwebrtc_sdp_cleanup_fingerprints(sdp)
317
- sdp = _smallwebrtc_sdp_cleanup_ice_candidates(sdp, host)
357
+ if host:
358
+ sdp = _smallwebrtc_sdp_cleanup_ice_candidates(sdp, host)
318
359
  return sdp
319
360
 
320
361
 
@@ -20,10 +20,10 @@ from pipecat.frames.frames import (
20
20
  Frame,
21
21
  InputAudioRawFrame,
22
22
  InputDTMFFrame,
23
+ InterruptionFrame,
24
+ OutputTransportMessageFrame,
25
+ OutputTransportMessageUrgentFrame,
23
26
  StartFrame,
24
- StartInterruptionFrame,
25
- TransportMessageFrame,
26
- TransportMessageUrgentFrame,
27
27
  )
28
28
  from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
29
29
 
@@ -98,7 +98,7 @@ class ExotelFrameSerializer(FrameSerializer):
98
98
  Returns:
99
99
  Serialized data as string or bytes, or None if the frame isn't handled.
100
100
  """
101
- if isinstance(frame, StartInterruptionFrame):
101
+ if isinstance(frame, InterruptionFrame):
102
102
  answer = {"event": "clear", "streamSid": self._stream_sid}
103
103
  return json.dumps(answer)
104
104
  elif isinstance(frame, AudioRawFrame):
@@ -121,7 +121,7 @@ class ExotelFrameSerializer(FrameSerializer):
121
121
  }
122
122
 
123
123
  return json.dumps(answer)
124
- elif isinstance(frame, (TransportMessageFrame, TransportMessageUrgentFrame)):
124
+ elif isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
125
125
  return json.dumps(frame.message)
126
126
 
127
127
  return None
@@ -25,11 +25,31 @@ except ModuleNotFoundError as e:
25
25
  class LivekitFrameSerializer(FrameSerializer):
26
26
  """Serializer for converting between Pipecat frames and LiveKit audio frames.
27
27
 
28
+ .. deprecated:: 0.0.90
29
+
30
+ This class is deprecated and will be removed in a future version.
31
+ Please use LiveKitTransport instead, which handles audio streaming
32
+ and frame conversion natively.
33
+
28
34
  This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects
29
35
  to LiveKit AudioFrame objects for transmission, and the reverse conversion
30
36
  for received audio data.
31
37
  """
32
38
 
39
+ def __init__(self):
40
+ """Initialize the LiveKit frame serializer."""
41
+ super().__init__()
42
+ import warnings
43
+
44
+ with warnings.catch_warnings():
45
+ warnings.simplefilter("always")
46
+ warnings.warn(
47
+ "LivekitFrameSerializer is deprecated and will be removed in a future version. "
48
+ "Please use LiveKitTransport instead, which handles audio streaming natively.",
49
+ DeprecationWarning,
50
+ stacklevel=2,
51
+ )
52
+
33
53
  @property
34
54
  def type(self) -> FrameSerializerType:
35
55
  """Get the serializer type.