dv-pipecat-ai 0.0.85.dev7__py3-none-any.whl → 0.0.85.dev698__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dv-pipecat-ai might be problematic. Click here for more details.
- {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/METADATA +78 -117
- {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/RECORD +156 -122
- pipecat/adapters/base_llm_adapter.py +38 -1
- pipecat/adapters/services/anthropic_adapter.py +9 -14
- pipecat/adapters/services/aws_nova_sonic_adapter.py +5 -0
- pipecat/adapters/services/bedrock_adapter.py +236 -13
- pipecat/adapters/services/gemini_adapter.py +12 -8
- pipecat/adapters/services/open_ai_adapter.py +19 -7
- pipecat/adapters/services/open_ai_realtime_adapter.py +5 -0
- pipecat/audio/filters/krisp_viva_filter.py +193 -0
- pipecat/audio/filters/noisereduce_filter.py +15 -0
- pipecat/audio/turn/base_turn_analyzer.py +9 -1
- pipecat/audio/turn/smart_turn/base_smart_turn.py +14 -8
- pipecat/audio/turn/smart_turn/data/__init__.py +0 -0
- pipecat/audio/turn/smart_turn/data/smart-turn-v3.0.onnx +0 -0
- pipecat/audio/turn/smart_turn/http_smart_turn.py +6 -2
- pipecat/audio/turn/smart_turn/local_smart_turn.py +1 -1
- pipecat/audio/turn/smart_turn/local_smart_turn_v2.py +1 -1
- pipecat/audio/turn/smart_turn/local_smart_turn_v3.py +124 -0
- pipecat/audio/vad/data/README.md +10 -0
- pipecat/audio/vad/vad_analyzer.py +13 -1
- pipecat/extensions/voicemail/voicemail_detector.py +5 -5
- pipecat/frames/frames.py +120 -87
- pipecat/observers/loggers/debug_log_observer.py +3 -3
- pipecat/observers/loggers/llm_log_observer.py +7 -3
- pipecat/observers/loggers/user_bot_latency_log_observer.py +22 -10
- pipecat/pipeline/runner.py +12 -4
- pipecat/pipeline/service_switcher.py +64 -36
- pipecat/pipeline/task.py +85 -24
- pipecat/processors/aggregators/dtmf_aggregator.py +28 -22
- pipecat/processors/aggregators/{gated_openai_llm_context.py → gated_llm_context.py} +9 -9
- pipecat/processors/aggregators/gated_open_ai_llm_context.py +12 -0
- pipecat/processors/aggregators/llm_response.py +6 -7
- pipecat/processors/aggregators/llm_response_universal.py +19 -15
- pipecat/processors/aggregators/user_response.py +6 -6
- pipecat/processors/aggregators/vision_image_frame.py +24 -2
- pipecat/processors/audio/audio_buffer_processor.py +43 -8
- pipecat/processors/filters/stt_mute_filter.py +2 -0
- pipecat/processors/frame_processor.py +103 -17
- pipecat/processors/frameworks/langchain.py +8 -2
- pipecat/processors/frameworks/rtvi.py +209 -68
- pipecat/processors/frameworks/strands_agents.py +170 -0
- pipecat/processors/logger.py +2 -2
- pipecat/processors/transcript_processor.py +4 -4
- pipecat/processors/user_idle_processor.py +3 -6
- pipecat/runner/run.py +270 -50
- pipecat/runner/types.py +2 -0
- pipecat/runner/utils.py +51 -10
- pipecat/serializers/exotel.py +5 -5
- pipecat/serializers/livekit.py +20 -0
- pipecat/serializers/plivo.py +6 -9
- pipecat/serializers/protobuf.py +6 -5
- pipecat/serializers/telnyx.py +2 -2
- pipecat/serializers/twilio.py +43 -23
- pipecat/services/ai_service.py +2 -6
- pipecat/services/anthropic/llm.py +2 -25
- pipecat/services/asyncai/tts.py +2 -3
- pipecat/services/aws/__init__.py +1 -0
- pipecat/services/aws/llm.py +122 -97
- pipecat/services/aws/nova_sonic/__init__.py +0 -0
- pipecat/services/aws/nova_sonic/context.py +367 -0
- pipecat/services/aws/nova_sonic/frames.py +25 -0
- pipecat/services/aws/nova_sonic/llm.py +1155 -0
- pipecat/services/aws/stt.py +1 -3
- pipecat/services/aws_nova_sonic/__init__.py +19 -1
- pipecat/services/aws_nova_sonic/aws.py +11 -1151
- pipecat/services/aws_nova_sonic/context.py +13 -355
- pipecat/services/aws_nova_sonic/frames.py +13 -17
- pipecat/services/azure/realtime/__init__.py +0 -0
- pipecat/services/azure/realtime/llm.py +65 -0
- pipecat/services/azure/stt.py +15 -0
- pipecat/services/cartesia/tts.py +2 -2
- pipecat/services/deepgram/__init__.py +1 -0
- pipecat/services/deepgram/flux/__init__.py +0 -0
- pipecat/services/deepgram/flux/stt.py +636 -0
- pipecat/services/elevenlabs/__init__.py +2 -1
- pipecat/services/elevenlabs/stt.py +254 -276
- pipecat/services/elevenlabs/tts.py +5 -5
- pipecat/services/fish/tts.py +2 -2
- pipecat/services/gemini_multimodal_live/events.py +38 -524
- pipecat/services/gemini_multimodal_live/file_api.py +23 -173
- pipecat/services/gemini_multimodal_live/gemini.py +41 -1403
- pipecat/services/gladia/stt.py +56 -72
- pipecat/services/google/__init__.py +1 -0
- pipecat/services/google/gemini_live/__init__.py +3 -0
- pipecat/services/google/gemini_live/file_api.py +189 -0
- pipecat/services/google/gemini_live/llm.py +1582 -0
- pipecat/services/google/gemini_live/llm_vertex.py +184 -0
- pipecat/services/google/llm.py +15 -11
- pipecat/services/google/llm_openai.py +3 -3
- pipecat/services/google/llm_vertex.py +86 -16
- pipecat/services/google/tts.py +7 -3
- pipecat/services/heygen/api.py +2 -0
- pipecat/services/heygen/client.py +8 -4
- pipecat/services/heygen/video.py +2 -0
- pipecat/services/hume/__init__.py +5 -0
- pipecat/services/hume/tts.py +220 -0
- pipecat/services/inworld/tts.py +6 -6
- pipecat/services/llm_service.py +15 -5
- pipecat/services/lmnt/tts.py +2 -2
- pipecat/services/mcp_service.py +4 -2
- pipecat/services/mem0/memory.py +6 -5
- pipecat/services/mistral/llm.py +29 -8
- pipecat/services/moondream/vision.py +42 -16
- pipecat/services/neuphonic/tts.py +2 -2
- pipecat/services/openai/__init__.py +1 -0
- pipecat/services/openai/base_llm.py +27 -20
- pipecat/services/openai/realtime/__init__.py +0 -0
- pipecat/services/openai/realtime/context.py +272 -0
- pipecat/services/openai/realtime/events.py +1106 -0
- pipecat/services/openai/realtime/frames.py +37 -0
- pipecat/services/openai/realtime/llm.py +829 -0
- pipecat/services/openai/tts.py +16 -8
- pipecat/services/openai_realtime/__init__.py +27 -0
- pipecat/services/openai_realtime/azure.py +21 -0
- pipecat/services/openai_realtime/context.py +21 -0
- pipecat/services/openai_realtime/events.py +21 -0
- pipecat/services/openai_realtime/frames.py +21 -0
- pipecat/services/openai_realtime_beta/azure.py +16 -0
- pipecat/services/openai_realtime_beta/openai.py +17 -5
- pipecat/services/playht/tts.py +31 -4
- pipecat/services/rime/tts.py +3 -4
- pipecat/services/sarvam/tts.py +2 -6
- pipecat/services/simli/video.py +2 -2
- pipecat/services/speechmatics/stt.py +1 -7
- pipecat/services/stt_service.py +34 -0
- pipecat/services/tavus/video.py +2 -2
- pipecat/services/tts_service.py +9 -9
- pipecat/services/vision_service.py +7 -6
- pipecat/tests/utils.py +4 -4
- pipecat/transcriptions/language.py +41 -1
- pipecat/transports/base_input.py +17 -42
- pipecat/transports/base_output.py +42 -26
- pipecat/transports/daily/transport.py +199 -26
- pipecat/transports/heygen/__init__.py +0 -0
- pipecat/transports/heygen/transport.py +381 -0
- pipecat/transports/livekit/transport.py +228 -63
- pipecat/transports/local/audio.py +6 -1
- pipecat/transports/local/tk.py +11 -2
- pipecat/transports/network/fastapi_websocket.py +1 -1
- pipecat/transports/smallwebrtc/connection.py +98 -19
- pipecat/transports/smallwebrtc/request_handler.py +204 -0
- pipecat/transports/smallwebrtc/transport.py +65 -23
- pipecat/transports/tavus/transport.py +23 -12
- pipecat/transports/websocket/client.py +41 -5
- pipecat/transports/websocket/fastapi.py +21 -11
- pipecat/transports/websocket/server.py +14 -7
- pipecat/transports/whatsapp/api.py +8 -0
- pipecat/transports/whatsapp/client.py +47 -0
- pipecat/utils/base_object.py +54 -22
- pipecat/utils/string.py +12 -1
- pipecat/utils/tracing/service_decorators.py +21 -21
- {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/WHEEL +0 -0
- {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/licenses/LICENSE +0 -0
- {dv_pipecat_ai-0.0.85.dev7.dist-info → dv_pipecat_ai-0.0.85.dev698.dist-info}/top_level.txt +0 -0
- /pipecat/services/{aws_nova_sonic → aws/nova_sonic}/ready.wav +0 -0
pipecat/runner/run.py
CHANGED
|
@@ -67,11 +67,15 @@ To run locally:
|
|
|
67
67
|
|
|
68
68
|
import argparse
|
|
69
69
|
import asyncio
|
|
70
|
+
import mimetypes
|
|
70
71
|
import os
|
|
71
72
|
import sys
|
|
72
73
|
from contextlib import asynccontextmanager
|
|
73
|
-
from
|
|
74
|
+
from pathlib import Path
|
|
75
|
+
from typing import Optional
|
|
74
76
|
|
|
77
|
+
import aiohttp
|
|
78
|
+
from fastapi.responses import FileResponse
|
|
75
79
|
from loguru import logger
|
|
76
80
|
|
|
77
81
|
from pipecat.runner.types import (
|
|
@@ -83,7 +87,7 @@ from pipecat.runner.types import (
|
|
|
83
87
|
try:
|
|
84
88
|
import uvicorn
|
|
85
89
|
from dotenv import load_dotenv
|
|
86
|
-
from fastapi import BackgroundTasks, FastAPI, Request, WebSocket
|
|
90
|
+
from fastapi import BackgroundTasks, FastAPI, Header, HTTPException, Request, WebSocket
|
|
87
91
|
from fastapi.middleware.cors import CORSMiddleware
|
|
88
92
|
from fastapi.responses import HTMLResponse, RedirectResponse
|
|
89
93
|
except ImportError as e:
|
|
@@ -97,6 +101,12 @@ except ImportError as e:
|
|
|
97
101
|
load_dotenv(override=True)
|
|
98
102
|
os.environ["ENV"] = "local"
|
|
99
103
|
|
|
104
|
+
TELEPHONY_TRANSPORTS = ["twilio", "telnyx", "plivo", "exotel"]
|
|
105
|
+
|
|
106
|
+
RUNNER_DOWNLOADS_FOLDER: Optional[str] = None
|
|
107
|
+
RUNNER_HOST: str = "localhost"
|
|
108
|
+
RUNNER_PORT: int = 7860
|
|
109
|
+
|
|
100
110
|
|
|
101
111
|
def _get_bot_module():
|
|
102
112
|
"""Get the bot module from the calling script."""
|
|
@@ -151,7 +161,12 @@ async def _run_telephony_bot(websocket: WebSocket):
|
|
|
151
161
|
|
|
152
162
|
|
|
153
163
|
def _create_server_app(
|
|
154
|
-
|
|
164
|
+
*,
|
|
165
|
+
transport_type: str,
|
|
166
|
+
host: str = "localhost",
|
|
167
|
+
proxy: str,
|
|
168
|
+
esp32_mode: bool = False,
|
|
169
|
+
folder: Optional[str] = None,
|
|
155
170
|
):
|
|
156
171
|
"""Create FastAPI app with transport-specific routes."""
|
|
157
172
|
app = FastAPI()
|
|
@@ -166,30 +181,34 @@ def _create_server_app(
|
|
|
166
181
|
|
|
167
182
|
# Set up transport-specific routes
|
|
168
183
|
if transport_type == "webrtc":
|
|
169
|
-
_setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host)
|
|
184
|
+
_setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host, folder=folder)
|
|
185
|
+
_setup_whatsapp_routes(app)
|
|
170
186
|
elif transport_type == "daily":
|
|
171
187
|
_setup_daily_routes(app)
|
|
172
|
-
elif transport_type in
|
|
173
|
-
_setup_telephony_routes(app, transport_type, proxy)
|
|
188
|
+
elif transport_type in TELEPHONY_TRANSPORTS:
|
|
189
|
+
_setup_telephony_routes(app, transport_type=transport_type, proxy=proxy)
|
|
174
190
|
else:
|
|
175
191
|
logger.warning(f"Unknown transport type: {transport_type}")
|
|
176
192
|
|
|
177
193
|
return app
|
|
178
194
|
|
|
179
195
|
|
|
180
|
-
def _setup_webrtc_routes(
|
|
196
|
+
def _setup_webrtc_routes(
|
|
197
|
+
app: FastAPI, *, esp32_mode: bool = False, host: str = "localhost", folder: Optional[str] = None
|
|
198
|
+
):
|
|
181
199
|
"""Set up WebRTC-specific routes."""
|
|
182
200
|
try:
|
|
183
201
|
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
|
|
184
202
|
|
|
185
203
|
from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
|
|
204
|
+
from pipecat.transports.smallwebrtc.request_handler import (
|
|
205
|
+
SmallWebRTCRequest,
|
|
206
|
+
SmallWebRTCRequestHandler,
|
|
207
|
+
)
|
|
186
208
|
except ImportError as e:
|
|
187
209
|
logger.error(f"WebRTC transport dependencies not installed: {e}")
|
|
188
210
|
return
|
|
189
211
|
|
|
190
|
-
# Store connections by pc_id
|
|
191
|
-
pcs_map: Dict[str, SmallWebRTCConnection] = {}
|
|
192
|
-
|
|
193
212
|
# Mount the frontend
|
|
194
213
|
app.mount("/client", SmallWebRTCPrebuiltUI)
|
|
195
214
|
|
|
@@ -198,53 +217,222 @@ def _setup_webrtc_routes(app: FastAPI, esp32_mode: bool = False, host: str = "lo
|
|
|
198
217
|
"""Redirect root requests to client interface."""
|
|
199
218
|
return RedirectResponse(url="/client/")
|
|
200
219
|
|
|
201
|
-
@app.
|
|
202
|
-
async def
|
|
203
|
-
"""Handle
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
pipecat_connection = pcs_map[pc_id]
|
|
208
|
-
logger.info(f"Reusing existing connection for pc_id: {pc_id}")
|
|
209
|
-
await pipecat_connection.renegotiate(
|
|
210
|
-
sdp=request["sdp"],
|
|
211
|
-
type=request["type"],
|
|
212
|
-
restart_pc=request.get("restart_pc", False),
|
|
213
|
-
)
|
|
214
|
-
else:
|
|
215
|
-
pipecat_connection = SmallWebRTCConnection()
|
|
216
|
-
await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
|
|
220
|
+
@app.get("/files/{filename}")
|
|
221
|
+
async def download_file(filename: str):
|
|
222
|
+
"""Handle file downloads."""
|
|
223
|
+
if not folder:
|
|
224
|
+
logger.warning(f"Attempting to dowload {filename}, but downloads folder not setup.")
|
|
225
|
+
return
|
|
217
226
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
|
|
222
|
-
pcs_map.pop(webrtc_connection.pc_id, None)
|
|
227
|
+
file_path = Path(folder) / filename
|
|
228
|
+
if not os.path.exists(file_path):
|
|
229
|
+
raise HTTPException(404)
|
|
223
230
|
|
|
224
|
-
|
|
225
|
-
runner_args = SmallWebRTCRunnerArguments(webrtc_connection=pipecat_connection)
|
|
226
|
-
background_tasks.add_task(bot_module.bot, runner_args)
|
|
231
|
+
media_type, _ = mimetypes.guess_type(file_path)
|
|
227
232
|
|
|
228
|
-
|
|
233
|
+
return FileResponse(path=file_path, media_type=media_type, filename=filename)
|
|
229
234
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
235
|
+
# Initialize the SmallWebRTC request handler
|
|
236
|
+
small_webrtc_handler: SmallWebRTCRequestHandler = SmallWebRTCRequestHandler(
|
|
237
|
+
esp32_mode=esp32_mode, host=host
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
@app.post("/api/offer")
|
|
241
|
+
async def offer(request: SmallWebRTCRequest, background_tasks: BackgroundTasks):
|
|
242
|
+
"""Handle WebRTC offer requests via SmallWebRTCRequestHandler."""
|
|
233
243
|
|
|
234
|
-
|
|
244
|
+
# Prepare runner arguments with the callback to run your bot
|
|
245
|
+
async def webrtc_connection_callback(connection):
|
|
246
|
+
bot_module = _get_bot_module()
|
|
247
|
+
runner_args = SmallWebRTCRunnerArguments(webrtc_connection=connection)
|
|
248
|
+
background_tasks.add_task(bot_module.bot, runner_args)
|
|
235
249
|
|
|
236
|
-
|
|
250
|
+
# Delegate handling to SmallWebRTCRequestHandler
|
|
251
|
+
answer = await small_webrtc_handler.handle_web_request(
|
|
252
|
+
request=request,
|
|
253
|
+
webrtc_connection_callback=webrtc_connection_callback,
|
|
254
|
+
)
|
|
237
255
|
return answer
|
|
238
256
|
|
|
239
257
|
@asynccontextmanager
|
|
240
|
-
async def
|
|
258
|
+
async def smallwebrtc_lifespan(app: FastAPI):
|
|
241
259
|
"""Manage FastAPI application lifecycle and cleanup connections."""
|
|
242
260
|
yield
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
261
|
+
await small_webrtc_handler.close()
|
|
262
|
+
|
|
263
|
+
# Add the SmallWebRTC lifespan to the app
|
|
264
|
+
_add_lifespan_to_app(app, smallwebrtc_lifespan)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _add_lifespan_to_app(app: FastAPI, new_lifespan):
|
|
268
|
+
"""Add a new lifespan context manager to the app, combining with existing if present.
|
|
269
|
+
|
|
270
|
+
Args:
|
|
271
|
+
app: The FastAPI application instance
|
|
272
|
+
new_lifespan: The new lifespan context manager to add
|
|
273
|
+
"""
|
|
274
|
+
if hasattr(app.router, "lifespan_context") and app.router.lifespan_context is not None:
|
|
275
|
+
# If there's already a lifespan context, combine them
|
|
276
|
+
existing_lifespan = app.router.lifespan_context
|
|
277
|
+
|
|
278
|
+
@asynccontextmanager
|
|
279
|
+
async def combined_lifespan(app: FastAPI):
|
|
280
|
+
async with existing_lifespan(app):
|
|
281
|
+
async with new_lifespan(app):
|
|
282
|
+
yield
|
|
283
|
+
|
|
284
|
+
app.router.lifespan_context = combined_lifespan
|
|
285
|
+
else:
|
|
286
|
+
# No existing lifespan, use the new one
|
|
287
|
+
app.router.lifespan_context = new_lifespan
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _setup_whatsapp_routes(app: FastAPI):
|
|
291
|
+
"""Set up WebRTC-specific routes."""
|
|
292
|
+
try:
|
|
293
|
+
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
|
|
246
294
|
|
|
247
|
-
|
|
295
|
+
from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
|
|
296
|
+
from pipecat.transports.smallwebrtc.request_handler import (
|
|
297
|
+
SmallWebRTCRequest,
|
|
298
|
+
SmallWebRTCRequestHandler,
|
|
299
|
+
)
|
|
300
|
+
from pipecat.transports.whatsapp.api import WhatsAppWebhookRequest
|
|
301
|
+
from pipecat.transports.whatsapp.client import WhatsAppClient
|
|
302
|
+
except ImportError as e:
|
|
303
|
+
logger.error(f"WebRTC transport dependencies not installed: {e}")
|
|
304
|
+
return
|
|
305
|
+
|
|
306
|
+
WHATSAPP_TOKEN = os.getenv("WHATSAPP_TOKEN")
|
|
307
|
+
WHATSAPP_PHONE_NUMBER_ID = os.getenv("WHATSAPP_PHONE_NUMBER_ID")
|
|
308
|
+
WHATSAPP_WEBHOOK_VERIFICATION_TOKEN = os.getenv("WHATSAPP_WEBHOOK_VERIFICATION_TOKEN")
|
|
309
|
+
WHATSAPP_APP_SECRET = os.getenv("WHATSAPP_APP_SECRET")
|
|
310
|
+
|
|
311
|
+
if not all(
|
|
312
|
+
[
|
|
313
|
+
WHATSAPP_TOKEN,
|
|
314
|
+
WHATSAPP_PHONE_NUMBER_ID,
|
|
315
|
+
WHATSAPP_WEBHOOK_VERIFICATION_TOKEN,
|
|
316
|
+
]
|
|
317
|
+
):
|
|
318
|
+
logger.debug(
|
|
319
|
+
"Missing required environment variables for WhatsApp transport. Keeping it disabled."
|
|
320
|
+
)
|
|
321
|
+
return
|
|
322
|
+
|
|
323
|
+
# Global WhatsApp client instance
|
|
324
|
+
whatsapp_client: Optional[WhatsAppClient] = None
|
|
325
|
+
|
|
326
|
+
@app.get(
|
|
327
|
+
"/whatsapp",
|
|
328
|
+
summary="Verify WhatsApp webhook",
|
|
329
|
+
description="Handles WhatsApp webhook verification requests from Meta",
|
|
330
|
+
)
|
|
331
|
+
async def verify_webhook(request: Request):
|
|
332
|
+
"""Verify WhatsApp webhook endpoint.
|
|
333
|
+
|
|
334
|
+
This endpoint is called by Meta's WhatsApp Business API to verify
|
|
335
|
+
the webhook URL during setup. It validates the verification token
|
|
336
|
+
and returns the challenge parameter if successful.
|
|
337
|
+
"""
|
|
338
|
+
if whatsapp_client is None:
|
|
339
|
+
logger.error("WhatsApp client is not initialized")
|
|
340
|
+
raise HTTPException(status_code=503, detail="Service unavailable")
|
|
341
|
+
|
|
342
|
+
params = dict(request.query_params)
|
|
343
|
+
logger.debug(f"Webhook verification request received with params: {list(params.keys())}")
|
|
344
|
+
|
|
345
|
+
try:
|
|
346
|
+
result = await whatsapp_client.handle_verify_webhook_request(
|
|
347
|
+
params=params, expected_verification_token=WHATSAPP_WEBHOOK_VERIFICATION_TOKEN
|
|
348
|
+
)
|
|
349
|
+
logger.info("Webhook verification successful")
|
|
350
|
+
return result
|
|
351
|
+
except ValueError as e:
|
|
352
|
+
logger.warning(f"Webhook verification failed: {e}")
|
|
353
|
+
raise HTTPException(status_code=403, detail="Verification failed")
|
|
354
|
+
|
|
355
|
+
@app.post(
|
|
356
|
+
"/whatsapp",
|
|
357
|
+
summary="Handle WhatsApp webhook events",
|
|
358
|
+
description="Processes incoming WhatsApp messages and call events",
|
|
359
|
+
)
|
|
360
|
+
async def whatsapp_webhook(
|
|
361
|
+
body: WhatsAppWebhookRequest,
|
|
362
|
+
background_tasks: BackgroundTasks,
|
|
363
|
+
request: Request,
|
|
364
|
+
x_hub_signature_256: str = Header(None),
|
|
365
|
+
):
|
|
366
|
+
"""Handle incoming WhatsApp webhook events.
|
|
367
|
+
|
|
368
|
+
For call events, establishes WebRTC connections and spawns bot instances
|
|
369
|
+
in the background to handle real-time communication.
|
|
370
|
+
"""
|
|
371
|
+
if whatsapp_client is None:
|
|
372
|
+
logger.error("WhatsApp client is not initialized")
|
|
373
|
+
raise HTTPException(status_code=503, detail="Service unavailable")
|
|
374
|
+
|
|
375
|
+
# Validate webhook object type
|
|
376
|
+
if body.object != "whatsapp_business_account":
|
|
377
|
+
logger.warning(f"Invalid webhook object type: {body.object}")
|
|
378
|
+
raise HTTPException(status_code=400, detail="Invalid object type")
|
|
379
|
+
|
|
380
|
+
logger.debug(f"Processing WhatsApp webhook: {body.model_dump()}")
|
|
381
|
+
|
|
382
|
+
async def connection_callback(connection: SmallWebRTCConnection):
|
|
383
|
+
"""Handle new WebRTC connections from WhatsApp calls.
|
|
384
|
+
|
|
385
|
+
Called when a WebRTC connection is established for a WhatsApp call.
|
|
386
|
+
Spawns a bot instance to handle the conversation.
|
|
387
|
+
|
|
388
|
+
Args:
|
|
389
|
+
connection: The established WebRTC connection
|
|
390
|
+
"""
|
|
391
|
+
bot_module = _get_bot_module()
|
|
392
|
+
runner_args = SmallWebRTCRunnerArguments(webrtc_connection=connection)
|
|
393
|
+
background_tasks.add_task(bot_module.bot, runner_args)
|
|
394
|
+
|
|
395
|
+
try:
|
|
396
|
+
# Process the webhook request
|
|
397
|
+
raw_body = await request.body()
|
|
398
|
+
result = await whatsapp_client.handle_webhook_request(
|
|
399
|
+
body, connection_callback, sha256_signature=x_hub_signature_256, raw_body=raw_body
|
|
400
|
+
)
|
|
401
|
+
logger.debug(f"Webhook processed successfully: {result}")
|
|
402
|
+
return {"status": "success", "message": "Webhook processed successfully"}
|
|
403
|
+
except ValueError as ve:
|
|
404
|
+
logger.warning(f"Invalid webhook request format: {ve}")
|
|
405
|
+
raise HTTPException(status_code=400, detail=f"Invalid request: {str(ve)}")
|
|
406
|
+
except Exception as e:
|
|
407
|
+
logger.error(f"Internal error processing webhook: {e}")
|
|
408
|
+
raise HTTPException(status_code=500, detail="Internal server error processing webhook")
|
|
409
|
+
|
|
410
|
+
@asynccontextmanager
|
|
411
|
+
async def whatsapp_lifespan(app: FastAPI):
|
|
412
|
+
"""Manage WhatsApp client lifecycle and cleanup connections."""
|
|
413
|
+
nonlocal whatsapp_client
|
|
414
|
+
|
|
415
|
+
# Initialize WhatsApp client with persistent HTTP session
|
|
416
|
+
async with aiohttp.ClientSession() as session:
|
|
417
|
+
whatsapp_client = WhatsAppClient(
|
|
418
|
+
whatsapp_token=WHATSAPP_TOKEN,
|
|
419
|
+
whatsapp_secret=WHATSAPP_APP_SECRET,
|
|
420
|
+
phone_number_id=WHATSAPP_PHONE_NUMBER_ID,
|
|
421
|
+
session=session,
|
|
422
|
+
)
|
|
423
|
+
logger.info("WhatsApp client initialized successfully")
|
|
424
|
+
|
|
425
|
+
try:
|
|
426
|
+
yield # Run the application
|
|
427
|
+
finally:
|
|
428
|
+
# Cleanup all active calls on shutdown
|
|
429
|
+
logger.info("Cleaning up WhatsApp client resources...")
|
|
430
|
+
if whatsapp_client:
|
|
431
|
+
await whatsapp_client.terminate_all_calls()
|
|
432
|
+
logger.info("WhatsApp cleanup completed")
|
|
433
|
+
|
|
434
|
+
# Add the WhatsApp lifespan to the app
|
|
435
|
+
_add_lifespan_to_app(app, whatsapp_lifespan)
|
|
248
436
|
|
|
249
437
|
|
|
250
438
|
def _setup_daily_routes(app: FastAPI):
|
|
@@ -332,7 +520,7 @@ def _setup_daily_routes(app: FastAPI):
|
|
|
332
520
|
return await _handle_rtvi_request(request)
|
|
333
521
|
|
|
334
522
|
|
|
335
|
-
def _setup_telephony_routes(app: FastAPI, transport_type: str, proxy: str):
|
|
523
|
+
def _setup_telephony_routes(app: FastAPI, *, transport_type: str, proxy: str):
|
|
336
524
|
"""Set up telephony-specific routes."""
|
|
337
525
|
# XML response templates (Exotel doesn't use XML webhooks)
|
|
338
526
|
XML_TEMPLATES = {
|
|
@@ -435,6 +623,21 @@ def _validate_and_clean_proxy(proxy: str) -> str:
|
|
|
435
623
|
return proxy
|
|
436
624
|
|
|
437
625
|
|
|
626
|
+
def runner_downloads_folder() -> Optional[str]:
|
|
627
|
+
"""Returns the folder where files are stored for later download."""
|
|
628
|
+
return RUNNER_DOWNLOADS_FOLDER
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def runner_host() -> str:
|
|
632
|
+
"""Returns the host name of this runner."""
|
|
633
|
+
return RUNNER_HOST
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def runner_port() -> int:
|
|
637
|
+
"""Returns the port of this runner."""
|
|
638
|
+
return RUNNER_PORT
|
|
639
|
+
|
|
640
|
+
|
|
438
641
|
def main():
|
|
439
642
|
"""Start the Pipecat development runner.
|
|
440
643
|
|
|
@@ -455,14 +658,16 @@ def main():
|
|
|
455
658
|
|
|
456
659
|
The bot file must contain a `bot(runner_args)` function as the entry point.
|
|
457
660
|
"""
|
|
661
|
+
global RUNNER_DOWNLOADS_FOLDER, RUNNER_HOST, RUNNER_PORT
|
|
662
|
+
|
|
458
663
|
parser = argparse.ArgumentParser(description="Pipecat Development Runner")
|
|
459
|
-
parser.add_argument("--host", type=str, default=
|
|
460
|
-
parser.add_argument("--port", type=int, default=
|
|
664
|
+
parser.add_argument("--host", type=str, default=RUNNER_HOST, help="Host address")
|
|
665
|
+
parser.add_argument("--port", type=int, default=RUNNER_PORT, help="Port number")
|
|
461
666
|
parser.add_argument(
|
|
462
667
|
"-t",
|
|
463
668
|
"--transport",
|
|
464
669
|
type=str,
|
|
465
|
-
choices=["daily", "webrtc",
|
|
670
|
+
choices=["daily", "webrtc", *TELEPHONY_TRANSPORTS],
|
|
466
671
|
default="webrtc",
|
|
467
672
|
help="Transport type",
|
|
468
673
|
)
|
|
@@ -480,6 +685,7 @@ def main():
|
|
|
480
685
|
default=False,
|
|
481
686
|
help="Connect directly to Daily room (automatically sets transport to daily)",
|
|
482
687
|
)
|
|
688
|
+
parser.add_argument("-f", "--folder", type=str, help="Path to downloads folder")
|
|
483
689
|
parser.add_argument(
|
|
484
690
|
"--verbose", "-v", action="count", default=0, help="Increase logging verbosity"
|
|
485
691
|
)
|
|
@@ -502,6 +708,10 @@ def main():
|
|
|
502
708
|
logger.error("For ESP32, you need to specify `--host IP` so we can do SDP munging.")
|
|
503
709
|
return
|
|
504
710
|
|
|
711
|
+
if args.transport in TELEPHONY_TRANSPORTS and not args.proxy:
|
|
712
|
+
logger.error(f"For telephony transports, you need to specify `--proxy PROXY`.")
|
|
713
|
+
return
|
|
714
|
+
|
|
505
715
|
# Log level
|
|
506
716
|
logger.remove()
|
|
507
717
|
logger.add(sys.stderr, level="TRACE" if args.verbose else "DEBUG")
|
|
@@ -532,8 +742,18 @@ def main():
|
|
|
532
742
|
print(f" → Open http://{args.host}:{args.port} in your browser to start a session")
|
|
533
743
|
print()
|
|
534
744
|
|
|
745
|
+
RUNNER_DOWNLOADS_FOLDER = args.folder
|
|
746
|
+
RUNNER_HOST = args.host
|
|
747
|
+
RUNNER_PORT = args.port
|
|
748
|
+
|
|
535
749
|
# Create the app with transport-specific setup
|
|
536
|
-
app = _create_server_app(
|
|
750
|
+
app = _create_server_app(
|
|
751
|
+
transport_type=args.transport,
|
|
752
|
+
host=args.host,
|
|
753
|
+
proxy=args.proxy,
|
|
754
|
+
esp32_mode=args.esp32,
|
|
755
|
+
folder=args.folder,
|
|
756
|
+
)
|
|
537
757
|
|
|
538
758
|
# Run the server
|
|
539
759
|
uvicorn.run(app, host=args.host, port=args.port)
|
pipecat/runner/types.py
CHANGED
|
@@ -51,9 +51,11 @@ class WebSocketRunnerArguments(RunnerArguments):
|
|
|
51
51
|
|
|
52
52
|
Parameters:
|
|
53
53
|
websocket: WebSocket connection for audio streaming
|
|
54
|
+
body: Additional request data
|
|
54
55
|
"""
|
|
55
56
|
|
|
56
57
|
websocket: WebSocket
|
|
58
|
+
body: Optional[Any] = field(default_factory=dict)
|
|
57
59
|
|
|
58
60
|
|
|
59
61
|
@dataclass
|
pipecat/runner/utils.py
CHANGED
|
@@ -99,16 +99,47 @@ async def parse_telephony_websocket(websocket: WebSocket):
|
|
|
99
99
|
tuple: (transport_type: str, call_data: dict)
|
|
100
100
|
|
|
101
101
|
call_data contains provider-specific fields:
|
|
102
|
-
|
|
103
|
-
-
|
|
104
|
-
|
|
105
|
-
|
|
102
|
+
|
|
103
|
+
- Twilio::
|
|
104
|
+
|
|
105
|
+
{
|
|
106
|
+
"stream_id": str,
|
|
107
|
+
"call_id": str,
|
|
108
|
+
"body": dict
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
- Telnyx::
|
|
112
|
+
|
|
113
|
+
{
|
|
114
|
+
"stream_id": str,
|
|
115
|
+
"call_control_id": str,
|
|
116
|
+
"outbound_encoding": str,
|
|
117
|
+
"from": str,
|
|
118
|
+
"to": str,
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
- Plivo::
|
|
122
|
+
|
|
123
|
+
{
|
|
124
|
+
"stream_id": str,
|
|
125
|
+
"call_id": str,
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
- Exotel::
|
|
129
|
+
|
|
130
|
+
{
|
|
131
|
+
"stream_id": str,
|
|
132
|
+
"call_id": str,
|
|
133
|
+
"account_sid": str,
|
|
134
|
+
"from": str,
|
|
135
|
+
"to": str,
|
|
136
|
+
}
|
|
106
137
|
|
|
107
138
|
Example usage::
|
|
108
139
|
|
|
109
140
|
transport_type, call_data = await parse_telephony_websocket(websocket)
|
|
110
|
-
if transport_type == "
|
|
111
|
-
|
|
141
|
+
if transport_type == "twilio":
|
|
142
|
+
user_id = call_data["body"]["user_id"]
|
|
112
143
|
"""
|
|
113
144
|
# Read first two messages
|
|
114
145
|
start_data = websocket.iter_text()
|
|
@@ -151,9 +182,12 @@ async def parse_telephony_websocket(websocket: WebSocket):
|
|
|
151
182
|
# Extract provider-specific data
|
|
152
183
|
if transport_type == "twilio":
|
|
153
184
|
start_data = call_data_raw.get("start", {})
|
|
185
|
+
body_data = start_data.get("customParameters", {})
|
|
154
186
|
call_data = {
|
|
155
187
|
"stream_id": start_data.get("streamSid"),
|
|
156
188
|
"call_id": start_data.get("callSid"),
|
|
189
|
+
# All custom parameters
|
|
190
|
+
"body": body_data,
|
|
157
191
|
}
|
|
158
192
|
|
|
159
193
|
elif transport_type == "telnyx":
|
|
@@ -163,6 +197,8 @@ async def parse_telephony_websocket(websocket: WebSocket):
|
|
|
163
197
|
"outbound_encoding": call_data_raw.get("start", {})
|
|
164
198
|
.get("media_format", {})
|
|
165
199
|
.get("encoding"),
|
|
200
|
+
"from": call_data_raw.get("start", {}).get("from", ""),
|
|
201
|
+
"to": call_data_raw.get("start", {}).get("to", ""),
|
|
166
202
|
}
|
|
167
203
|
|
|
168
204
|
elif transport_type == "plivo":
|
|
@@ -178,6 +214,8 @@ async def parse_telephony_websocket(websocket: WebSocket):
|
|
|
178
214
|
"stream_id": start_data.get("stream_sid"),
|
|
179
215
|
"call_id": start_data.get("call_sid"),
|
|
180
216
|
"account_sid": start_data.get("account_sid"),
|
|
217
|
+
"from": start_data.get("from", ""),
|
|
218
|
+
"to": start_data.get("to", ""),
|
|
181
219
|
}
|
|
182
220
|
|
|
183
221
|
else:
|
|
@@ -275,6 +313,7 @@ def _smallwebrtc_sdp_cleanup_ice_candidates(text: str, pattern: str) -> str:
|
|
|
275
313
|
Returns:
|
|
276
314
|
Cleaned SDP text with filtered ICE candidates.
|
|
277
315
|
"""
|
|
316
|
+
logger.debug("Removing unsupported ICE candidates from SDP")
|
|
278
317
|
result = []
|
|
279
318
|
lines = text.splitlines()
|
|
280
319
|
for line in lines:
|
|
@@ -283,7 +322,7 @@ def _smallwebrtc_sdp_cleanup_ice_candidates(text: str, pattern: str) -> str:
|
|
|
283
322
|
result.append(line)
|
|
284
323
|
else:
|
|
285
324
|
result.append(line)
|
|
286
|
-
return "\r\n".join(result)
|
|
325
|
+
return "\r\n".join(result) + "\r\n"
|
|
287
326
|
|
|
288
327
|
|
|
289
328
|
def _smallwebrtc_sdp_cleanup_fingerprints(text: str) -> str:
|
|
@@ -295,15 +334,16 @@ def _smallwebrtc_sdp_cleanup_fingerprints(text: str) -> str:
|
|
|
295
334
|
Returns:
|
|
296
335
|
SDP text with sha-384 and sha-512 fingerprints removed.
|
|
297
336
|
"""
|
|
337
|
+
logger.debug("Removing unsupported fingerprints from SDP")
|
|
298
338
|
result = []
|
|
299
339
|
lines = text.splitlines()
|
|
300
340
|
for line in lines:
|
|
301
341
|
if not re.search("sha-384", line) and not re.search("sha-512", line):
|
|
302
342
|
result.append(line)
|
|
303
|
-
return "\r\n".join(result)
|
|
343
|
+
return "\r\n".join(result) + "\r\n"
|
|
304
344
|
|
|
305
345
|
|
|
306
|
-
def smallwebrtc_sdp_munging(sdp: str, host: str) -> str:
|
|
346
|
+
def smallwebrtc_sdp_munging(sdp: str, host: Optional[str]) -> str:
|
|
307
347
|
"""Apply SDP modifications for SmallWebRTC compatibility.
|
|
308
348
|
|
|
309
349
|
Args:
|
|
@@ -314,7 +354,8 @@ def smallwebrtc_sdp_munging(sdp: str, host: str) -> str:
|
|
|
314
354
|
Modified SDP string with fingerprint and ICE candidate cleanup.
|
|
315
355
|
"""
|
|
316
356
|
sdp = _smallwebrtc_sdp_cleanup_fingerprints(sdp)
|
|
317
|
-
|
|
357
|
+
if host:
|
|
358
|
+
sdp = _smallwebrtc_sdp_cleanup_ice_candidates(sdp, host)
|
|
318
359
|
return sdp
|
|
319
360
|
|
|
320
361
|
|
pipecat/serializers/exotel.py
CHANGED
|
@@ -20,10 +20,10 @@ from pipecat.frames.frames import (
|
|
|
20
20
|
Frame,
|
|
21
21
|
InputAudioRawFrame,
|
|
22
22
|
InputDTMFFrame,
|
|
23
|
+
InterruptionFrame,
|
|
24
|
+
OutputTransportMessageFrame,
|
|
25
|
+
OutputTransportMessageUrgentFrame,
|
|
23
26
|
StartFrame,
|
|
24
|
-
StartInterruptionFrame,
|
|
25
|
-
TransportMessageFrame,
|
|
26
|
-
TransportMessageUrgentFrame,
|
|
27
27
|
)
|
|
28
28
|
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
|
|
29
29
|
|
|
@@ -98,7 +98,7 @@ class ExotelFrameSerializer(FrameSerializer):
|
|
|
98
98
|
Returns:
|
|
99
99
|
Serialized data as string or bytes, or None if the frame isn't handled.
|
|
100
100
|
"""
|
|
101
|
-
if isinstance(frame,
|
|
101
|
+
if isinstance(frame, InterruptionFrame):
|
|
102
102
|
answer = {"event": "clear", "streamSid": self._stream_sid}
|
|
103
103
|
return json.dumps(answer)
|
|
104
104
|
elif isinstance(frame, AudioRawFrame):
|
|
@@ -121,7 +121,7 @@ class ExotelFrameSerializer(FrameSerializer):
|
|
|
121
121
|
}
|
|
122
122
|
|
|
123
123
|
return json.dumps(answer)
|
|
124
|
-
elif isinstance(frame, (
|
|
124
|
+
elif isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
|
|
125
125
|
return json.dumps(frame.message)
|
|
126
126
|
|
|
127
127
|
return None
|
pipecat/serializers/livekit.py
CHANGED
|
@@ -25,11 +25,31 @@ except ModuleNotFoundError as e:
|
|
|
25
25
|
class LivekitFrameSerializer(FrameSerializer):
|
|
26
26
|
"""Serializer for converting between Pipecat frames and LiveKit audio frames.
|
|
27
27
|
|
|
28
|
+
.. deprecated:: 0.0.90
|
|
29
|
+
|
|
30
|
+
This class is deprecated and will be removed in a future version.
|
|
31
|
+
Please use LiveKitTransport instead, which handles audio streaming
|
|
32
|
+
and frame conversion natively.
|
|
33
|
+
|
|
28
34
|
This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects
|
|
29
35
|
to LiveKit AudioFrame objects for transmission, and the reverse conversion
|
|
30
36
|
for received audio data.
|
|
31
37
|
"""
|
|
32
38
|
|
|
39
|
+
def __init__(self):
|
|
40
|
+
"""Initialize the LiveKit frame serializer."""
|
|
41
|
+
super().__init__()
|
|
42
|
+
import warnings
|
|
43
|
+
|
|
44
|
+
with warnings.catch_warnings():
|
|
45
|
+
warnings.simplefilter("always")
|
|
46
|
+
warnings.warn(
|
|
47
|
+
"LivekitFrameSerializer is deprecated and will be removed in a future version. "
|
|
48
|
+
"Please use LiveKitTransport instead, which handles audio streaming natively.",
|
|
49
|
+
DeprecationWarning,
|
|
50
|
+
stacklevel=2,
|
|
51
|
+
)
|
|
52
|
+
|
|
33
53
|
@property
|
|
34
54
|
def type(self) -> FrameSerializerType:
|
|
35
55
|
"""Get the serializer type.
|