pipecat-aircaps 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,11 @@
1
+ #
2
+ # Copyright (c) 2026, AirCaps
3
+ #
4
+ # SPDX-License-Identifier: BSD 2-Clause License
5
+ #
6
+
7
+ """AirCaps speech-to-text services for Pipecat."""
8
+
9
+ from pipecat_aircaps.stt import AirCapsLiveSTTService, AirCapsLiveSTTSettings
10
+
11
+ __all__ = ["AirCapsLiveSTTService", "AirCapsLiveSTTSettings"]
File without changes
pipecat_aircaps/stt.py ADDED
@@ -0,0 +1,582 @@
1
+ #
2
+ # Copyright (c) 2026, AirCaps
3
+ #
4
+ # SPDX-License-Identifier: BSD 2-Clause License
5
+ #
6
+
7
+ """AirCaps Live 1 speech-to-text service for Pipecat.
8
+
9
+ AirCaps Live 1 (``aircaps-live-1``) is a low-latency English streaming
10
+ speech-to-text model for voice agents. Audio streams over a WebSocket and the
11
+ transcript comes back as the user speaks. Turns end one of two ways: Pipecat's
12
+ VAD reports that the user stopped speaking and the service asks for the final
13
+ transcript at once, or, with ``end_of_turn=True``, the model detects the end of
14
+ the turn itself.
15
+ """
16
+
17
+ import asyncio
18
+ import json
19
+ from collections.abc import AsyncGenerator
20
+ from dataclasses import dataclass, field
21
+ from typing import Any, Literal
22
+
23
+ import aiohttp
24
+ from loguru import logger
25
+ from pipecat.frames.frames import (
26
+ Frame,
27
+ InterimTranscriptionFrame,
28
+ ProposedUserStartedSpeakingFrame,
29
+ ProposedUserStoppedSpeakingFrame,
30
+ StartFrame,
31
+ STTMetadataFrame,
32
+ TranscriptionFrame,
33
+ UserStartedSpeakingFrame,
34
+ UserStoppedSpeakingFrame,
35
+ VADUserStoppedSpeakingFrame,
36
+ )
37
+ from pipecat.processors.frame_processor import FrameDirection
38
+ from pipecat.services.settings import STTSettings
39
+ from pipecat.services.stt_service import WebsocketSTTService
40
+ from pipecat.transcriptions.language import Language
41
+ from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies
42
+ from pipecat.utils.time import time_now_iso8601
43
+ from pipecat.utils.tracing.service_decorators import traced_stt
44
+ from pipecat.utils.types import NOT_GIVEN, NotGiven, is_given
45
+ from websockets.protocol import State
46
+
47
+ # P99 time from end of speech to final transcript, with VADParams.stop_secs=0.2
48
+ # (the Pipecat default), measured from US East. Includes the VAD stop wait.
49
+ AIRCAPS_LIVE_TTFS_P99 = 0.25
50
+
51
+ # The model takes 16 kHz mono PCM16; other input rates are resampled to it.
52
+ _SAMPLE_RATE = 16000
53
+
54
+ # The server closes a session that sends nothing for 45 seconds (for example
55
+ # while the STT is muted), so an idle connection is pinged before then.
56
+ _KEEPALIVE_TIMEOUT = 20.0
57
+
58
+ # Punctuation that can close the previous turn's text after it was final.
59
+ _LEADING_PUNCTUATION = " .,!?;:"
60
+
61
+ # End-of-turn settings that override single values of the mode.
62
+ _END_OF_TURN_OVERRIDES = (
63
+ "end_of_turn_threshold",
64
+ "min_turn_silence_ms",
65
+ "max_turn_silence_ms",
66
+ "vad_threshold",
67
+ )
68
+
69
+ # New words that VAD did not hear are finalized after this many seconds without
70
+ # newer words (see _restart_missed_speech_timer).
71
+ _MISSED_SPEECH_WAIT = 1.0
72
+ # With end_of_turn: how long a new turn's first transcript waits for Pipecat to
73
+ # start the turn it proposed (see _on_transcript).
74
+ _TURN_START_WAIT = 0.5
75
+
76
+ # How long to wait for the session to be ready after connecting.
77
+ _READY_TIMEOUT = 15.0
78
+
79
+
80
+ class _LookupError(ConnectionError):
81
+ """The endpoint lookup was refused; ``status`` lets Pipecat classify why."""
82
+
83
+ def __init__(self, status: int, message: str):
84
+ super().__init__(message)
85
+ self.status = status
86
+
87
+
88
+ @dataclass
89
+ class AirCapsLiveSTTSettings(STTSettings):
90
+ """Settings for AirCapsLiveSTTService.
91
+
92
+ Parameters:
93
+ model: The AirCaps model id. Defaults to ``"aircaps-live-1"``.
94
+ language: Transcription language. AirCaps Live 1 transcribes English.
95
+ end_of_turn_mode: How the model trades speed for certainty when it
96
+ detects the end of a turn: ``"min_latency"``, ``"balanced"`` (the
97
+ default) or ``"max_accuracy"``. Used only with ``end_of_turn=True``.
98
+ The fields below override single values of the mode; None keeps
99
+ the mode's value.
100
+ end_of_turn_threshold: How sure the model must be that the turn ended,
101
+ 0.2 to 0.9 (modes: 0.3, 0.5, 0.6).
102
+ min_turn_silence_ms: The shortest pause that can end a turn, 0 to 1000
103
+ ms (modes: 40, 80, 400).
104
+ max_turn_silence_ms: A pause this long always ends the turn: 500 to
105
+ 5000 ms, or 0 for no limit (modes: 2560).
106
+ vad_threshold: How sure the model's voice detector must be that it
107
+ hears speech, 0.3 to 0.9.
108
+ """
109
+
110
+ end_of_turn_mode: Literal["min_latency", "balanced", "max_accuracy"] | NotGiven = field(
111
+ default_factory=lambda: NOT_GIVEN
112
+ )
113
+ end_of_turn_threshold: float | None | NotGiven = field(default_factory=lambda: NOT_GIVEN)
114
+ min_turn_silence_ms: int | None | NotGiven = field(default_factory=lambda: NOT_GIVEN)
115
+ max_turn_silence_ms: int | None | NotGiven = field(default_factory=lambda: NOT_GIVEN)
116
+ vad_threshold: float | None | NotGiven = field(default_factory=lambda: NOT_GIVEN)
117
+
118
+
119
+ class AirCapsLiveSTTService(WebsocketSTTService):
120
+ """Speech-to-text service using AirCaps Live 1.
121
+
122
+ Streams audio to AirCaps over a WebSocket and pushes interim transcripts as
123
+ the user speaks.
124
+
125
+ By default Pipecat decides when a turn ends: when VAD reports that the user
126
+ stopped speaking, the service sends ``finalize`` and pushes the turn's final
127
+ ``TranscriptionFrame`` as soon as it arrives.
128
+
129
+ With ``end_of_turn=True`` the model decides instead. The service pushes a
130
+ ``ProposedUserStartedSpeakingFrame`` when a turn's first words arrive, and
131
+ the turn's final ``TranscriptionFrame`` followed by a
132
+ ``ProposedUserStoppedSpeakingFrame`` when the model ends the turn. It
133
+ recommends ``ExternalUserTurnStrategies``, so the user aggregator follows
134
+ these frames rather than its own VAD and turn analyzer.
135
+
136
+ Before each connection the service asks the AirCaps API where to connect
137
+ (``GET /v1/realtime/endpoint``), so audio goes straight to the machine
138
+ running the model.
139
+ """
140
+
141
+ Settings = AirCapsLiveSTTSettings
142
+ _settings: Settings
143
+
144
+ def __init__(
145
+ self,
146
+ *,
147
+ api_key: str,
148
+ base_url: str = "https://api.aircaps.com",
149
+ sample_rate: int | None = None,
150
+ end_of_turn: bool = False,
151
+ should_interrupt: bool = True,
152
+ settings: Settings | None = None,
153
+ ttfs_p99_latency: float | None = AIRCAPS_LIVE_TTFS_P99,
154
+ **kwargs,
155
+ ):
156
+ """Initialize the AirCaps Live STT service.
157
+
158
+ Args:
159
+ api_key: AirCaps API key.
160
+ base_url: AirCaps API address, used to look up the WebSocket address.
161
+ sample_rate: Input audio sample rate. If None, uses the pipeline's
162
+ input rate. Audio is resampled to 16 kHz before it is sent.
163
+ end_of_turn: Whether the model detects the end of each turn, rather
164
+ than Pipecat's VAD and turn strategies.
165
+ should_interrupt: With ``end_of_turn``, whether a new user turn
166
+ interrupts the bot. Passed to the recommended
167
+ ``ExternalUserTurnStrategies``; a user-supplied
168
+ ``user_turn_strategies`` overrides it.
169
+ settings: Runtime-updatable settings.
170
+ ttfs_p99_latency: P99 latency from speech end to final transcript in
171
+ seconds. Override for your deployment. See
172
+ https://github.com/pipecat-ai/stt-benchmark
173
+ **kwargs: Additional arguments passed to the parent WebsocketSTTService.
174
+ """
175
+ default_settings = self.Settings(
176
+ model="aircaps-live-1",
177
+ language=Language.EN,
178
+ end_of_turn_mode="balanced",
179
+ end_of_turn_threshold=None,
180
+ min_turn_silence_ms=None,
181
+ max_turn_silence_ms=None,
182
+ vad_threshold=None,
183
+ )
184
+ if settings is not None:
185
+ default_settings.apply_update(settings)
186
+
187
+ super().__init__(
188
+ sample_rate=sample_rate,
189
+ ttfs_p99_latency=ttfs_p99_latency,
190
+ keepalive_timeout=_KEEPALIVE_TIMEOUT,
191
+ settings=default_settings,
192
+ **kwargs,
193
+ )
194
+
195
+ self._api_key = api_key
196
+ self._base_url = base_url.rstrip("/")
197
+ self._end_of_turn = end_of_turn
198
+ self._should_interrupt = should_interrupt
199
+ # With end_of_turn: whether a turn has started and not yet ended, and
200
+ # whether Pipecat has started the user turn proposed for it.
201
+ self._in_turn = False
202
+ self._turn_started = asyncio.Event()
203
+ self._receive_task: asyncio.Task | None = None
204
+ # The session's whole transcript so far, and how much of it is final.
205
+ self._text = ""
206
+ self._final_length = 0
207
+ # The last interim pushed: an unchanged repeat is not pushed again.
208
+ self._last_interim = ""
209
+ self._missed_speech_task: asyncio.Task | None = None
210
+
211
+ def can_generate_metrics(self) -> bool:
212
+ """Check if this service can generate processing metrics.
213
+
214
+ Returns:
215
+ True, as AirCaps Live STT supports metrics generation.
216
+ """
217
+ return True
218
+
219
+ @property
220
+ def send_sample_rate(self) -> int:
221
+ """Get the rate audio is sent at: always 16 kHz.
222
+
223
+ Returns:
224
+ The sample rate in Hz.
225
+ """
226
+ return _SAMPLE_RATE
227
+
228
+ @property
229
+ def supports_ttfs(self) -> bool:
230
+ """Whether a speech-end to final-transcript time applies.
231
+
232
+ Returns:
233
+ False with ``end_of_turn``, where the model defines the turn's end.
234
+ """
235
+ return not self._end_of_turn
236
+
237
+ def service_metadata_frame(self) -> STTMetadataFrame:
238
+ """Recommend turn strategies that follow the model's turns, with ``end_of_turn``.
239
+
240
+ Returns:
241
+ The STT metadata frame broadcast at start.
242
+ """
243
+ frame = super().service_metadata_frame()
244
+ if self._end_of_turn:
245
+ frame.user_turn_strategies = ExternalUserTurnStrategies(
246
+ enable_interruptions=self._should_interrupt
247
+ )
248
+ return frame
249
+
250
+ async def start(self, frame: StartFrame):
251
+ """Start the service and connect.
252
+
253
+ Args:
254
+ frame: The start frame.
255
+ """
256
+ await super().start(frame)
257
+ await self._connect()
258
+
259
+ async def process_frame(self, frame: Frame, direction: FrameDirection):
260
+ """Process frames, asking for the final transcript when the user stops speaking.
261
+
262
+ Args:
263
+ frame: The frame to process.
264
+ direction: The direction of frame processing.
265
+ """
266
+ await super().process_frame(frame, direction)
267
+
268
+ if isinstance(frame, UserStartedSpeakingFrame):
269
+ self._turn_started.set()
270
+ if isinstance(frame, UserStoppedSpeakingFrame):
271
+ # Pipecat ended the user turn, maybe on its own (its stop timeout, when
272
+ # a final that does not end the turn is the last thing the model sent).
273
+ # The next words start a new turn: propose its start, or Pipecat never
274
+ # opens it and drops what the user says next.
275
+ self._in_turn = False
276
+ if isinstance(frame, VADUserStoppedSpeakingFrame) and not self._end_of_turn:
277
+ await self._cancel_missed_speech_timer()
278
+ await self._finalize()
279
+
280
+ async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame | None, None]:
281
+ """Send audio to AirCaps.
282
+
283
+ Args:
284
+ audio: Raw 16-bit mono PCM audio at 16 kHz.
285
+
286
+ Yields:
287
+ None, as transcripts arrive on the WebSocket.
288
+ """
289
+ # The receive loop reconnects a dropped socket; audio arriving
290
+ # meanwhile is dropped.
291
+ if self._is_open():
292
+ try:
293
+ await self._websocket.send(audio)
294
+ except Exception as e:
295
+ logger.warning(f"{self}: send failed: {e}")
296
+ yield None
297
+
298
+ async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
299
+ """Apply a settings delta, reconnecting to use it.
300
+
301
+ Args:
302
+ delta: An STT settings delta.
303
+
304
+ Returns:
305
+ Dict mapping changed field names to their previous values.
306
+ """
307
+ changed = await super()._update_settings(delta)
308
+ if changed:
309
+ await self._request_reconnect()
310
+ return changed
311
+
312
+ async def _connect(self):
313
+ await self._connect_websocket()
314
+
315
+ await super()._connect()
316
+
317
+ # Started even when the connection failed: with no socket the receive
318
+ # loop goes straight to its reconnect path, which retries the connect.
319
+ if not self._receive_task:
320
+ self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
321
+
322
+ async def _disconnect(self):
323
+ await super()._disconnect()
324
+ await self._cancel_missed_speech_timer()
325
+
326
+ if self._receive_task:
327
+ await self.cancel_task(self._receive_task)
328
+ self._receive_task = None
329
+
330
+ await self._disconnect_websocket()
331
+
332
+ async def _connect_websocket(self):
333
+ if self._is_open():
334
+ return
335
+ try:
336
+ logger.debug("Connecting to AirCaps Live STT")
337
+ url = await self._websocket_url()
338
+ websocket = await self._websocket_connect(
339
+ url, additional_headers={"Authorization": f"Bearer {self._api_key}"}
340
+ )
341
+ try:
342
+ await self._start_session(websocket)
343
+ except BaseException:
344
+ await websocket.close()
345
+ raise
346
+ self._text = ""
347
+ self._final_length = 0
348
+ self._last_interim = ""
349
+ self._websocket = websocket
350
+ logger.info(f"{self}: Connected to AirCaps Live STT")
351
+ await self._call_event_handler("on_connected")
352
+ except Exception as e:
353
+ self._websocket = None
354
+ await self.push_error(error_msg=f"Unable to connect to AirCaps: {e}", exception=e)
355
+
356
+ async def _disconnect_websocket(self):
357
+ websocket = self._websocket
358
+ try:
359
+ if websocket and websocket.state is State.OPEN:
360
+ logger.debug(f"{self}: Disconnecting from AirCaps Live STT")
361
+ await websocket.send(json.dumps({"type": "finish"}))
362
+ await websocket.close()
363
+ except Exception as e:
364
+ await self.push_error(error_msg=f"Error closing websocket: {e}", exception=e)
365
+ finally:
366
+ # Only clear if no concurrent connect has already replaced it.
367
+ if self._websocket is websocket:
368
+ self._websocket = None
369
+ await self._call_event_handler("on_disconnected")
370
+
371
+ async def _websocket_url(self) -> str:
372
+ """Ask the AirCaps API where to open this model's WebSocket."""
373
+ async with aiohttp.ClientSession() as session:
374
+ async with session.get(
375
+ f"{self._base_url}/v1/realtime/endpoint",
376
+ params={"model": str(self._settings.model)},
377
+ headers={"Authorization": f"Bearer {self._api_key}"},
378
+ timeout=aiohttp.ClientTimeout(total=10),
379
+ ) as response:
380
+ body = await response.json(content_type=None)
381
+ if response.status != 200:
382
+ error = body.get("error") if isinstance(body, dict) else None
383
+ if not isinstance(error, dict):
384
+ error = {}
385
+ # A refused key or model is permanent: the service stops retrying.
386
+ raise _LookupError(
387
+ response.status,
388
+ f"endpoint lookup failed ({response.status} "
389
+ f"{error.get('code', 'error')}): {error.get('message', body)}",
390
+ )
391
+ return body["websocket_url"]
392
+
393
+ async def _start_session(self, websocket):
394
+ """Send the start message and wait until the session is ready for audio."""
395
+ start = {
396
+ "type": "start",
397
+ "model": self._settings.model,
398
+ "encoding": "pcm_s16le",
399
+ "sample_rate": _SAMPLE_RATE,
400
+ "channels": 1,
401
+ }
402
+ if self._end_of_turn:
403
+ start.update(end_of_turn=True, end_of_turn_mode=self._settings.end_of_turn_mode)
404
+ for name in _END_OF_TURN_OVERRIDES:
405
+ value = getattr(self._settings, name)
406
+ if is_given(value) and value is not None:
407
+ start[name] = value
408
+ await websocket.send(json.dumps(start))
409
+ async with asyncio.timeout(_READY_TIMEOUT):
410
+ async for message in websocket:
411
+ data = json.loads(message)
412
+ if data.get("type") == "ready":
413
+ return
414
+ if data.get("type") == "error":
415
+ raise ConnectionError(f"{data.get('code')}: {data.get('message')}")
416
+ raise ConnectionError("connection closed before the session was ready")
417
+
418
+ def _restart_missed_speech_timer(self):
419
+ """Ask for the final if new words arrive that VAD did not hear.
420
+
421
+ VAD can miss a short or quiet word. Then it never reports the user
422
+ stopping, nothing asks for the final, and the turn the words started
423
+ stays open. If no newer words arrive within ``_MISSED_SPEECH_WAIT``
424
+ while VAD hears no speech, the service asks for the final itself.
425
+ """
426
+ if self._missed_speech_task:
427
+ self._missed_speech_task.cancel()
428
+ self._missed_speech_task = self.create_task(self._finalize_missed_speech())
429
+
430
+ async def _finalize_missed_speech(self):
431
+ await asyncio.sleep(_MISSED_SPEECH_WAIT)
432
+ self._missed_speech_task = None
433
+ if not self._user_speaking:
434
+ await self._finalize()
435
+
436
+ async def _cancel_missed_speech_timer(self):
437
+ if self._missed_speech_task:
438
+ task, self._missed_speech_task = self._missed_speech_task, None
439
+ await self.cancel_task(task)
440
+
441
+ async def _finalize(self):
442
+ """Ask for the final transcript of everything heard so far."""
443
+ if self._is_open():
444
+ self.request_finalize()
445
+ await self._send_json({"type": "finalize"})
446
+
447
+ def _is_open(self) -> bool:
448
+ return self._websocket is not None and self._websocket.state is State.OPEN
449
+
450
+ async def _send_json(self, message: dict):
451
+ try:
452
+ await self._websocket.send(json.dumps(message))
453
+ except Exception as e:
454
+ logger.warning(f"{self}: send failed: {e}")
455
+
456
+ async def _send_keepalive(self, silence: bytes):
457
+ """Ping the session so the server keeps an idle connection open.
458
+
459
+ Args:
460
+ silence: Unused: a ping keeps the session open without sending audio.
461
+ """
462
+ await self._send_json({"type": "ping"})
463
+
464
+ def _get_websocket(self):
465
+ if self._websocket:
466
+ return self._websocket
467
+ raise Exception("Websocket not connected")
468
+
469
+ async def _receive_messages(self):
470
+ """Process incoming WebSocket messages."""
471
+ try:
472
+ async for message in self._get_websocket():
473
+ try:
474
+ await self._process_message(json.loads(message))
475
+ except json.JSONDecodeError:
476
+ logger.warning(f"{self}: received non-JSON message: {message}")
477
+ except Exception as e:
478
+ logger.error(f"{self}: error processing message: {e}")
479
+ finally:
480
+ # A turn lives in one session, so it ends when the session does.
481
+ if self._in_turn and not self._disconnecting:
482
+ await self._end_turn(self._text[self._final_length :].strip(), None)
483
+
484
+ async def _process_message(self, data: dict):
485
+ kind = data.get("type")
486
+ if kind == "transcript":
487
+ await self._on_transcript(data)
488
+ elif kind == "error":
489
+ await self.push_error(
490
+ error_msg=f"AirCaps error {data.get('code')}: {data.get('message')}"
491
+ )
492
+ elif kind == "session_limit":
493
+ # The server ends the session next; the receive loop reconnects.
494
+ logger.warning(f"{self}: {data.get('message')}")
495
+
496
+ @traced_stt
497
+ async def _handle_transcription(
498
+ self, transcript: str, is_final: bool, language: Language | None = None
499
+ ):
500
+ """Handle a transcription result with tracing."""
501
+ pass
502
+
503
+ async def _on_transcript(self, data: dict):
504
+ # Each event carries the session's whole transcript; the characters
505
+ # before the last final never change, so the current turn is the rest.
506
+ self._text = data.get("text", "")
507
+ # The model may punctuate the end of the previous turn once it hears
508
+ # more ("evening" becomes "evening. Actually ..."): that punctuation
509
+ # belongs to the previous turn, not this one.
510
+ turn = self._text[self._final_length :].lstrip(_LEADING_PUNCTUATION).strip()
511
+ language = Language.EN
512
+
513
+ # Text that is only punctuation is not speech: the model can add "?" to
514
+ # the end of the previous turn after its final. Pushed as a transcript,
515
+ # it would start a new user turn in Pipecat and interrupt the bot.
516
+ if not any(c.isalnum() for c in turn):
517
+ turn = ""
518
+
519
+ if self._end_of_turn and not self._in_turn and turn:
520
+ self._in_turn = True
521
+ self._turn_started.clear()
522
+ await self.broadcast_frame(ProposedUserStartedSpeakingFrame)
523
+ # The turn's words go out once Pipecat has started the turn: an app
524
+ # that files transcripts under the user turn open when they arrive
525
+ # (Pipecat's web client) would otherwise put them in the previous one.
526
+ try:
527
+ await asyncio.wait_for(self._turn_started.wait(), _TURN_START_WAIT)
528
+ except TimeoutError:
529
+ pass
530
+
531
+ if not data.get("is_final"):
532
+ if turn and turn != self._last_interim:
533
+ self._last_interim = turn
534
+ await self.push_frame(
535
+ InterimTranscriptionFrame(
536
+ turn, self._user_id, time_now_iso8601(), language, result=data
537
+ )
538
+ )
539
+ if not self._end_of_turn:
540
+ self._restart_missed_speech_timer()
541
+ return
542
+
543
+ self._final_length = len(self._text)
544
+ self._last_interim = ""
545
+ await self._cancel_missed_speech_timer()
546
+ if self._end_of_turn and data.get("end_of_turn"):
547
+ # The model ended the turn: its final, then the turn's end.
548
+ if self._in_turn:
549
+ await self._end_turn(turn, data)
550
+ return
551
+
552
+ # A final within a turn (end_of_turn), or the answer to finalize.
553
+ self.confirm_finalize()
554
+ await self._push_final(turn, data)
555
+
556
+ async def _end_turn(self, turn: str, data: dict | None):
557
+ """Push the turn's final transcript and propose the end of the turn.
558
+
559
+ ``data`` is None for a turn cut short by its session ending: its text so
560
+ far is pushed, not marked finalized, since the model never ended it.
561
+ """
562
+ self._in_turn = False
563
+ await self._push_final(turn, data, finalized=data is not None)
564
+ await self.broadcast_frame(ProposedUserStoppedSpeakingFrame)
565
+
566
+ async def _push_final(self, turn: str, data: dict | None, finalized: bool = False):
567
+ if not turn:
568
+ return
569
+ # Report usage before the transcription frame so tracing can attach it
570
+ # to the STT span the frame closes.
571
+ await self.emit_stt_usage_metrics()
572
+ await self.push_frame(
573
+ TranscriptionFrame(
574
+ turn,
575
+ self._user_id,
576
+ time_now_iso8601(),
577
+ Language.EN,
578
+ result=data,
579
+ finalized=finalized,
580
+ )
581
+ )
582
+ await self._handle_transcription(turn, True, Language.EN)
@@ -0,0 +1,181 @@
1
+ Metadata-Version: 2.5
2
+ Name: pipecat-aircaps
3
+ Version: 0.1.0
4
+ Summary: AirCaps Live speech-to-text for Pipecat voice agents
5
+ Project-URL: Homepage, https://aircaps.ai
6
+ Project-URL: Documentation, https://aircaps.ai/docs/guides/live-1
7
+ Project-URL: Repository, https://github.com/aircaps/pipecat-aircaps
8
+ Project-URL: Issues, https://github.com/aircaps/pipecat-aircaps/issues
9
+ Project-URL: Changelog, https://github.com/aircaps/pipecat-aircaps/blob/main/CHANGELOG.md
10
+ Author: AirCaps
11
+ License-Expression: BSD-2-Clause
12
+ License-File: LICENSE
13
+ Keywords: aircaps,pipecat,realtime,speech-to-text,stt,voice-agents
14
+ Classifier: Development Status :: 4 - Beta
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Multimedia :: Sound/Audio :: Speech
23
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
24
+ Classifier: Typing :: Typed
25
+ Requires-Python: >=3.11
26
+ Requires-Dist: pipecat-ai<2,>=1.16.0
27
+ Provides-Extra: example
28
+ Requires-Dist: pipecat-ai[cartesia,openai,runner,silero,webrtc]<2,>=1.16.0; extra == 'example'
29
+ Requires-Dist: python-dotenv; extra == 'example'
30
+ Description-Content-Type: text/markdown
31
+
32
+ # pipecat-aircaps
33
+
34
+ [![PyPI](https://img.shields.io/pypi/v/pipecat-aircaps)](https://pypi.org/project/pipecat-aircaps/)
35
+ [![CI](https://github.com/aircaps/pipecat-aircaps/actions/workflows/ci.yml/badge.svg)](https://github.com/aircaps/pipecat-aircaps/actions/workflows/ci.yml)
36
+
37
+ [AirCaps](https://aircaps.ai) speech-to-text for [Pipecat](https://github.com/pipecat-ai/pipecat).
38
+
39
+ `AirCapsLiveSTTService` streams the user's audio to **AirCaps Live 1** (`aircaps-live-1`), a low-latency English
40
+ speech-to-text model built for voice agents. It pushes interim transcripts while the user speaks, and when VAD
41
+ reports the end of speech it asks for the turn's final transcript at once. It can also let the model decide when each
42
+ turn ends (`end_of_turn=True`).
43
+
44
+ Maintained by AirCaps, the company that provides AirCaps Live 1. This is a Pipecat community integration.
45
+
46
+ ## Installation
47
+
48
+ ```bash
49
+ uv add pipecat-aircaps
50
+ ```
51
+
52
+ or `pip install pipecat-aircaps`.
53
+
54
+ ## Prerequisites
55
+
56
+ An AirCaps API key: sign in to the [AirCaps console](https://console.aircaps.ai), open **API keys** and create one.
57
+ AirCaps Live 1 is available to every account, one session at a time by default
58
+ ([talk to us](https://aircaps.ai/contact) for more). The [AirCaps Live 1 guide](https://aircaps.ai/docs/guides/live-1)
59
+ describes the model and its protocol.
60
+
61
+ ```bash
62
+ export AIRCAPS_API_KEY=...
63
+ ```
64
+
65
+ ## Usage
66
+
67
+ ```python
68
+ import os
69
+
70
+ from pipecat_aircaps import AirCapsLiveSTTService
71
+
72
+ stt = AirCapsLiveSTTService(api_key=os.environ["AIRCAPS_API_KEY"])
73
+
74
+ pipeline = Pipeline(
75
+ [
76
+ transport.input(),
77
+ stt,
78
+ user_aggregator,
79
+ llm,
80
+ tts,
81
+ transport.output(),
82
+ assistant_aggregator,
83
+ ]
84
+ )
85
+ ```
86
+
87
+ Use it as you would any Pipecat STT service. It needs VAD frames to know when a turn ends, which the user aggregator
88
+ provides when it has a VAD analyzer (`LLMUserAggregatorParams(vad_analyzer=SileroVADAnalyzer())`), as in the example.
89
+
90
+ | Parameter | Default | |
91
+ | --- | --- | --- |
92
+ | `api_key` | required | AirCaps API key |
93
+ | `base_url` | `https://api.aircaps.com` | AirCaps API address |
94
+ | `sample_rate` | pipeline input rate | Input audio rate. Audio is resampled to 16 kHz before it is sent |
95
+ | `end_of_turn` | `False` | Let the model detect the end of each turn (see below) |
96
+ | `should_interrupt` | `True` | With `end_of_turn`: whether a new user turn interrupts the bot |
97
+ | `settings` | `model="aircaps-live-1"`, `language=Language.EN`, `end_of_turn_mode="balanced"` | `AirCapsLiveSTTService.Settings(...)` |
98
+ | `ttfs_p99_latency` | `0.25` | P99 seconds from end of speech to final transcript, for Pipecat's turn timing. Measure your own with [stt-benchmark](https://github.com/pipecat-ai/stt-benchmark) |
99
+
100
+ How it works:
101
+
102
+ - Before each connection the service asks `GET /v1/realtime/endpoint?model=aircaps-live-1` where to connect, then
103
+ opens that WebSocket, so audio goes straight to the model.
104
+ - On `VADUserStoppedSpeakingFrame` it sends `finalize`; the final transcript of the turn comes back as one
105
+ `TranscriptionFrame`, marked finalized, and Pipecat reports STT TTFB from it.
106
+ - If words arrive that VAD did not hear (a short or quiet word), the service asks for their final itself one second
107
+ after the last new words, so the user turn they started still ends.
108
+ - A session the server ends (sessions last up to an hour) is reopened automatically.
109
+ - While no audio is sent (for example when the STT is muted), the service pings the session so it stays open.
110
+
111
+ ## Model-detected end of turn
112
+
113
+ ```python
114
+ stt = AirCapsLiveSTTService(
115
+ api_key=os.environ["AIRCAPS_API_KEY"],
116
+ end_of_turn=True,
117
+ settings=AirCapsLiveSTTService.Settings(end_of_turn_mode="balanced"),
118
+ )
119
+ ```
120
+
121
+ With `end_of_turn=True` the model decides when the user has finished, instead of Pipecat's VAD and turn analyzer:
122
+
123
+ - When a turn's first words arrive, the service pushes `ProposedUserStartedSpeakingFrame`.
124
+ - When the model ends the turn, it pushes the turn's final `TranscriptionFrame` (finalized), then
125
+ `ProposedUserStoppedSpeakingFrame`.
126
+ - The service recommends `ExternalUserTurnStrategies`, so the user aggregator follows these frames. You don't need a
127
+ VAD analyzer in the user aggregator for turn ends. If you pass your own `user_turn_strategies`, they win.
128
+
129
+ `end_of_turn_mode` trades speed for certainty: `min_latency`, `balanced` (default) or `max_accuracy`. These settings
130
+ override single values of the mode (leave them unset to keep the mode's):
131
+
132
+ | Setting | Range | `min_latency` / `balanced` / `max_accuracy` |
133
+ | --- | --- | --- |
134
+ | `end_of_turn_threshold` | 0.2 to 0.9: how sure the model must be that the turn ended | 0.3 / 0.5 / 0.6 |
135
+ | `min_turn_silence_ms` | 0 to 1000: the shortest pause that can end a turn | 40 / 80 / 400 |
136
+ | `max_turn_silence_ms` | 500 to 5000, or 0 for no limit: a pause this long always ends the turn | 2560 |
137
+ | `vad_threshold` | 0.3 to 0.9: how sure the model's voice detector must be that it hears speech | |
138
+
139
+ ```python
140
+ settings = AirCapsLiveSTTService.Settings(end_of_turn_mode="balanced", min_turn_silence_ms=300)
141
+ ```
142
+
143
+ All of them can be changed with `STTUpdateSettingsFrame`; the session reopens to apply them.
144
+
145
+ Not supported by AirCaps Live 1: languages other than English, key terms, speaker labels.
146
+
147
+ ## Running the example
148
+
149
+ [`examples/voice-agent.py`](examples/voice-agent.py) is Pipecat's standard voice agent (OpenAI LLM, Cartesia voice)
150
+ with AirCaps as its speech-to-text.
151
+
152
+ ```bash
153
+ git clone https://github.com/aircaps/pipecat-aircaps.git
154
+ cd pipecat-aircaps
155
+ cp .env.example .env # add AIRCAPS_API_KEY, OPENAI_API_KEY, CARTESIA_API_KEY
156
+ uv sync --extra example
157
+ uv run examples/voice-agent.py
158
+ ```
159
+
160
+ Open http://localhost:7860/client and talk to it.
161
+
162
+ ## Tests
163
+
164
+ ```bash
165
+ uv sync
166
+ uv run pytest
167
+ ```
168
+
169
+ The tests run against a local fake of the AirCaps API; no key is needed.
170
+
171
+ ## Compatibility
172
+
173
+ Tested with Pipecat v1.16.0, on Python 3.11, 3.12 and 3.13.
174
+
175
+ ## Support
176
+
177
+ Open an [issue](https://github.com/aircaps/pipecat-aircaps/issues), or [contact AirCaps](https://aircaps.ai/contact).
178
+
179
+ ## License
180
+
181
+ BSD 2-Clause, see [LICENSE](LICENSE).
@@ -0,0 +1,7 @@
1
+ pipecat_aircaps/__init__.py,sha256=22JROnmZsIFACsOUTC3Z8FqZs7ElLvw7BeXVMoDvqSU,278
2
+ pipecat_aircaps/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
3
+ pipecat_aircaps/stt.py,sha256=UuTmKljA5j5H7vAJQpA9lrbcEGiqIEgHN4H78tFIQzc,23499
4
+ pipecat_aircaps-0.1.0.dist-info/METADATA,sha256=hDnkpi_4mwwfzXiZQMel9l4SK3w_nyXZrASyY7ZxRDE,7484
5
+ pipecat_aircaps-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
6
+ pipecat_aircaps-0.1.0.dist-info/licenses/LICENSE,sha256=aQhr5u0p3bXDNM6WHkKhdhBe4NdPXzvLZbBRXTAYphY,1296
7
+ pipecat_aircaps-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,24 @@
1
+ BSD 2-Clause License
2
+
3
+ Copyright (c) 2026, AirCaps
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
19
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
21
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
22
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
24
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.