pipecat-sprag 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipecat_sprag/__init__.py +1 -0
- pipecat_sprag/common/__init__.py +26 -0
- pipecat_sprag/llm/__init__.py +5 -0
- pipecat_sprag/llm/service.py +113 -0
- pipecat_sprag/py.typed +0 -0
- pipecat_sprag/stt/__init__.py +5 -0
- pipecat_sprag/stt/service.py +62 -0
- pipecat_sprag/tts/__init__.py +5 -0
- pipecat_sprag/tts/service.py +366 -0
- pipecat_sprag-0.1.0.dist-info/METADATA +88 -0
- pipecat_sprag-0.1.0.dist-info/RECORD +13 -0
- pipecat_sprag-0.1.0.dist-info/WHEEL +4 -0
- pipecat_sprag-0.1.0.dist-info/licenses/LICENSE +24 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Sprag speech and language models for Pipecat."""
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Configuration shared by the Sprag Pipecat services."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from importlib.metadata import version
|
|
5
|
+
|
|
6
|
+
__version__ = version("pipecat-sprag")
|
|
7
|
+
|
|
8
|
+
SPRAG_BASE_URL = "https://api.sprag.ai/v1"
|
|
9
|
+
SPRAG_TRANSCRIPTION_URL = "wss://api.sprag.ai/v1/audio/transcriptions"
|
|
10
|
+
SPRAG_SPEECH_URL = "wss://api.sprag.ai/v1/audio/speech"
|
|
11
|
+
ATTRIBUTION_HEADERS = {"X-Sprag-Integration": f"pipecat/{__version__}"}
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def resolve_api_key(api_key: str | None) -> str:
|
|
15
|
+
"""Return the given API key, or ``SPRAG_API_KEY`` from the environment.
|
|
16
|
+
|
|
17
|
+
Raises:
|
|
18
|
+
ValueError: If neither is set.
|
|
19
|
+
"""
|
|
20
|
+
resolved = api_key or os.environ.get("SPRAG_API_KEY")
|
|
21
|
+
if not resolved:
|
|
22
|
+
raise ValueError(
|
|
23
|
+
"SPRAG_API_KEY is required, either as the api_key argument or the "
|
|
24
|
+
"SPRAG_API_KEY environment variable"
|
|
25
|
+
)
|
|
26
|
+
return resolved
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Sprag's Symphony model as a Pipecat LLM service."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import AsyncIterator, Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from openai import AsyncStream
|
|
10
|
+
from openai.types.chat import ChatCompletionChunk
|
|
11
|
+
from openai.types.completion_usage import CompletionTokensDetails, PromptTokensDetails
|
|
12
|
+
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
13
|
+
from pipecat.services.openai.base_llm import OpenAILLMSettings
|
|
14
|
+
from pipecat.services.openai.llm import OpenAILLMService
|
|
15
|
+
|
|
16
|
+
from pipecat_sprag.common import ATTRIBUTION_HEADERS, SPRAG_BASE_URL, resolve_api_key
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class SpragLLMSettings(OpenAILLMSettings):
|
|
21
|
+
"""Settings for SpragLLMService."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SpragLLMService(OpenAILLMService):
|
|
25
|
+
"""Sprag's Symphony model through Sprag's OpenAI-compatible chat completions API.
|
|
26
|
+
|
|
27
|
+
Sprag reports usage per billing axis (``usage.input.text.tokens``,
|
|
28
|
+
``usage.input.audio.duration`` and so on) rather than as ``prompt_tokens`` and
|
|
29
|
+
``completion_tokens``. This service reports the text tokens in the fields Pipecat
|
|
30
|
+
reads, so usage metrics and function calls work. Media is billed on its own axes,
|
|
31
|
+
which Pipecat's token metrics have no place for; Sprag's full usage stays on the
|
|
32
|
+
chunk for anyone who needs it.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
Settings = SpragLLMSettings
|
|
36
|
+
_settings: SpragLLMSettings
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
*,
|
|
41
|
+
api_key: str | None = None,
|
|
42
|
+
base_url: str = SPRAG_BASE_URL,
|
|
43
|
+
default_headers: Mapping[str, str] | None = None,
|
|
44
|
+
settings: SpragLLMSettings | None = None,
|
|
45
|
+
**kwargs: Any,
|
|
46
|
+
) -> None:
|
|
47
|
+
"""Initialize the Sprag LLM service.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
api_key: Your Sprag API key. Defaults to the ``SPRAG_API_KEY`` environment
|
|
51
|
+
variable.
|
|
52
|
+
base_url: The Sprag API base URL.
|
|
53
|
+
default_headers: Extra headers sent with every request.
|
|
54
|
+
settings: Runtime-updatable settings. The model defaults to ``symphony``.
|
|
55
|
+
**kwargs: Additional arguments passed to OpenAILLMService.
|
|
56
|
+
"""
|
|
57
|
+
default_settings = self.Settings(model="symphony")
|
|
58
|
+
if settings is not None:
|
|
59
|
+
default_settings.apply_update(settings)
|
|
60
|
+
super().__init__(
|
|
61
|
+
api_key=resolve_api_key(api_key),
|
|
62
|
+
base_url=base_url,
|
|
63
|
+
default_headers={**ATTRIBUTION_HEADERS, **(default_headers or {})},
|
|
64
|
+
settings=default_settings,
|
|
65
|
+
**kwargs,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
async def get_chat_completions( # type: ignore[override]
|
|
69
|
+
self, context: LLMContext
|
|
70
|
+
) -> _OpenAIUsageStream:
|
|
71
|
+
"""Stream chat completions, with Sprag's usage mapped onto OpenAI's fields.
|
|
72
|
+
|
|
73
|
+
Args:
|
|
74
|
+
context: The context to complete.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
The chunk stream.
|
|
78
|
+
"""
|
|
79
|
+
return _OpenAIUsageStream(await super().get_chat_completions(context))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _OpenAIUsageStream:
|
|
83
|
+
def __init__(self, stream: AsyncStream[ChatCompletionChunk]) -> None:
|
|
84
|
+
self._stream = stream
|
|
85
|
+
|
|
86
|
+
def __aiter__(self) -> AsyncIterator[ChatCompletionChunk]:
|
|
87
|
+
return self._iterate()
|
|
88
|
+
|
|
89
|
+
async def _iterate(self) -> AsyncIterator[ChatCompletionChunk]:
|
|
90
|
+
async for chunk in self._stream:
|
|
91
|
+
yield _with_openai_usage(chunk)
|
|
92
|
+
|
|
93
|
+
async def close(self) -> None:
|
|
94
|
+
await self._stream.close()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _with_openai_usage(chunk: ChatCompletionChunk) -> ChatCompletionChunk:
|
|
98
|
+
usage = chunk.usage
|
|
99
|
+
if usage is None or usage.prompt_tokens is not None:
|
|
100
|
+
return chunk
|
|
101
|
+
extra = usage.model_extra or {}
|
|
102
|
+
text_in: dict[str, Any] = (extra.get("input") or {}).get("text") or {}
|
|
103
|
+
text_out: dict[str, Any] = (extra.get("output") or {}).get("text") or {}
|
|
104
|
+
usage.prompt_tokens = text_in.get("tokens") or 0
|
|
105
|
+
usage.completion_tokens = text_out.get("tokens") or 0
|
|
106
|
+
usage.total_tokens = usage.prompt_tokens + usage.completion_tokens
|
|
107
|
+
if text_in.get("cached_tokens"):
|
|
108
|
+
usage.prompt_tokens_details = PromptTokensDetails(cached_tokens=text_in["cached_tokens"])
|
|
109
|
+
if text_out.get("reasoning_tokens"):
|
|
110
|
+
usage.completion_tokens_details = CompletionTokensDetails(
|
|
111
|
+
reasoning_tokens=text_out["reasoning_tokens"]
|
|
112
|
+
)
|
|
113
|
+
return chunk
|
pipecat_sprag/py.typed
ADDED
|
File without changes
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Sprag's Rhythm model as a Pipecat speech-to-text service."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import Any, Literal
|
|
7
|
+
|
|
8
|
+
from pipecat.services.openai.stt import OpenAIRealtimeSTTService, OpenAIRealtimeSTTSettings
|
|
9
|
+
|
|
10
|
+
from pipecat_sprag.common import SPRAG_TRANSCRIPTION_URL, resolve_api_key
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass
|
|
14
|
+
class SpragSTTSettings(OpenAIRealtimeSTTSettings):
|
|
15
|
+
"""Settings for SpragSTTService."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SpragSTTService(OpenAIRealtimeSTTService):
|
|
19
|
+
"""Streaming transcription with Rhythm over Sprag's streaming transcription socket.
|
|
20
|
+
|
|
21
|
+
By default the pipeline's own VAD decides when a turn ends. Pass
|
|
22
|
+
``turn_detection={"type": "server_vad"}`` to have Sprag detect turns instead.
|
|
23
|
+
|
|
24
|
+
Unlike the LLM and TTS services, it sends no ``X-Sprag-Integration`` header: Pipecat's
|
|
25
|
+
realtime STT connects with a fixed set of headers.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
Settings = SpragSTTSettings
|
|
29
|
+
_settings: SpragSTTSettings
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
*,
|
|
34
|
+
api_key: str | None = None,
|
|
35
|
+
base_url: str = SPRAG_TRANSCRIPTION_URL,
|
|
36
|
+
turn_detection: dict[str, Any] | Literal[False] | None = False,
|
|
37
|
+
settings: SpragSTTSettings | None = None,
|
|
38
|
+
**kwargs: Any,
|
|
39
|
+
) -> None:
|
|
40
|
+
"""Initialize the Sprag STT service.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
api_key: Your Sprag API key. Defaults to the ``SPRAG_API_KEY`` environment
|
|
44
|
+
variable.
|
|
45
|
+
base_url: The Sprag streaming transcription WebSocket URL.
|
|
46
|
+
turn_detection: ``False`` (the default) leaves turn detection to the
|
|
47
|
+
pipeline's VAD. A dict such as ``{"type": "server_vad"}`` enables
|
|
48
|
+
Sprag's server-side VAD.
|
|
49
|
+
settings: Runtime-updatable settings. The model defaults to ``rhythm``.
|
|
50
|
+
**kwargs: Additional arguments passed to OpenAIRealtimeSTTService.
|
|
51
|
+
"""
|
|
52
|
+
# Rhythm takes no language parameter
|
|
53
|
+
default_settings = self.Settings(model="rhythm", language=None)
|
|
54
|
+
if settings is not None:
|
|
55
|
+
default_settings.apply_update(settings)
|
|
56
|
+
super().__init__(
|
|
57
|
+
api_key=resolve_api_key(api_key),
|
|
58
|
+
base_url=base_url,
|
|
59
|
+
turn_detection=turn_detection,
|
|
60
|
+
settings=default_settings,
|
|
61
|
+
**kwargs,
|
|
62
|
+
)
|
|
@@ -0,0 +1,366 @@
|
|
|
1
|
+
"""Sprag's Chorus models as a Pipecat text-to-speech service."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import base64
|
|
7
|
+
import json
|
|
8
|
+
import uuid
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from collections.abc import AsyncGenerator
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import Any
|
|
13
|
+
from urllib.parse import urlencode, urlsplit, urlunsplit
|
|
14
|
+
|
|
15
|
+
from pipecat.frames.frames import (
|
|
16
|
+
ErrorFrame,
|
|
17
|
+
Frame,
|
|
18
|
+
StartFrame,
|
|
19
|
+
TTSAudioRawFrame,
|
|
20
|
+
TTSStoppedFrame,
|
|
21
|
+
)
|
|
22
|
+
from pipecat.services.settings import TTSSettings
|
|
23
|
+
from pipecat.services.tts_service import InterruptibleTTSService
|
|
24
|
+
from websockets.asyncio.client import ClientConnection
|
|
25
|
+
from websockets.protocol import State
|
|
26
|
+
|
|
27
|
+
from pipecat_sprag.common import ATTRIBUTION_HEADERS, SPRAG_SPEECH_URL, resolve_api_key
|
|
28
|
+
|
|
29
|
+
SAMPLE_RATE = 24000
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class SpragTTSSettings(TTSSettings):
|
|
34
|
+
"""Settings for SpragTTSService."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class SpragTTSService(InterruptibleTTSService):
|
|
38
|
+
"""Streaming speech with Sprag's Chorus models over Sprag's streaming speech socket.
|
|
39
|
+
|
|
40
|
+
Each sentence is spoken as its own response, sent as soon as Pipecat hands it over;
|
|
41
|
+
the session queues responses and speaks them in order. Each response carries its
|
|
42
|
+
turn's context id as metadata, which routes its audio back to that turn. Sprag
|
|
43
|
+
cannot stop a response once it has started, so an interruption replaces the
|
|
44
|
+
connection rather than cancelling. Sentences sent on a connection that drops are
|
|
45
|
+
lost; sentences that arrive while it reconnects are spoken on the new one.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
Settings = SpragTTSSettings
|
|
49
|
+
_settings: SpragTTSSettings
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
*,
|
|
54
|
+
api_key: str | None = None,
|
|
55
|
+
url: str = SPRAG_SPEECH_URL,
|
|
56
|
+
connect_timeout: float = 10.0,
|
|
57
|
+
settings: SpragTTSSettings | None = None,
|
|
58
|
+
**kwargs: Any,
|
|
59
|
+
) -> None:
|
|
60
|
+
"""Initialize the Sprag TTS service.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
api_key: Your Sprag API key. Defaults to the ``SPRAG_API_KEY`` environment
|
|
64
|
+
variable.
|
|
65
|
+
url: The Sprag streaming speech WebSocket URL.
|
|
66
|
+
connect_timeout: Seconds the session may take to be configured.
|
|
67
|
+
settings: Runtime-updatable settings. Defaults to the ``chorus`` model
|
|
68
|
+
and the ``wade`` voice.
|
|
69
|
+
**kwargs: Additional arguments passed to InterruptibleTTSService.
|
|
70
|
+
"""
|
|
71
|
+
default_settings = self.Settings(model="chorus", voice="wade", language=None)
|
|
72
|
+
if settings is not None:
|
|
73
|
+
default_settings.apply_update(settings)
|
|
74
|
+
super().__init__(
|
|
75
|
+
push_start_frame=True,
|
|
76
|
+
push_stop_frames=True,
|
|
77
|
+
pause_frame_processing=True,
|
|
78
|
+
sample_rate=SAMPLE_RATE,
|
|
79
|
+
settings=default_settings,
|
|
80
|
+
**kwargs,
|
|
81
|
+
)
|
|
82
|
+
self._api_key = resolve_api_key(api_key)
|
|
83
|
+
self._url = url
|
|
84
|
+
self._connect_timeout = connect_timeout
|
|
85
|
+
self._receive_task: asyncio.Task[None] | None = None
|
|
86
|
+
self._unfinished: Counter[str] = Counter()
|
|
87
|
+
self._flushed: set[str] = set()
|
|
88
|
+
self._unstarted: dict[str, tuple[str, str]] = {}
|
|
89
|
+
self._held: list[tuple[str, str]] = []
|
|
90
|
+
self._speaking_context: str | None = None
|
|
91
|
+
# run_tts and the receive loop's reconnect may both reach for a new socket
|
|
92
|
+
self._connect_lock = asyncio.Lock()
|
|
93
|
+
|
|
94
|
+
def can_generate_metrics(self) -> bool:
|
|
95
|
+
"""Report that this service emits TTFB and usage metrics.
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
True.
|
|
99
|
+
"""
|
|
100
|
+
return True
|
|
101
|
+
|
|
102
|
+
async def start(self, frame: StartFrame) -> None:
|
|
103
|
+
"""Connect when the pipeline starts.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
frame: The start frame.
|
|
107
|
+
"""
|
|
108
|
+
await super().start(frame)
|
|
109
|
+
await self._connect()
|
|
110
|
+
|
|
111
|
+
async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]: # type: ignore[override]
|
|
112
|
+
changed = await super()._update_settings(delta)
|
|
113
|
+
if changed:
|
|
114
|
+
# model and voice are fixed for the life of a session
|
|
115
|
+
await self._disconnect()
|
|
116
|
+
await self._connect()
|
|
117
|
+
return changed
|
|
118
|
+
|
|
119
|
+
async def _connect(self) -> None:
|
|
120
|
+
await super()._connect()
|
|
121
|
+
await self._connect_websocket()
|
|
122
|
+
if self._websocket and (not self._receive_task or self._receive_task.done()):
|
|
123
|
+
self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
|
|
124
|
+
|
|
125
|
+
async def _disconnect(self) -> None:
|
|
126
|
+
await super()._disconnect()
|
|
127
|
+
if self._receive_task:
|
|
128
|
+
await self.cancel_task(self._receive_task, timeout=1.0)
|
|
129
|
+
self._receive_task = None
|
|
130
|
+
await self._disconnect_websocket()
|
|
131
|
+
|
|
132
|
+
async def _connect_websocket(self) -> None:
|
|
133
|
+
async with self._connect_lock:
|
|
134
|
+
await self._open_session()
|
|
135
|
+
|
|
136
|
+
async def _open_session(self) -> None:
|
|
137
|
+
if self._websocket and self._websocket.state is State.OPEN:
|
|
138
|
+
return
|
|
139
|
+
if self._websocket is not None:
|
|
140
|
+
# closed without passing through _disconnect_websocket, e.g. with no reconnect
|
|
141
|
+
await self._strand()
|
|
142
|
+
self._websocket = None
|
|
143
|
+
parts = urlsplit(self._url)
|
|
144
|
+
query = "&".join(q for q in (parts.query, urlencode({"model": self._settings.model})) if q)
|
|
145
|
+
# an interrupted turn's held sentences are dropped
|
|
146
|
+
waiting, self._held = self._held, []
|
|
147
|
+
unsent = []
|
|
148
|
+
for context_id, text in waiting:
|
|
149
|
+
if self.audio_context_available(context_id):
|
|
150
|
+
unsent.append((context_id, text))
|
|
151
|
+
else:
|
|
152
|
+
await self._finish_response(context_id)
|
|
153
|
+
websocket: ClientConnection | None = None
|
|
154
|
+
try:
|
|
155
|
+
websocket = await self._websocket_connect(
|
|
156
|
+
urlunsplit(parts._replace(query=query)),
|
|
157
|
+
additional_headers={
|
|
158
|
+
"Authorization": f"Bearer {self._api_key}",
|
|
159
|
+
**ATTRIBUTION_HEADERS,
|
|
160
|
+
},
|
|
161
|
+
)
|
|
162
|
+
await asyncio.wait_for(
|
|
163
|
+
self._configure_session(websocket), timeout=self._connect_timeout
|
|
164
|
+
)
|
|
165
|
+
# sent before the socket is visible, so they keep their place ahead of new sentences
|
|
166
|
+
while unsent:
|
|
167
|
+
context_id, text = unsent.pop(0)
|
|
168
|
+
await self._request(websocket, context_id, text)
|
|
169
|
+
except BaseException as e:
|
|
170
|
+
if websocket:
|
|
171
|
+
await websocket.close()
|
|
172
|
+
for context_id, _ in self._unstarted.values():
|
|
173
|
+
await self._finish_response(context_id)
|
|
174
|
+
self._unstarted.clear()
|
|
175
|
+
self._held = unsent + self._held
|
|
176
|
+
if not isinstance(e, Exception):
|
|
177
|
+
raise
|
|
178
|
+
await self.push_error(error_msg=f"Error connecting to Sprag TTS: {e}", exception=e)
|
|
179
|
+
return
|
|
180
|
+
# run_tts may send as soon as the socket is visible, so it becomes visible configured
|
|
181
|
+
self._websocket = websocket
|
|
182
|
+
await self._call_event_handler("on_connected")
|
|
183
|
+
|
|
184
|
+
async def _configure_session(self, websocket: ClientConnection) -> None:
|
|
185
|
+
session = {"type": "realtime", "audio": {"output": {"voice": self._settings.voice}}}
|
|
186
|
+
await websocket.send(json.dumps({"type": "session.update", "session": session}))
|
|
187
|
+
async for message in websocket:
|
|
188
|
+
event = json.loads(message)
|
|
189
|
+
if event.get("type") == "session.updated":
|
|
190
|
+
return
|
|
191
|
+
if event.get("type") == "error":
|
|
192
|
+
raise RuntimeError(_error_message(event))
|
|
193
|
+
raise ConnectionError("the session closed before it was configured")
|
|
194
|
+
|
|
195
|
+
async def _disconnect_websocket(self) -> None:
|
|
196
|
+
try:
|
|
197
|
+
if self._websocket:
|
|
198
|
+
await self._websocket.close()
|
|
199
|
+
finally:
|
|
200
|
+
self._websocket = None
|
|
201
|
+
await self._strand()
|
|
202
|
+
await self._call_event_handler("on_disconnected")
|
|
203
|
+
|
|
204
|
+
async def _strand(self) -> None:
|
|
205
|
+
stranded = [context_id for context_id, _ in self._unstarted.values()]
|
|
206
|
+
self._unstarted.clear()
|
|
207
|
+
speaking, self._speaking_context = self._speaking_context, None
|
|
208
|
+
if speaking is not None:
|
|
209
|
+
stranded.append(speaking)
|
|
210
|
+
for context_id in stranded:
|
|
211
|
+
await self._finish_response(context_id)
|
|
212
|
+
|
|
213
|
+
async def flush_audio(self, context_id: str | None = None) -> None:
|
|
214
|
+
"""Close the turn's audio context once its last response has finished.
|
|
215
|
+
|
|
216
|
+
Args:
|
|
217
|
+
context_id: The turn's audio context.
|
|
218
|
+
"""
|
|
219
|
+
if not context_id:
|
|
220
|
+
return
|
|
221
|
+
if self._unfinished[context_id]:
|
|
222
|
+
self._flushed.add(context_id)
|
|
223
|
+
else:
|
|
224
|
+
await self._close_context(context_id)
|
|
225
|
+
|
|
226
|
+
async def _close_context(self, context_id: str) -> None:
|
|
227
|
+
if self.audio_context_available(context_id):
|
|
228
|
+
await self.append_to_audio_context(context_id, TTSStoppedFrame(context_id=context_id))
|
|
229
|
+
await self.remove_audio_context(context_id)
|
|
230
|
+
|
|
231
|
+
async def _receive_messages(self) -> None:
|
|
232
|
+
assert self._websocket is not None
|
|
233
|
+
async for message in self._websocket:
|
|
234
|
+
if not isinstance(message, str):
|
|
235
|
+
continue
|
|
236
|
+
event = json.loads(message)
|
|
237
|
+
event_type = event.get("type")
|
|
238
|
+
if event_type == "response.created":
|
|
239
|
+
self._on_response_created(event)
|
|
240
|
+
elif event_type == "response.output_audio.delta":
|
|
241
|
+
await self._on_audio(event)
|
|
242
|
+
elif event_type == "response.done":
|
|
243
|
+
await self._on_response_done(event)
|
|
244
|
+
elif event_type == "error":
|
|
245
|
+
await self._on_error(event)
|
|
246
|
+
|
|
247
|
+
def _on_response_created(self, event: dict[str, Any]) -> None:
|
|
248
|
+
context_id = _context_of(event)
|
|
249
|
+
self._speaking_context = context_id
|
|
250
|
+
# responses start in the order they were asked for, so this answers the turn's oldest
|
|
251
|
+
for event_id, (owner, _) in self._unstarted.items():
|
|
252
|
+
if owner == context_id:
|
|
253
|
+
del self._unstarted[event_id]
|
|
254
|
+
break
|
|
255
|
+
|
|
256
|
+
async def _on_audio(self, event: dict[str, Any]) -> None:
|
|
257
|
+
context_id = self._speaking_context
|
|
258
|
+
if context_id is None or not self.audio_context_available(context_id):
|
|
259
|
+
return # an interrupted turn's queued responses still play out on the session
|
|
260
|
+
await self.stop_ttfb_metrics()
|
|
261
|
+
frame = TTSAudioRawFrame(
|
|
262
|
+
audio=base64.b64decode(event["delta"]),
|
|
263
|
+
sample_rate=self.sample_rate,
|
|
264
|
+
num_channels=1,
|
|
265
|
+
context_id=context_id,
|
|
266
|
+
)
|
|
267
|
+
await self.append_to_audio_context(context_id, frame)
|
|
268
|
+
|
|
269
|
+
async def _on_response_done(self, event: dict[str, Any]) -> None:
|
|
270
|
+
response = event.get("response") or {}
|
|
271
|
+
if response.get("status") != "completed":
|
|
272
|
+
error = (response.get("status_details") or {}).get("error") or {}
|
|
273
|
+
await self.push_error(
|
|
274
|
+
error_msg=f"Sprag TTS response ended as {response.get('status')}: "
|
|
275
|
+
f"{error.get('code', 'unknown')}"
|
|
276
|
+
)
|
|
277
|
+
self._speaking_context = None
|
|
278
|
+
context_id = _context_of(event)
|
|
279
|
+
if context_id is not None:
|
|
280
|
+
await self._finish_response(context_id)
|
|
281
|
+
|
|
282
|
+
async def _on_error(self, event: dict[str, Any]) -> None:
|
|
283
|
+
await self.push_error(error_msg=_error_message(event))
|
|
284
|
+
# a refused response.create opens no response
|
|
285
|
+
refused = self._unstarted.pop((event.get("error") or {}).get("event_id") or "", None)
|
|
286
|
+
if refused is not None:
|
|
287
|
+
await self._finish_response(refused[0])
|
|
288
|
+
|
|
289
|
+
async def _finish_response(self, context_id: str) -> None:
|
|
290
|
+
self._unfinished[context_id] -= 1
|
|
291
|
+
if self._unfinished[context_id] > 0:
|
|
292
|
+
return
|
|
293
|
+
del self._unfinished[context_id]
|
|
294
|
+
if context_id in self._flushed:
|
|
295
|
+
self._flushed.discard(context_id)
|
|
296
|
+
await self._close_context(context_id)
|
|
297
|
+
|
|
298
|
+
async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame | None, None]:
|
|
299
|
+
"""Speak one sentence as a response on the session.
|
|
300
|
+
|
|
301
|
+
Args:
|
|
302
|
+
text: The sentence to speak.
|
|
303
|
+
context_id: The turn's audio context.
|
|
304
|
+
|
|
305
|
+
Yields:
|
|
306
|
+
Nothing directly; audio arrives on the receive task.
|
|
307
|
+
"""
|
|
308
|
+
reconnecting = self._receive_task is not None and not self._receive_task.done()
|
|
309
|
+
if self._websocket and self._websocket.state is not State.OPEN and reconnecting:
|
|
310
|
+
# the receive loop's reconnect sends it once the new session is configured
|
|
311
|
+
self._unfinished[context_id] += 1
|
|
312
|
+
self._held.append((context_id, text))
|
|
313
|
+
await self.start_tts_usage_metrics(text)
|
|
314
|
+
yield None
|
|
315
|
+
return
|
|
316
|
+
if not self._websocket or self._websocket.state is not State.OPEN:
|
|
317
|
+
await self._connect()
|
|
318
|
+
if not self._websocket:
|
|
319
|
+
yield ErrorFrame(error="Sprag TTS is not connected")
|
|
320
|
+
return
|
|
321
|
+
self._unfinished[context_id] += 1
|
|
322
|
+
websocket = self._websocket
|
|
323
|
+
try:
|
|
324
|
+
await self._request(websocket, context_id, text)
|
|
325
|
+
except Exception as e:
|
|
326
|
+
yield ErrorFrame(error=f"Sprag TTS send error: {e}")
|
|
327
|
+
return
|
|
328
|
+
await self.start_tts_usage_metrics(text)
|
|
329
|
+
yield None
|
|
330
|
+
|
|
331
|
+
async def _request(self, websocket: ClientConnection, context_id: str, text: str) -> None:
|
|
332
|
+
item = {
|
|
333
|
+
"type": "message",
|
|
334
|
+
"role": "user",
|
|
335
|
+
"content": [{"type": "input_text", "text": text}],
|
|
336
|
+
}
|
|
337
|
+
event_id = f"evt_{uuid.uuid4().hex}"
|
|
338
|
+
create = {
|
|
339
|
+
"type": "response.create",
|
|
340
|
+
"event_id": event_id,
|
|
341
|
+
"response": {"metadata": {_CONTEXT_KEY: context_id}},
|
|
342
|
+
}
|
|
343
|
+
# recorded first: the response can start before the send returns
|
|
344
|
+
self._unstarted[event_id] = (context_id, text)
|
|
345
|
+
try:
|
|
346
|
+
await websocket.send(json.dumps({"type": "conversation.item.create", "item": item}))
|
|
347
|
+
await websocket.send(json.dumps(create))
|
|
348
|
+
except Exception:
|
|
349
|
+
# a drop noticed mid-send has already released it
|
|
350
|
+
if self._unstarted.pop(event_id, None) is not None:
|
|
351
|
+
await self._finish_response(context_id)
|
|
352
|
+
raise
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
_CONTEXT_KEY = "context_id"
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _context_of(event: dict[str, Any]) -> str | None:
|
|
359
|
+
metadata = (event.get("response") or {}).get("metadata") or {}
|
|
360
|
+
context_id = metadata.get(_CONTEXT_KEY)
|
|
361
|
+
return context_id if isinstance(context_id, str) else None
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _error_message(event: dict[str, Any]) -> str:
|
|
365
|
+
error = event.get("error") or {}
|
|
366
|
+
return f"Sprag TTS error [{error.get('code', 'unknown')}]: {error.get('message', '')}"
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pipecat-sprag
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Sprag speech and language models for Pipecat: Rhythm STT, Symphony LLM, and Chorus TTS
|
|
5
|
+
Project-URL: Homepage, https://sprag.ai
|
|
6
|
+
Project-URL: Source, https://github.com/sprag-ai/pipecat-sprag
|
|
7
|
+
Project-URL: Changelog, https://github.com/sprag-ai/pipecat-sprag/blob/main/CHANGELOG.md
|
|
8
|
+
Author: Sprag AI Inc.
|
|
9
|
+
License-Expression: BSD-2-Clause
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: ai,llm,pipecat,sprag,stt,tts,voice
|
|
12
|
+
Requires-Python: >=3.11
|
|
13
|
+
Requires-Dist: pipecat-ai<2,>=1.8
|
|
14
|
+
Description-Content-Type: text/markdown
|
|
15
|
+
|
|
16
|
+
# pipecat-sprag
|
|
17
|
+
|
|
18
|
+
[Sprag](https://sprag.ai/) speech and language models for [Pipecat](https://github.com/pipecat-ai/pipecat).
|
|
19
|
+
|
|
20
|
+
This integration is maintained by Sprag.
|
|
21
|
+
|
|
22
|
+
| Service | Import | Model |
|
|
23
|
+
| --- | --- | --- |
|
|
24
|
+
| `SpragSTTService` | `pipecat_sprag.stt` | Rhythm (speech to text) |
|
|
25
|
+
| `SpragLLMService` | `pipecat_sprag.llm` | Symphony (LLM) |
|
|
26
|
+
| `SpragTTSService` | `pipecat_sprag.tts` | Chorus (text to speech) |
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
uv add pipecat-sprag
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Each service reads your API key from the `SPRAG_API_KEY` environment variable, or takes it as `api_key=`. You can generate a key from [API keys](https://sprag.ai/app/api-keys) in the Sprag app.
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
Each service drops into a Pipecat pipeline in place of any other STT, LLM, or TTS service:
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from pipecat_sprag.llm import SpragLLMService
|
|
42
|
+
from pipecat_sprag.stt import SpragSTTService
|
|
43
|
+
from pipecat_sprag.tts import SpragTTSService
|
|
44
|
+
|
|
45
|
+
stt = SpragSTTService()
|
|
46
|
+
llm = SpragLLMService()
|
|
47
|
+
tts = SpragTTSService(settings=SpragTTSService.Settings(voice="wade"))
|
|
48
|
+
|
|
49
|
+
pipeline = Pipeline(
|
|
50
|
+
[transport.input(), stt, user_aggregator, llm, tts, transport.output(), assistant_aggregator]
|
|
51
|
+
)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
- **`SpragSTTService`** streams audio to Rhythm over Sprag's streaming transcription socket (`wss://api.sprag.ai/v1/audio/transcriptions`). By default your pipeline's VAD decides when a turn ends; pass `turn_detection={"type": "server_vad"}` to let Sprag detect turns instead.
|
|
55
|
+
- **`SpragLLMService`** runs Symphony through Sprag's chat completions API, with function calling and token usage metrics.
|
|
56
|
+
- **`SpragTTSService`** streams Chorus speech over Sprag's streaming speech socket (`wss://api.sprag.ai/v1/audio/speech`), one sentence at a time. It defaults to the `chorus` model and the `wade` voice; the available voices are listed in [voice presets](https://sprag.ai/docs/tts/voices/presets).
|
|
57
|
+
|
|
58
|
+
Each service takes `settings=<Service>.Settings(...)` for its model and other options, and `api_key=` if you don't use the `SPRAG_API_KEY` environment variable.
|
|
59
|
+
|
|
60
|
+
## Example
|
|
61
|
+
|
|
62
|
+
[`examples/voice_agent.py`](https://github.com/sprag-ai/pipecat-sprag/blob/main/examples/voice_agent.py) is a complete voice agent running on Rhythm, Symphony, and Chorus. It uses Pipecat's development runner:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
uv sync --group examples
|
|
66
|
+
SPRAG_API_KEY=... uv run examples/voice_agent.py
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Then open the printed URL in your browser and talk to it.
|
|
70
|
+
|
|
71
|
+
## Compatibility
|
|
72
|
+
|
|
73
|
+
Requires `pipecat-ai>=1.8,<2`. Tested with Pipecat v1.12.0.
|
|
74
|
+
|
|
75
|
+
## Development
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
uv sync
|
|
79
|
+
uv run ruff check && uv run ruff format --check
|
|
80
|
+
uv run mypy src
|
|
81
|
+
uv run pytest
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Live tests call api.sprag.ai and are skipped by default: `SPRAG_API_KEY=... uv run pytest -m live`.
|
|
85
|
+
|
|
86
|
+
## License
|
|
87
|
+
|
|
88
|
+
BSD 2-Clause. See [LICENSE](https://github.com/sprag-ai/pipecat-sprag/blob/main/LICENSE).
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
pipecat_sprag/__init__.py,sha256=jj2K0aFVnG84gV3ejUynpqi-17Y-PSiq8QXsYLtQMkI,52
|
|
2
|
+
pipecat_sprag/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
pipecat_sprag/common/__init__.py,sha256=0NNFzV3i8kMxiAr9tr7Be21AfeG4-Tmwqm50yKK0JDU,842
|
|
4
|
+
pipecat_sprag/llm/__init__.py,sha256=yeeXswaYPrHgANdEbSFD-T1tJL4NK0Bw18sRn-Cp_Vk,161
|
|
5
|
+
pipecat_sprag/llm/service.py,sha256=F8K-ymsQAvXyxbBifyZ76SdXuCFssMwAGfVO7ERazoY,4246
|
|
6
|
+
pipecat_sprag/stt/__init__.py,sha256=jGqgnxSc20aEw2cd_9yEFnNTET32LypKyv5hcRK2HL4,161
|
|
7
|
+
pipecat_sprag/stt/service.py,sha256=IN5D_9dog7s5M3MoXMLMenwqWTMvk4YnSrzDUec_dA0,2281
|
|
8
|
+
pipecat_sprag/tts/__init__.py,sha256=qMGkkp7nXpdI4hDtgKQxlXsV73S-UMB1lT_aWEDU5LU,161
|
|
9
|
+
pipecat_sprag/tts/service.py,sha256=vP6vZZHvHklzQDEEz5T14DfLtH_Qwwf2x4s6nGtJYUw,14657
|
|
10
|
+
pipecat_sprag-0.1.0.dist-info/METADATA,sha256=XZuiKZMlh3isIJMxjrv3FUs1vjxjqrNm6QQVgRIK5MM,3354
|
|
11
|
+
pipecat_sprag-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
12
|
+
pipecat_sprag-0.1.0.dist-info/licenses/LICENSE,sha256=Rxht0FLOjOvZJpEgMfaL9cpa0abj13KIs3gW1y2n2OY,1302
|
|
13
|
+
pipecat_sprag-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
BSD 2-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Sprag AI Inc.
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
16
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
17
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
18
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
19
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
20
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
21
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
22
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
23
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
24
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|