cartesia 1.0.12__py2.py3-none-any.whl → 1.0.14__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cartesia/__init__.py +2 -1
- cartesia/_async_sse.py +105 -0
- cartesia/_async_websocket.py +323 -0
- cartesia/_constants.py +10 -0
- cartesia/_logger.py +3 -0
- cartesia/_sse.py +152 -0
- cartesia/_types.py +3 -2
- cartesia/_websocket.py +374 -0
- cartesia/async_client.py +82 -0
- cartesia/async_tts.py +22 -0
- cartesia/client.py +4 -1325
- cartesia/resource.py +44 -0
- cartesia/tts.py +109 -0
- cartesia/utils/tts.py +25 -0
- cartesia/version.py +1 -1
- cartesia/voices.py +170 -0
- {cartesia-1.0.12.dist-info → cartesia-1.0.14.dist-info}/METADATA +14 -10
- cartesia-1.0.14.dist-info/RECORD +24 -0
- cartesia-1.0.12.dist-info/RECORD +0 -12
- {cartesia-1.0.12.dist-info → cartesia-1.0.14.dist-info}/LICENSE.md +0 -0
- {cartesia-1.0.12.dist-info → cartesia-1.0.14.dist-info}/WHEEL +0 -0
- {cartesia-1.0.12.dist-info → cartesia-1.0.14.dist-info}/top_level.txt +0 -0
cartesia/client.py
CHANGED
@@ -1,59 +1,10 @@
|
|
1
|
-
import asyncio
|
2
|
-
import base64
|
3
|
-
import json
|
4
|
-
import logging
|
5
1
|
import os
|
6
|
-
import uuid
|
7
|
-
from collections import defaultdict
|
8
2
|
from types import TracebackType
|
9
|
-
from typing import
|
10
|
-
Any,
|
11
|
-
AsyncGenerator,
|
12
|
-
Callable,
|
13
|
-
Dict,
|
14
|
-
Generator,
|
15
|
-
Iterator,
|
16
|
-
List,
|
17
|
-
Optional,
|
18
|
-
Set,
|
19
|
-
Tuple,
|
20
|
-
Union,
|
21
|
-
)
|
3
|
+
from typing import Optional, Union
|
22
4
|
|
23
|
-
import
|
24
|
-
import
|
25
|
-
import
|
26
|
-
|
27
|
-
try:
|
28
|
-
from websockets.sync.client import connect
|
29
|
-
|
30
|
-
IS_WEBSOCKET_SYNC_AVAILABLE = True
|
31
|
-
except ImportError:
|
32
|
-
IS_WEBSOCKET_SYNC_AVAILABLE = False
|
33
|
-
|
34
|
-
from iterators import TimeoutIterator
|
35
|
-
|
36
|
-
from cartesia._types import (
|
37
|
-
DeprecatedOutputFormatMapping,
|
38
|
-
EventType,
|
39
|
-
OutputFormat,
|
40
|
-
OutputFormatMapping,
|
41
|
-
VoiceControls,
|
42
|
-
VoiceMetadata,
|
43
|
-
)
|
44
|
-
from cartesia.utils.retry import retry_on_connection_error, retry_on_connection_error_async
|
45
|
-
|
46
|
-
DEFAULT_MODEL_ID = "sonic-english" # latest default model
|
47
|
-
MULTILINGUAL_MODEL_ID = "sonic-multilingual" # latest multilingual model
|
48
|
-
DEFAULT_BASE_URL = "api.cartesia.ai"
|
49
|
-
DEFAULT_CARTESIA_VERSION = "2024-06-10" # latest version
|
50
|
-
DEFAULT_TIMEOUT = 30 # seconds
|
51
|
-
DEFAULT_NUM_CONNECTIONS = 10 # connections per client
|
52
|
-
|
53
|
-
BACKOFF_FACTOR = 1
|
54
|
-
MAX_RETRIES = 3
|
55
|
-
|
56
|
-
logger = logging.getLogger(__name__)
|
5
|
+
from cartesia._constants import DEFAULT_BASE_URL, DEFAULT_TIMEOUT
|
6
|
+
from cartesia.tts import TTS
|
7
|
+
from cartesia.voices import Voices
|
57
8
|
|
58
9
|
|
59
10
|
class BaseClient:
|
@@ -74,49 +25,6 @@ class BaseClient:
|
|
74
25
|
return self._base_url
|
75
26
|
|
76
27
|
|
77
|
-
class Resource:
|
78
|
-
def __init__(
|
79
|
-
self,
|
80
|
-
api_key: str,
|
81
|
-
base_url: str,
|
82
|
-
timeout: float,
|
83
|
-
):
|
84
|
-
"""Constructor for the Resource class. Used by the Voices and TTS classes."""
|
85
|
-
self.api_key = api_key
|
86
|
-
self.timeout = timeout
|
87
|
-
self._base_url = base_url
|
88
|
-
self.cartesia_version = DEFAULT_CARTESIA_VERSION
|
89
|
-
self.headers = {
|
90
|
-
"X-API-Key": self.api_key,
|
91
|
-
"Cartesia-Version": self.cartesia_version,
|
92
|
-
"Content-Type": "application/json",
|
93
|
-
}
|
94
|
-
|
95
|
-
@property
|
96
|
-
def base_url(self):
|
97
|
-
return self._base_url
|
98
|
-
|
99
|
-
def _http_url(self):
|
100
|
-
"""Returns the HTTP URL for the Cartesia API.
|
101
|
-
If the base URL is localhost, the URL will start with 'http'. Otherwise, it will start with 'https'.
|
102
|
-
"""
|
103
|
-
if self._base_url.startswith("http://") or self._base_url.startswith("https://"):
|
104
|
-
return self._base_url
|
105
|
-
else:
|
106
|
-
prefix = "http" if "localhost" in self._base_url else "https"
|
107
|
-
return f"{prefix}://{self._base_url}"
|
108
|
-
|
109
|
-
def _ws_url(self):
|
110
|
-
"""Returns the WebSocket URL for the Cartesia API.
|
111
|
-
If the base URL is localhost, the URL will start with 'ws'. Otherwise, it will start with 'wss'.
|
112
|
-
"""
|
113
|
-
if self._base_url.startswith("ws://") or self._base_url.startswith("wss://"):
|
114
|
-
return self._base_url
|
115
|
-
else:
|
116
|
-
prefix = "ws" if "localhost" in self._base_url else "wss"
|
117
|
-
return f"{prefix}://{self._base_url}"
|
118
|
-
|
119
|
-
|
120
28
|
class Cartesia(BaseClient):
|
121
29
|
"""
|
122
30
|
The client for Cartesia's text-to-speech library.
|
@@ -159,1232 +67,3 @@ class Cartesia(BaseClient):
|
|
159
67
|
exc_tb: Union[TracebackType, None],
|
160
68
|
):
|
161
69
|
pass
|
162
|
-
|
163
|
-
|
164
|
-
class Voices(Resource):
|
165
|
-
"""This resource contains methods to list, get, clone, and create voices in your Cartesia voice library.
|
166
|
-
|
167
|
-
Usage:
|
168
|
-
>>> client = Cartesia(api_key="your_api_key")
|
169
|
-
>>> voices = client.voices.list()
|
170
|
-
>>> voice = client.voices.get(id="a0e99841-438c-4a64-b679-ae501e7d6091")
|
171
|
-
>>> print("Voice Name:", voice["name"], "Voice Description:", voice["description"])
|
172
|
-
>>> embedding = client.voices.clone(filepath="path/to/clip.wav")
|
173
|
-
>>> new_voice = client.voices.create(
|
174
|
-
... name="My Voice", description="A new voice", embedding=embedding
|
175
|
-
... )
|
176
|
-
"""
|
177
|
-
|
178
|
-
def list(self) -> List[VoiceMetadata]:
|
179
|
-
"""List all voices in your voice library.
|
180
|
-
|
181
|
-
Returns:
|
182
|
-
This method returns a list of VoiceMetadata objects.
|
183
|
-
"""
|
184
|
-
response = httpx.get(
|
185
|
-
f"{self._http_url()}/voices",
|
186
|
-
headers=self.headers,
|
187
|
-
timeout=self.timeout,
|
188
|
-
)
|
189
|
-
|
190
|
-
if not response.is_success:
|
191
|
-
raise ValueError(f"Failed to get voices. Error: {response.text}")
|
192
|
-
|
193
|
-
voices = response.json()
|
194
|
-
return voices
|
195
|
-
|
196
|
-
def get(self, id: str) -> VoiceMetadata:
|
197
|
-
"""Get a voice by its ID.
|
198
|
-
|
199
|
-
Args:
|
200
|
-
id: The ID of the voice.
|
201
|
-
|
202
|
-
Returns:
|
203
|
-
A VoiceMetadata object containing the voice metadata.
|
204
|
-
"""
|
205
|
-
url = f"{self._http_url()}/voices/{id}"
|
206
|
-
response = httpx.get(url, headers=self.headers, timeout=self.timeout)
|
207
|
-
|
208
|
-
if not response.is_success:
|
209
|
-
raise ValueError(
|
210
|
-
f"Failed to get voice. Status Code: {response.status_code}\n"
|
211
|
-
f"Error: {response.text}"
|
212
|
-
)
|
213
|
-
|
214
|
-
return response.json()
|
215
|
-
|
216
|
-
def clone(self, filepath: Optional[str] = None, enhance: str = True) -> List[float]:
|
217
|
-
"""Clone a voice from a clip.
|
218
|
-
|
219
|
-
Args:
|
220
|
-
filepath: The path to the clip file.
|
221
|
-
enhance: Whether to enhance the clip before cloning the voice (highly recommended). Defaults to True.
|
222
|
-
|
223
|
-
Returns:
|
224
|
-
The embedding of the cloned voice as a list of floats.
|
225
|
-
"""
|
226
|
-
if not filepath:
|
227
|
-
raise ValueError("Filepath must be specified.")
|
228
|
-
url = f"{self._http_url()}/voices/clone/clip"
|
229
|
-
with open(filepath, "rb") as file:
|
230
|
-
files = {"clip": file}
|
231
|
-
files["enhance"] = str(enhance).lower()
|
232
|
-
headers = self.headers.copy()
|
233
|
-
headers.pop("Content-Type", None)
|
234
|
-
response = httpx.post(url, headers=headers, files=files, timeout=self.timeout)
|
235
|
-
if not response.is_success:
|
236
|
-
raise ValueError(f"Failed to clone voice from clip. Error: {response.text}")
|
237
|
-
|
238
|
-
return response.json()["embedding"]
|
239
|
-
|
240
|
-
def create(self, name: str, description: str, embedding: List[float]) -> VoiceMetadata:
|
241
|
-
"""Create a new voice.
|
242
|
-
|
243
|
-
Args:
|
244
|
-
name: The name of the voice.
|
245
|
-
description: The description of the voice.
|
246
|
-
embedding: The embedding of the voice. This should be generated with :meth:`clone`.
|
247
|
-
|
248
|
-
Returns:
|
249
|
-
A dictionary containing the voice metadata.
|
250
|
-
"""
|
251
|
-
response = httpx.post(
|
252
|
-
f"{self._http_url()}/voices",
|
253
|
-
headers=self.headers,
|
254
|
-
json={"name": name, "description": description, "embedding": embedding},
|
255
|
-
timeout=self.timeout,
|
256
|
-
)
|
257
|
-
|
258
|
-
if not response.is_success:
|
259
|
-
raise ValueError(f"Failed to create voice. Error: {response.text}")
|
260
|
-
|
261
|
-
return response.json()
|
262
|
-
|
263
|
-
def mix(self, voices: List[Dict[str, Union[str, float]]]) -> List[float]:
|
264
|
-
"""Mix multiple voices together.
|
265
|
-
|
266
|
-
Args:
|
267
|
-
voices: A list of dictionaries, each containing either:
|
268
|
-
- 'id': The ID of an existing voice
|
269
|
-
- 'embedding': A voice embedding
|
270
|
-
AND
|
271
|
-
- 'weight': The weight of the voice in the mix (0.0 to 1.0)
|
272
|
-
|
273
|
-
Returns:
|
274
|
-
The embedding of the mixed voice as a list of floats.
|
275
|
-
|
276
|
-
Raises:
|
277
|
-
ValueError: If the request fails or if the input is invalid.
|
278
|
-
"""
|
279
|
-
url = f"{self._http_url()}/voices/mix"
|
280
|
-
|
281
|
-
if not voices or not isinstance(voices, list):
|
282
|
-
raise ValueError("voices must be a non-empty list")
|
283
|
-
|
284
|
-
response = httpx.post(
|
285
|
-
url,
|
286
|
-
headers=self.headers,
|
287
|
-
json={"voices": voices},
|
288
|
-
timeout=self.timeout,
|
289
|
-
)
|
290
|
-
|
291
|
-
if not response.is_success:
|
292
|
-
raise ValueError(f"Failed to mix voices. Error: {response.text}")
|
293
|
-
|
294
|
-
result = response.json()
|
295
|
-
return result["embedding"]
|
296
|
-
|
297
|
-
|
298
|
-
class _TTSContext:
|
299
|
-
"""Manage a single context over a WebSocket.
|
300
|
-
|
301
|
-
This class can be used to stream inputs, as they become available, to a specific `context_id`. See README for usage.
|
302
|
-
|
303
|
-
See :class:`_AsyncTTSContext` for asynchronous use cases.
|
304
|
-
|
305
|
-
Each TTSContext will close automatically when a done message is received for that context. It also closes if there is an error.
|
306
|
-
"""
|
307
|
-
|
308
|
-
def __init__(self, context_id: str, websocket: "_WebSocket"):
|
309
|
-
self._context_id = context_id
|
310
|
-
self._websocket = websocket
|
311
|
-
self._error = None
|
312
|
-
|
313
|
-
def __del__(self):
|
314
|
-
self._close()
|
315
|
-
|
316
|
-
@property
|
317
|
-
def context_id(self) -> str:
|
318
|
-
return self._context_id
|
319
|
-
|
320
|
-
def send(
|
321
|
-
self,
|
322
|
-
model_id: str,
|
323
|
-
transcript: Iterator[str],
|
324
|
-
output_format: OutputFormat,
|
325
|
-
voice_id: Optional[str] = None,
|
326
|
-
voice_embedding: Optional[List[float]] = None,
|
327
|
-
context_id: Optional[str] = None,
|
328
|
-
duration: Optional[int] = None,
|
329
|
-
language: Optional[str] = None,
|
330
|
-
add_timestamps: bool = False,
|
331
|
-
_experimental_voice_controls: Optional[VoiceControls] = None,
|
332
|
-
) -> Generator[bytes, None, None]:
|
333
|
-
"""Send audio generation requests to the WebSocket and yield responses.
|
334
|
-
|
335
|
-
Args:
|
336
|
-
model_id: The ID of the model to use for generating audio.
|
337
|
-
transcript: Iterator over text chunks with <1s latency.
|
338
|
-
output_format: A dictionary containing the details of the output format.
|
339
|
-
voice_id: The ID of the voice to use for generating audio.
|
340
|
-
voice_embedding: The embedding of the voice to use for generating audio.
|
341
|
-
context_id: The context ID to use for the request. If not specified, a random context ID will be generated.
|
342
|
-
duration: The duration of the audio in seconds.
|
343
|
-
language: The language code for the audio request. This can only be used with `model_id = sonic-multilingual`
|
344
|
-
add_timestamps: Whether to return word-level timestamps.
|
345
|
-
_experimental_voice_controls: Experimental voice controls for controlling speed and emotion.
|
346
|
-
Note: This is an experimental feature and may change rapidly in future releases.
|
347
|
-
|
348
|
-
Yields:
|
349
|
-
Dictionary containing the following key(s):
|
350
|
-
- audio: The audio as bytes.
|
351
|
-
- context_id: The context ID for the request.
|
352
|
-
|
353
|
-
Raises:
|
354
|
-
ValueError: If provided context_id doesn't match the current context.
|
355
|
-
RuntimeError: If there's an error generating audio.
|
356
|
-
"""
|
357
|
-
if context_id is not None and context_id != self._context_id:
|
358
|
-
raise ValueError("Context ID does not match the context ID of the current context.")
|
359
|
-
|
360
|
-
self._websocket.connect()
|
361
|
-
|
362
|
-
voice = TTS._validate_and_construct_voice(
|
363
|
-
voice_id,
|
364
|
-
voice_embedding=voice_embedding,
|
365
|
-
experimental_voice_controls=_experimental_voice_controls,
|
366
|
-
)
|
367
|
-
|
368
|
-
# Create the initial request body
|
369
|
-
request_body = {
|
370
|
-
"model_id": model_id,
|
371
|
-
"voice": voice,
|
372
|
-
"output_format": {
|
373
|
-
"container": output_format["container"],
|
374
|
-
"encoding": output_format["encoding"],
|
375
|
-
"sample_rate": output_format["sample_rate"],
|
376
|
-
},
|
377
|
-
"context_id": self._context_id,
|
378
|
-
"language": language,
|
379
|
-
"add_timestamps": add_timestamps,
|
380
|
-
}
|
381
|
-
|
382
|
-
if duration is not None:
|
383
|
-
request_body["duration"] = duration
|
384
|
-
|
385
|
-
try:
|
386
|
-
# Create an iterator with a timeout to get text chunks
|
387
|
-
text_iterator = TimeoutIterator(
|
388
|
-
transcript, timeout=0.001
|
389
|
-
) # 1ms timeout for nearly non-blocking receive
|
390
|
-
next_chunk = next(text_iterator, None)
|
391
|
-
|
392
|
-
while True:
|
393
|
-
# Send the next text chunk to the WebSocket if available
|
394
|
-
if next_chunk is not None and next_chunk != text_iterator.get_sentinel():
|
395
|
-
request_body["transcript"] = next_chunk
|
396
|
-
request_body["continue"] = True
|
397
|
-
self._websocket.websocket.send(json.dumps(request_body))
|
398
|
-
next_chunk = next(text_iterator, None)
|
399
|
-
|
400
|
-
try:
|
401
|
-
# Receive responses from the WebSocket with a small timeout
|
402
|
-
response = json.loads(
|
403
|
-
self._websocket.websocket.recv(timeout=0.001)
|
404
|
-
) # 1ms timeout for nearly non-blocking receive
|
405
|
-
if response["context_id"] != self._context_id:
|
406
|
-
pass
|
407
|
-
if "error" in response:
|
408
|
-
raise RuntimeError(f"Error generating audio:\n{response['error']}")
|
409
|
-
if response["done"]:
|
410
|
-
break
|
411
|
-
if response["data"]:
|
412
|
-
yield self._websocket._convert_response(
|
413
|
-
response=response, include_context_id=True
|
414
|
-
)
|
415
|
-
except TimeoutError:
|
416
|
-
pass
|
417
|
-
|
418
|
-
# Continuously receive from WebSocket until the next text chunk is available
|
419
|
-
while next_chunk == text_iterator.get_sentinel():
|
420
|
-
try:
|
421
|
-
response = json.loads(self._websocket.websocket.recv(timeout=0.001))
|
422
|
-
if response["context_id"] != self._context_id:
|
423
|
-
continue
|
424
|
-
if "error" in response:
|
425
|
-
raise RuntimeError(f"Error generating audio:\n{response['error']}")
|
426
|
-
if response["done"]:
|
427
|
-
break
|
428
|
-
if response["data"]:
|
429
|
-
yield self._websocket._convert_response(
|
430
|
-
response=response, include_context_id=True
|
431
|
-
)
|
432
|
-
except TimeoutError:
|
433
|
-
pass
|
434
|
-
next_chunk = next(text_iterator, None)
|
435
|
-
|
436
|
-
# Send final message if all input text chunks are exhausted
|
437
|
-
if next_chunk is None:
|
438
|
-
request_body["transcript"] = ""
|
439
|
-
request_body["continue"] = False
|
440
|
-
self._websocket.websocket.send(json.dumps(request_body))
|
441
|
-
break
|
442
|
-
|
443
|
-
# Receive remaining messages from the WebSocket until "done" is received
|
444
|
-
while True:
|
445
|
-
response = json.loads(self._websocket.websocket.recv())
|
446
|
-
if response["context_id"] != self._context_id:
|
447
|
-
continue
|
448
|
-
if "error" in response:
|
449
|
-
raise RuntimeError(f"Error generating audio:\n{response['error']}")
|
450
|
-
if response["done"]:
|
451
|
-
break
|
452
|
-
yield self._websocket._convert_response(response=response, include_context_id=True)
|
453
|
-
|
454
|
-
except Exception as e:
|
455
|
-
self._websocket.close()
|
456
|
-
raise RuntimeError(f"Failed to generate audio. {e}")
|
457
|
-
|
458
|
-
def _close(self):
|
459
|
-
"""Closes the context. Automatically called when a done message is received for this context."""
|
460
|
-
self._websocket._remove_context(self._context_id)
|
461
|
-
|
462
|
-
def is_closed(self):
|
463
|
-
"""Check if the context is closed or not. Returns True if closed."""
|
464
|
-
return self._context_id not in self._websocket._contexts
|
465
|
-
|
466
|
-
|
467
|
-
class _WebSocket:
|
468
|
-
"""This class contains methods to generate audio using WebSocket. Ideal for low-latency audio generation.
|
469
|
-
|
470
|
-
Usage:
|
471
|
-
>>> ws = client.tts.websocket()
|
472
|
-
>>> for audio_chunk in ws.send(
|
473
|
-
... model_id="sonic-english", transcript="Hello world!", voice_embedding=embedding,
|
474
|
-
... output_format={"container": "raw", "encoding": "pcm_f32le", "sample_rate": 44100},
|
475
|
-
... context_id=context_id, stream=True
|
476
|
-
... ):
|
477
|
-
... audio = audio_chunk["audio"]
|
478
|
-
"""
|
479
|
-
|
480
|
-
def __init__(
|
481
|
-
self,
|
482
|
-
ws_url: str,
|
483
|
-
api_key: str,
|
484
|
-
cartesia_version: str,
|
485
|
-
):
|
486
|
-
self.ws_url = ws_url
|
487
|
-
self.api_key = api_key
|
488
|
-
self.cartesia_version = cartesia_version
|
489
|
-
self.websocket = None
|
490
|
-
self._contexts: Set[str] = set()
|
491
|
-
|
492
|
-
def __del__(self):
|
493
|
-
try:
|
494
|
-
self.close()
|
495
|
-
except Exception as e:
|
496
|
-
raise RuntimeError("Failed to close WebSocket: ", e)
|
497
|
-
|
498
|
-
def connect(self):
|
499
|
-
"""This method connects to the WebSocket if it is not already connected.
|
500
|
-
|
501
|
-
Raises:
|
502
|
-
RuntimeError: If the connection to the WebSocket fails.
|
503
|
-
"""
|
504
|
-
if not IS_WEBSOCKET_SYNC_AVAILABLE:
|
505
|
-
raise ImportError(
|
506
|
-
"The synchronous WebSocket client is not available. Please ensure that you have 'websockets>=12.0' or compatible version installed."
|
507
|
-
)
|
508
|
-
if self.websocket is None or self._is_websocket_closed():
|
509
|
-
route = "tts/websocket"
|
510
|
-
try:
|
511
|
-
self.websocket = connect(
|
512
|
-
f"{self.ws_url}/{route}?api_key={self.api_key}&cartesia_version={self.cartesia_version}"
|
513
|
-
)
|
514
|
-
except Exception as e:
|
515
|
-
raise RuntimeError(f"Failed to connect to WebSocket. {e}")
|
516
|
-
|
517
|
-
def _is_websocket_closed(self):
|
518
|
-
return self.websocket.socket.fileno() == -1
|
519
|
-
|
520
|
-
def close(self):
|
521
|
-
"""This method closes the WebSocket connection. *Highly* recommended to call this method when done using the WebSocket."""
|
522
|
-
if self.websocket and not self._is_websocket_closed():
|
523
|
-
self.websocket.close()
|
524
|
-
|
525
|
-
if self._contexts:
|
526
|
-
self._contexts.clear()
|
527
|
-
|
528
|
-
def _convert_response(
|
529
|
-
self, response: Dict[str, any], include_context_id: bool
|
530
|
-
) -> Dict[str, Any]:
|
531
|
-
out = {}
|
532
|
-
if response["type"] == EventType.AUDIO:
|
533
|
-
out["audio"] = base64.b64decode(response["data"])
|
534
|
-
elif response["type"] == EventType.TIMESTAMPS:
|
535
|
-
out["word_timestamps"] = response["word_timestamps"]
|
536
|
-
|
537
|
-
if include_context_id:
|
538
|
-
out["context_id"] = response["context_id"]
|
539
|
-
|
540
|
-
return out
|
541
|
-
|
542
|
-
def send(
|
543
|
-
self,
|
544
|
-
model_id: str,
|
545
|
-
transcript: str,
|
546
|
-
output_format: dict,
|
547
|
-
voice_id: Optional[str] = None,
|
548
|
-
voice_embedding: Optional[List[float]] = None,
|
549
|
-
context_id: Optional[str] = None,
|
550
|
-
duration: Optional[int] = None,
|
551
|
-
language: Optional[str] = None,
|
552
|
-
stream: bool = True,
|
553
|
-
add_timestamps: bool = False,
|
554
|
-
_experimental_voice_controls: Optional[VoiceControls] = None,
|
555
|
-
) -> Union[bytes, Generator[bytes, None, None]]:
|
556
|
-
"""Send a request to the WebSocket to generate audio.
|
557
|
-
|
558
|
-
Args:
|
559
|
-
model_id: The ID of the model to use for generating audio.
|
560
|
-
transcript: The text to convert to speech.
|
561
|
-
output_format: A dictionary containing the details of the output format.
|
562
|
-
voice_id: The ID of the voice to use for generating audio.
|
563
|
-
voice_embedding: The embedding of the voice to use for generating audio.
|
564
|
-
context_id: The context ID to use for the request. If not specified, a random context ID will be generated.
|
565
|
-
duration: The duration of the audio in seconds.
|
566
|
-
language: The language code for the audio request. This can only be used with `model_id = sonic-multilingual`
|
567
|
-
stream: Whether to stream the audio or not.
|
568
|
-
add_timestamps: Whether to return word-level timestamps.
|
569
|
-
_experimental_voice_controls: Experimental voice controls for controlling speed and emotion.
|
570
|
-
Note: This is an experimental feature and may change rapidly in future releases.
|
571
|
-
|
572
|
-
Returns:
|
573
|
-
If `stream` is True, the method returns a generator that yields chunks. Each chunk is a dictionary.
|
574
|
-
If `stream` is False, the method returns a dictionary.
|
575
|
-
Both the generator and the dictionary contain the following key(s):
|
576
|
-
- audio: The audio as bytes.
|
577
|
-
- context_id: The context ID for the request.
|
578
|
-
"""
|
579
|
-
self.connect()
|
580
|
-
|
581
|
-
if context_id is None:
|
582
|
-
context_id = str(uuid.uuid4())
|
583
|
-
|
584
|
-
voice = TTS._validate_and_construct_voice(
|
585
|
-
voice_id,
|
586
|
-
voice_embedding=voice_embedding,
|
587
|
-
experimental_voice_controls=_experimental_voice_controls,
|
588
|
-
)
|
589
|
-
|
590
|
-
request_body = {
|
591
|
-
"model_id": model_id,
|
592
|
-
"transcript": transcript,
|
593
|
-
"voice": voice,
|
594
|
-
"output_format": {
|
595
|
-
"container": output_format["container"],
|
596
|
-
"encoding": output_format["encoding"],
|
597
|
-
"sample_rate": output_format["sample_rate"],
|
598
|
-
},
|
599
|
-
"context_id": context_id,
|
600
|
-
"language": language,
|
601
|
-
"add_timestamps": add_timestamps,
|
602
|
-
}
|
603
|
-
|
604
|
-
if duration is not None:
|
605
|
-
request_body["duration"] = duration
|
606
|
-
|
607
|
-
generator = self._websocket_generator(request_body)
|
608
|
-
|
609
|
-
if stream:
|
610
|
-
return generator
|
611
|
-
|
612
|
-
chunks = []
|
613
|
-
word_timestamps = defaultdict(list)
|
614
|
-
for chunk in generator:
|
615
|
-
if "audio" in chunk:
|
616
|
-
chunks.append(chunk["audio"])
|
617
|
-
if add_timestamps and "word_timestamps" in chunk:
|
618
|
-
for k, v in chunk["word_timestamps"].items():
|
619
|
-
word_timestamps[k].extend(v)
|
620
|
-
out = {"audio": b"".join(chunks), "context_id": context_id}
|
621
|
-
if add_timestamps:
|
622
|
-
out["word_timestamps"] = word_timestamps
|
623
|
-
return out
|
624
|
-
|
625
|
-
def _websocket_generator(self, request_body: Dict[str, Any]):
|
626
|
-
self.websocket.send(json.dumps(request_body))
|
627
|
-
|
628
|
-
try:
|
629
|
-
while True:
|
630
|
-
response = json.loads(self.websocket.recv())
|
631
|
-
if "error" in response:
|
632
|
-
raise RuntimeError(f"Error generating audio:\n{response['error']}")
|
633
|
-
if response["done"]:
|
634
|
-
break
|
635
|
-
yield self._convert_response(response=response, include_context_id=True)
|
636
|
-
except Exception as e:
|
637
|
-
# Close the websocket connection if an error occurs.
|
638
|
-
self.close()
|
639
|
-
raise RuntimeError(f"Failed to generate audio. {response}") from e
|
640
|
-
|
641
|
-
def _remove_context(self, context_id: str):
|
642
|
-
if context_id in self._contexts:
|
643
|
-
self._contexts.remove(context_id)
|
644
|
-
|
645
|
-
def context(self, context_id: Optional[str] = None) -> _TTSContext:
|
646
|
-
if context_id in self._contexts:
|
647
|
-
raise ValueError(f"Context for context ID {context_id} already exists.")
|
648
|
-
if context_id is None:
|
649
|
-
context_id = str(uuid.uuid4())
|
650
|
-
if context_id not in self._contexts:
|
651
|
-
self._contexts.add(context_id)
|
652
|
-
return _TTSContext(context_id, self)
|
653
|
-
|
654
|
-
|
655
|
-
class _SSE:
|
656
|
-
"""This class contains methods to generate audio using Server-Sent Events.
|
657
|
-
|
658
|
-
Usage:
|
659
|
-
>>> for audio_chunk in client.tts.sse(
|
660
|
-
... model_id="sonic-english", transcript="Hello world!", voice_embedding=embedding,
|
661
|
-
... output_format={"container": "raw", "encoding": "pcm_f32le", "sample_rate": 44100}, stream=True
|
662
|
-
... ):
|
663
|
-
... audio = audio_chunk["audio"]
|
664
|
-
"""
|
665
|
-
|
666
|
-
def __init__(
|
667
|
-
self,
|
668
|
-
http_url: str,
|
669
|
-
headers: Dict[str, str],
|
670
|
-
timeout: float,
|
671
|
-
):
|
672
|
-
self.http_url = http_url
|
673
|
-
self.headers = headers
|
674
|
-
self.timeout = timeout
|
675
|
-
|
676
|
-
def _update_buffer(self, buffer: str, chunk_bytes: bytes) -> Tuple[str, List[Dict[str, Any]]]:
|
677
|
-
buffer += chunk_bytes.decode("utf-8")
|
678
|
-
outputs = []
|
679
|
-
while "{" in buffer and "}" in buffer:
|
680
|
-
start_index = buffer.find("{")
|
681
|
-
end_index = buffer.find("}", start_index)
|
682
|
-
if start_index != -1 and end_index != -1:
|
683
|
-
try:
|
684
|
-
chunk_json = json.loads(buffer[start_index : end_index + 1])
|
685
|
-
if "error" in chunk_json:
|
686
|
-
raise RuntimeError(f"Error generating audio:\n{chunk_json['error']}")
|
687
|
-
if chunk_json["done"]:
|
688
|
-
break
|
689
|
-
audio = base64.b64decode(chunk_json["data"])
|
690
|
-
outputs.append({"audio": audio})
|
691
|
-
buffer = buffer[end_index + 1 :]
|
692
|
-
except json.JSONDecodeError:
|
693
|
-
break
|
694
|
-
return buffer, outputs
|
695
|
-
|
696
|
-
def send(
|
697
|
-
self,
|
698
|
-
model_id: str,
|
699
|
-
transcript: str,
|
700
|
-
output_format: OutputFormat,
|
701
|
-
voice_id: Optional[str] = None,
|
702
|
-
voice_embedding: Optional[List[float]] = None,
|
703
|
-
duration: Optional[int] = None,
|
704
|
-
language: Optional[str] = None,
|
705
|
-
stream: bool = True,
|
706
|
-
_experimental_voice_controls: Optional[VoiceControls] = None,
|
707
|
-
) -> Union[bytes, Generator[bytes, None, None]]:
|
708
|
-
"""Send a request to the server to generate audio using Server-Sent Events.
|
709
|
-
|
710
|
-
Args:
|
711
|
-
model_id: The ID of the model to use for generating audio.
|
712
|
-
transcript: The text to convert to speech.
|
713
|
-
voice_id: The ID of the voice to use for generating audio.
|
714
|
-
voice_embedding: The embedding of the voice to use for generating audio.
|
715
|
-
output_format: A dictionary containing the details of the output format.
|
716
|
-
duration: The duration of the audio in seconds.
|
717
|
-
language: The language code for the audio request. This can only be used with `model_id = sonic-multilingual`
|
718
|
-
stream: Whether to stream the audio or not.
|
719
|
-
_experimental_voice_controls: Experimental voice controls for controlling speed and emotion.
|
720
|
-
Note: This is an experimental feature and may change rapidly in future releases.
|
721
|
-
|
722
|
-
Returns:
|
723
|
-
If `stream` is True, the method returns a generator that yields chunks. Each chunk is a dictionary.
|
724
|
-
If `stream` is False, the method returns a dictionary.
|
725
|
-
Both the generator and the dictionary contain the following key(s):
|
726
|
-
- audio: The audio as bytes.
|
727
|
-
"""
|
728
|
-
voice = TTS._validate_and_construct_voice(
|
729
|
-
voice_id,
|
730
|
-
voice_embedding=voice_embedding,
|
731
|
-
experimental_voice_controls=_experimental_voice_controls,
|
732
|
-
)
|
733
|
-
request_body = {
|
734
|
-
"model_id": model_id,
|
735
|
-
"transcript": transcript,
|
736
|
-
"voice": voice,
|
737
|
-
"output_format": {
|
738
|
-
"container": output_format["container"],
|
739
|
-
"encoding": output_format["encoding"],
|
740
|
-
"sample_rate": output_format["sample_rate"],
|
741
|
-
},
|
742
|
-
"language": language,
|
743
|
-
}
|
744
|
-
|
745
|
-
if duration is not None:
|
746
|
-
request_body["duration"] = duration
|
747
|
-
|
748
|
-
generator = self._sse_generator_wrapper(request_body)
|
749
|
-
|
750
|
-
if stream:
|
751
|
-
return generator
|
752
|
-
|
753
|
-
chunks = []
|
754
|
-
for chunk in generator:
|
755
|
-
chunks.append(chunk["audio"])
|
756
|
-
|
757
|
-
return {"audio": b"".join(chunks)}
|
758
|
-
|
759
|
-
@retry_on_connection_error(
|
760
|
-
max_retries=MAX_RETRIES, backoff_factor=BACKOFF_FACTOR, logger=logger
|
761
|
-
)
|
762
|
-
def _sse_generator_wrapper(self, request_body: Dict[str, Any]):
|
763
|
-
"""Need to wrap the sse generator in a function for the retry decorator to work."""
|
764
|
-
try:
|
765
|
-
for chunk in self._sse_generator(request_body):
|
766
|
-
yield chunk
|
767
|
-
except Exception as e:
|
768
|
-
raise RuntimeError(f"Error generating audio. {e}")
|
769
|
-
|
770
|
-
def _sse_generator(self, request_body: Dict[str, Any]):
|
771
|
-
response = requests.post(
|
772
|
-
f"{self.http_url}/tts/sse",
|
773
|
-
stream=True,
|
774
|
-
data=json.dumps(request_body),
|
775
|
-
headers=self.headers,
|
776
|
-
timeout=(self.timeout, self.timeout),
|
777
|
-
)
|
778
|
-
if not response.ok:
|
779
|
-
raise ValueError(f"Failed to generate audio. {response.text}")
|
780
|
-
|
781
|
-
buffer = ""
|
782
|
-
for chunk_bytes in response.iter_content(chunk_size=None):
|
783
|
-
buffer, outputs = self._update_buffer(buffer=buffer, chunk_bytes=chunk_bytes)
|
784
|
-
for output in outputs:
|
785
|
-
yield output
|
786
|
-
|
787
|
-
if buffer:
|
788
|
-
try:
|
789
|
-
chunk_json = json.loads(buffer)
|
790
|
-
audio = base64.b64decode(chunk_json["data"])
|
791
|
-
yield {"audio": audio}
|
792
|
-
except json.JSONDecodeError:
|
793
|
-
pass
|
794
|
-
|
795
|
-
|
796
|
-
class TTS(Resource):
|
797
|
-
"""This resource contains methods to generate audio using Cartesia's text-to-speech API."""
|
798
|
-
|
799
|
-
def __init__(self, api_key: str, base_url: str, timeout: float):
|
800
|
-
super().__init__(
|
801
|
-
api_key=api_key,
|
802
|
-
base_url=base_url,
|
803
|
-
timeout=timeout,
|
804
|
-
)
|
805
|
-
self._sse_class = _SSE(self._http_url(), self.headers, self.timeout)
|
806
|
-
self.sse = self._sse_class.send
|
807
|
-
|
808
|
-
def websocket(self) -> _WebSocket:
|
809
|
-
"""This method returns a WebSocket object that can be used to generate audio using WebSocket.
|
810
|
-
|
811
|
-
Returns:
|
812
|
-
_WebSocket: A WebSocket object that can be used to generate audio using WebSocket.
|
813
|
-
"""
|
814
|
-
ws = _WebSocket(self._ws_url(), self.api_key, self.cartesia_version)
|
815
|
-
ws.connect()
|
816
|
-
return ws
|
817
|
-
|
818
|
-
@staticmethod
|
819
|
-
def get_output_format(output_format_name: str) -> OutputFormat:
|
820
|
-
"""Convenience method to get the output_format dictionary from a given output format name.
|
821
|
-
|
822
|
-
Args:
|
823
|
-
output_format_name (str): The name of the output format.
|
824
|
-
|
825
|
-
Returns:
|
826
|
-
OutputFormat: A dictionary containing the details of the output format to be passed into tts.sse() or tts.websocket().send()
|
827
|
-
|
828
|
-
Raises:
|
829
|
-
ValueError: If the output_format name is not supported
|
830
|
-
"""
|
831
|
-
if output_format_name in OutputFormatMapping._format_mapping:
|
832
|
-
output_format_obj = OutputFormatMapping.get_format(output_format_name)
|
833
|
-
elif output_format_name in DeprecatedOutputFormatMapping._format_mapping:
|
834
|
-
output_format_obj = DeprecatedOutputFormatMapping.get_format_deprecated(
|
835
|
-
output_format_name
|
836
|
-
)
|
837
|
-
else:
|
838
|
-
raise ValueError(f"Unsupported format: {output_format_name}")
|
839
|
-
|
840
|
-
return OutputFormat(
|
841
|
-
container=output_format_obj["container"],
|
842
|
-
encoding=output_format_obj["encoding"],
|
843
|
-
sample_rate=output_format_obj["sample_rate"],
|
844
|
-
)
|
845
|
-
|
846
|
-
@staticmethod
|
847
|
-
def get_sample_rate(self, output_format_name: str) -> int:
|
848
|
-
"""Convenience method to get the sample rate for a given output format.
|
849
|
-
|
850
|
-
Args:
|
851
|
-
output_format_name (str): The name of the output format.
|
852
|
-
|
853
|
-
Returns:
|
854
|
-
int: The sample rate for the output format.
|
855
|
-
|
856
|
-
Raises:
|
857
|
-
ValueError: If the output_format name is not supported
|
858
|
-
"""
|
859
|
-
if output_format_name in OutputFormatMapping._format_mapping:
|
860
|
-
output_format_obj = OutputFormatMapping.get_format(output_format_name)
|
861
|
-
elif output_format_name in DeprecatedOutputFormatMapping._format_mapping:
|
862
|
-
output_format_obj = DeprecatedOutputFormatMapping.get_format_deprecated(
|
863
|
-
output_format_name
|
864
|
-
)
|
865
|
-
else:
|
866
|
-
raise ValueError(f"Unsupported format: {output_format_name}")
|
867
|
-
|
868
|
-
return output_format_obj["sample_rate"]
|
869
|
-
|
870
|
-
@staticmethod
|
871
|
-
def _validate_and_construct_voice(
|
872
|
-
voice_id: Optional[str] = None,
|
873
|
-
voice_embedding: Optional[List[float]] = None,
|
874
|
-
experimental_voice_controls: Optional[VoiceControls] = None,
|
875
|
-
) -> dict:
|
876
|
-
"""Validate and construct the voice dictionary for the request.
|
877
|
-
|
878
|
-
Args:
|
879
|
-
voice_id: The ID of the voice to use for generating audio.
|
880
|
-
voice_embedding: The embedding of the voice to use for generating audio.
|
881
|
-
experimental_voice_controls: Voice controls for emotion and speed.
|
882
|
-
Note: This is an experimental feature and may rapidly change in the future.
|
883
|
-
|
884
|
-
Returns:
|
885
|
-
A dictionary representing the voice configuration.
|
886
|
-
|
887
|
-
Raises:
|
888
|
-
ValueError: If neither or both voice_id and voice_embedding are specified.
|
889
|
-
"""
|
890
|
-
if voice_id is None and voice_embedding is None:
|
891
|
-
raise ValueError("Either voice_id or voice_embedding must be specified.")
|
892
|
-
|
893
|
-
if voice_id is not None and voice_embedding is not None:
|
894
|
-
raise ValueError("Only one of voice_id or voice_embedding should be specified.")
|
895
|
-
|
896
|
-
if voice_id:
|
897
|
-
voice = {"id": voice_id}
|
898
|
-
else:
|
899
|
-
voice = {"embedding": voice_embedding}
|
900
|
-
if experimental_voice_controls is not None:
|
901
|
-
voice["__experimental_controls"] = experimental_voice_controls
|
902
|
-
return voice
|
903
|
-
|
904
|
-
|
905
|
-
class AsyncCartesia(Cartesia):
|
906
|
-
"""The asynchronous version of the Cartesia client."""
|
907
|
-
|
908
|
-
def __init__(
|
909
|
-
self,
|
910
|
-
*,
|
911
|
-
api_key: Optional[str] = None,
|
912
|
-
base_url: Optional[str] = None,
|
913
|
-
timeout: float = DEFAULT_TIMEOUT,
|
914
|
-
max_num_connections: int = DEFAULT_NUM_CONNECTIONS,
|
915
|
-
):
|
916
|
-
"""
|
917
|
-
Args:
|
918
|
-
api_key: See :class:`Cartesia`.
|
919
|
-
base_url: See :class:`Cartesia`.
|
920
|
-
timeout: See :class:`Cartesia`.
|
921
|
-
max_num_connections: The maximum number of concurrent connections to use for the client.
|
922
|
-
This is used to limit the number of connections that can be made to the server.
|
923
|
-
"""
|
924
|
-
self._session = None
|
925
|
-
self._loop = None
|
926
|
-
super().__init__(api_key=api_key, base_url=base_url, timeout=timeout)
|
927
|
-
self.max_num_connections = max_num_connections
|
928
|
-
self.tts = AsyncTTS(
|
929
|
-
api_key=self.api_key,
|
930
|
-
base_url=self._base_url,
|
931
|
-
timeout=self.timeout,
|
932
|
-
get_session=self._get_session,
|
933
|
-
)
|
934
|
-
|
935
|
-
async def _get_session(self):
|
936
|
-
current_loop = asyncio.get_event_loop()
|
937
|
-
if self._loop is not current_loop:
|
938
|
-
# If the loop has changed, close the session and create a new one.
|
939
|
-
await self.close()
|
940
|
-
if self._session is None or self._session.closed:
|
941
|
-
timeout = aiohttp.ClientTimeout(total=self.timeout)
|
942
|
-
connector = aiohttp.TCPConnector(limit=self.max_num_connections)
|
943
|
-
self._session = aiohttp.ClientSession(timeout=timeout, connector=connector)
|
944
|
-
self._loop = current_loop
|
945
|
-
return self._session
|
946
|
-
|
947
|
-
async def close(self):
|
948
|
-
"""This method closes the session.
|
949
|
-
|
950
|
-
It is *strongly* recommended to call this method when you are done using the client.
|
951
|
-
"""
|
952
|
-
if self._session is not None and not self._session.closed:
|
953
|
-
await self._session.close()
|
954
|
-
|
955
|
-
def __del__(self):
|
956
|
-
try:
|
957
|
-
loop = asyncio.get_running_loop()
|
958
|
-
except RuntimeError:
|
959
|
-
loop = None
|
960
|
-
|
961
|
-
if loop is None:
|
962
|
-
asyncio.run(self.close())
|
963
|
-
elif loop.is_running():
|
964
|
-
loop.create_task(self.close())
|
965
|
-
|
966
|
-
async def __aenter__(self):
|
967
|
-
return self
|
968
|
-
|
969
|
-
async def __aexit__(
|
970
|
-
self,
|
971
|
-
exc_type: Union[type, None],
|
972
|
-
exc: Union[BaseException, None],
|
973
|
-
exc_tb: Union[TracebackType, None],
|
974
|
-
):
|
975
|
-
await self.close()
|
976
|
-
|
977
|
-
|
978
|
-
class _AsyncSSE(_SSE):
|
979
|
-
"""This class contains methods to generate audio using Server-Sent Events asynchronously."""
|
980
|
-
|
981
|
-
def __init__(
|
982
|
-
self,
|
983
|
-
http_url: str,
|
984
|
-
headers: Dict[str, str],
|
985
|
-
timeout: float,
|
986
|
-
get_session: Callable[[], Optional[aiohttp.ClientSession]],
|
987
|
-
):
|
988
|
-
super().__init__(http_url, headers, timeout)
|
989
|
-
self._get_session = get_session
|
990
|
-
|
991
|
-
async def send(
|
992
|
-
self,
|
993
|
-
model_id: str,
|
994
|
-
transcript: str,
|
995
|
-
output_format: OutputFormat,
|
996
|
-
voice_id: Optional[str] = None,
|
997
|
-
voice_embedding: Optional[List[float]] = None,
|
998
|
-
duration: Optional[int] = None,
|
999
|
-
language: Optional[str] = None,
|
1000
|
-
stream: bool = True,
|
1001
|
-
_experimental_voice_controls: Optional[VoiceControls] = None,
|
1002
|
-
) -> Union[bytes, AsyncGenerator[bytes, None]]:
|
1003
|
-
voice = TTS._validate_and_construct_voice(
|
1004
|
-
voice_id,
|
1005
|
-
voice_embedding=voice_embedding,
|
1006
|
-
experimental_voice_controls=_experimental_voice_controls,
|
1007
|
-
)
|
1008
|
-
|
1009
|
-
request_body = {
|
1010
|
-
"model_id": model_id,
|
1011
|
-
"transcript": transcript,
|
1012
|
-
"voice": voice,
|
1013
|
-
"output_format": {
|
1014
|
-
"container": output_format["container"],
|
1015
|
-
"encoding": output_format["encoding"],
|
1016
|
-
"sample_rate": output_format["sample_rate"],
|
1017
|
-
},
|
1018
|
-
"language": language,
|
1019
|
-
}
|
1020
|
-
|
1021
|
-
if duration is not None:
|
1022
|
-
request_body["duration"] = duration
|
1023
|
-
|
1024
|
-
generator = self._sse_generator_wrapper(request_body)
|
1025
|
-
|
1026
|
-
if stream:
|
1027
|
-
return generator
|
1028
|
-
|
1029
|
-
chunks = []
|
1030
|
-
async for chunk in generator:
|
1031
|
-
chunks.append(chunk["audio"])
|
1032
|
-
|
1033
|
-
return {"audio": b"".join(chunks)}
|
1034
|
-
|
1035
|
-
@retry_on_connection_error_async(
|
1036
|
-
max_retries=MAX_RETRIES, backoff_factor=BACKOFF_FACTOR, logger=logger
|
1037
|
-
)
|
1038
|
-
async def _sse_generator_wrapper(self, request_body: Dict[str, Any]):
|
1039
|
-
"""Need to wrap the sse generator in a function for the retry decorator to work."""
|
1040
|
-
try:
|
1041
|
-
async for chunk in self._sse_generator(request_body):
|
1042
|
-
yield chunk
|
1043
|
-
except Exception as e:
|
1044
|
-
raise RuntimeError(f"Error generating audio. {e}")
|
1045
|
-
|
1046
|
-
async def _sse_generator(self, request_body: Dict[str, Any]):
|
1047
|
-
session = await self._get_session()
|
1048
|
-
async with session.post(
|
1049
|
-
f"{self.http_url}/tts/sse", data=json.dumps(request_body), headers=self.headers
|
1050
|
-
) as response:
|
1051
|
-
if not response.ok:
|
1052
|
-
raise ValueError(f"Failed to generate audio. {await response.text()}")
|
1053
|
-
|
1054
|
-
buffer = ""
|
1055
|
-
async for chunk_bytes in response.content.iter_any():
|
1056
|
-
buffer, outputs = self._update_buffer(buffer=buffer, chunk_bytes=chunk_bytes)
|
1057
|
-
for output in outputs:
|
1058
|
-
yield output
|
1059
|
-
|
1060
|
-
if buffer:
|
1061
|
-
try:
|
1062
|
-
chunk_json = json.loads(buffer)
|
1063
|
-
audio = base64.b64decode(chunk_json["data"])
|
1064
|
-
yield {"audio": audio}
|
1065
|
-
except json.JSONDecodeError:
|
1066
|
-
pass
|
1067
|
-
|
1068
|
-
|
1069
|
-
class _AsyncTTSContext:
|
1070
|
-
"""Manage a single context over an AsyncWebSocket.
|
1071
|
-
|
1072
|
-
This class separates sending requests and receiving responses into two separate methods.
|
1073
|
-
This can be used for sending multiple requests without awaiting the response.
|
1074
|
-
Then you can listen to the responses in the order they were sent. See README for usage.
|
1075
|
-
|
1076
|
-
Each AsyncTTSContext will close automatically when a done message is received for that context.
|
1077
|
-
This happens when the no_more_inputs method is called (equivalent to sending a request with `continue_ = False`),
|
1078
|
-
or if no requests have been sent for 5 seconds on the same context. It also closes if there is an error.
|
1079
|
-
|
1080
|
-
"""
|
1081
|
-
|
1082
|
-
def __init__(self, context_id: str, websocket: "_AsyncWebSocket", timeout: float):
|
1083
|
-
self._context_id = context_id
|
1084
|
-
self._websocket = websocket
|
1085
|
-
self.timeout = timeout
|
1086
|
-
self._error = None
|
1087
|
-
|
1088
|
-
@property
|
1089
|
-
def context_id(self) -> str:
|
1090
|
-
return self._context_id
|
1091
|
-
|
1092
|
-
async def send(
|
1093
|
-
self,
|
1094
|
-
model_id: str,
|
1095
|
-
transcript: str,
|
1096
|
-
output_format: OutputFormat,
|
1097
|
-
voice_id: Optional[str] = None,
|
1098
|
-
voice_embedding: Optional[List[float]] = None,
|
1099
|
-
context_id: Optional[str] = None,
|
1100
|
-
continue_: bool = False,
|
1101
|
-
duration: Optional[int] = None,
|
1102
|
-
language: Optional[str] = None,
|
1103
|
-
add_timestamps: bool = False,
|
1104
|
-
_experimental_voice_controls: Optional[VoiceControls] = None,
|
1105
|
-
) -> None:
|
1106
|
-
"""Send audio generation requests to the WebSocket. The response can be received using the `receive` method.
|
1107
|
-
|
1108
|
-
Args:
|
1109
|
-
model_id: The ID of the model to use for generating audio.
|
1110
|
-
transcript: The text to convert to speech.
|
1111
|
-
output_format: A dictionary containing the details of the output format.
|
1112
|
-
voice_id: The ID of the voice to use for generating audio.
|
1113
|
-
voice_embedding: The embedding of the voice to use for generating audio.
|
1114
|
-
context_id: The context ID to use for the request. If not specified, a random context ID will be generated.
|
1115
|
-
continue_: Whether to continue the audio generation from the previous transcript or not.
|
1116
|
-
duration: The duration of the audio in seconds.
|
1117
|
-
language: The language code for the audio request. This can only be used with `model_id = sonic-multilingual`.
|
1118
|
-
add_timestamps: Whether to return word-level timestamps.
|
1119
|
-
_experimental_voice_controls: Experimental voice controls for controlling speed and emotion.
|
1120
|
-
Note: This is an experimental feature and may change rapidly in future releases.
|
1121
|
-
|
1122
|
-
Returns:
|
1123
|
-
None.
|
1124
|
-
"""
|
1125
|
-
if context_id is not None and context_id != self._context_id:
|
1126
|
-
raise ValueError("Context ID does not match the context ID of the current context.")
|
1127
|
-
if continue_ and transcript == "":
|
1128
|
-
raise ValueError("Transcript cannot be empty when continue_ is True.")
|
1129
|
-
|
1130
|
-
await self._websocket.connect()
|
1131
|
-
|
1132
|
-
voice = TTS._validate_and_construct_voice(
|
1133
|
-
voice_id, voice_embedding, experimental_voice_controls=_experimental_voice_controls
|
1134
|
-
)
|
1135
|
-
|
1136
|
-
request_body = {
|
1137
|
-
"model_id": model_id,
|
1138
|
-
"transcript": transcript,
|
1139
|
-
"voice": voice,
|
1140
|
-
"output_format": {
|
1141
|
-
"container": output_format["container"],
|
1142
|
-
"encoding": output_format["encoding"],
|
1143
|
-
"sample_rate": output_format["sample_rate"],
|
1144
|
-
},
|
1145
|
-
"context_id": self._context_id,
|
1146
|
-
"continue": continue_,
|
1147
|
-
"language": language,
|
1148
|
-
"add_timestamps": add_timestamps,
|
1149
|
-
}
|
1150
|
-
|
1151
|
-
if duration is not None:
|
1152
|
-
request_body["duration"] = duration
|
1153
|
-
|
1154
|
-
await self._websocket.websocket.send_json(request_body)
|
1155
|
-
|
1156
|
-
# Start listening for responses on the WebSocket
|
1157
|
-
self._websocket._dispatch_listener()
|
1158
|
-
|
1159
|
-
async def no_more_inputs(self) -> None:
|
1160
|
-
"""Send a request to the WebSocket to indicate that no more requests will be sent."""
|
1161
|
-
await self.send(
|
1162
|
-
model_id=DEFAULT_MODEL_ID,
|
1163
|
-
transcript="",
|
1164
|
-
output_format=TTS.get_output_format("raw_pcm_f32le_44100"),
|
1165
|
-
voice_id="a0e99841-438c-4a64-b679-ae501e7d6091", # Default voice ID since it's a required input for now
|
1166
|
-
context_id=self._context_id,
|
1167
|
-
continue_=False,
|
1168
|
-
)
|
1169
|
-
|
1170
|
-
async def receive(self) -> AsyncGenerator[Dict[str, Any], None]:
|
1171
|
-
"""Receive the audio chunks from the WebSocket. This method is a generator that yields audio chunks.
|
1172
|
-
|
1173
|
-
Returns:
|
1174
|
-
An async generator that yields audio chunks. Each chunk is a dictionary containing the audio as bytes.
|
1175
|
-
"""
|
1176
|
-
try:
|
1177
|
-
while True:
|
1178
|
-
response = await self._websocket._get_message(
|
1179
|
-
self._context_id, timeout=self.timeout
|
1180
|
-
)
|
1181
|
-
if "error" in response:
|
1182
|
-
raise RuntimeError(f"Error generating audio:\n{response['error']}")
|
1183
|
-
if response["done"]:
|
1184
|
-
break
|
1185
|
-
yield self._websocket._convert_response(response, include_context_id=True)
|
1186
|
-
except Exception as e:
|
1187
|
-
if isinstance(e, asyncio.TimeoutError):
|
1188
|
-
raise RuntimeError("Timeout while waiting for audio chunk")
|
1189
|
-
raise RuntimeError(f"Failed to generate audio:\n{e}")
|
1190
|
-
finally:
|
1191
|
-
self._close()
|
1192
|
-
|
1193
|
-
def _close(self) -> None:
|
1194
|
-
"""Closes the context. Automatically called when a done message is received for this context."""
|
1195
|
-
self._websocket._remove_context(self._context_id)
|
1196
|
-
|
1197
|
-
def is_closed(self):
|
1198
|
-
"""Check if the context is closed or not. Returns True if closed."""
|
1199
|
-
return self._context_id not in self._websocket._context_queues
|
1200
|
-
|
1201
|
-
async def __aenter__(self):
|
1202
|
-
return self
|
1203
|
-
|
1204
|
-
async def __aexit__(
|
1205
|
-
self,
|
1206
|
-
exc_type: Union[type, None],
|
1207
|
-
exc: Union[BaseException, None],
|
1208
|
-
exc_tb: Union[TracebackType, None],
|
1209
|
-
):
|
1210
|
-
self._close()
|
1211
|
-
|
1212
|
-
def __del__(self):
|
1213
|
-
self._close()
|
1214
|
-
|
1215
|
-
|
1216
|
-
class _AsyncWebSocket(_WebSocket):
|
1217
|
-
"""This class contains methods to generate audio using WebSocket asynchronously."""
|
1218
|
-
|
1219
|
-
def __init__(
|
1220
|
-
self,
|
1221
|
-
ws_url: str,
|
1222
|
-
api_key: str,
|
1223
|
-
cartesia_version: str,
|
1224
|
-
timeout: float,
|
1225
|
-
get_session: Callable[[], Optional[aiohttp.ClientSession]],
|
1226
|
-
):
|
1227
|
-
"""
|
1228
|
-
Args:
|
1229
|
-
ws_url: The WebSocket URL for the Cartesia API.
|
1230
|
-
api_key: The API key to use for authorization.
|
1231
|
-
cartesia_version: The version of the Cartesia API to use.
|
1232
|
-
timeout: The timeout for responses on the WebSocket in seconds.
|
1233
|
-
get_session: A function that returns an aiohttp.ClientSession object.
|
1234
|
-
"""
|
1235
|
-
super().__init__(ws_url, api_key, cartesia_version)
|
1236
|
-
self.timeout = timeout
|
1237
|
-
self._get_session = get_session
|
1238
|
-
self.websocket = None
|
1239
|
-
self._context_queues: Dict[str, asyncio.Queue] = {}
|
1240
|
-
self._processing_task: asyncio.Task = None
|
1241
|
-
|
1242
|
-
def __del__(self):
|
1243
|
-
try:
|
1244
|
-
loop = asyncio.get_running_loop()
|
1245
|
-
except RuntimeError:
|
1246
|
-
loop = None
|
1247
|
-
|
1248
|
-
if loop is None:
|
1249
|
-
asyncio.run(self.close())
|
1250
|
-
elif loop.is_running():
|
1251
|
-
loop.create_task(self.close())
|
1252
|
-
|
1253
|
-
async def connect(self):
|
1254
|
-
if self.websocket is None or self._is_websocket_closed():
|
1255
|
-
route = "tts/websocket"
|
1256
|
-
session = await self._get_session()
|
1257
|
-
try:
|
1258
|
-
self.websocket = await session.ws_connect(
|
1259
|
-
f"{self.ws_url}/{route}?api_key={self.api_key}&cartesia_version={self.cartesia_version}"
|
1260
|
-
)
|
1261
|
-
except Exception as e:
|
1262
|
-
raise RuntimeError(f"Failed to connect to WebSocket. {e}")
|
1263
|
-
|
1264
|
-
def _is_websocket_closed(self):
|
1265
|
-
return self.websocket.closed
|
1266
|
-
|
1267
|
-
async def close(self):
|
1268
|
-
"""This method closes the websocket connection. *Highly* recommended to call this method when done."""
|
1269
|
-
if self.websocket is not None and not self._is_websocket_closed():
|
1270
|
-
await self.websocket.close()
|
1271
|
-
if self._processing_task:
|
1272
|
-
self._processing_task.cancel()
|
1273
|
-
try:
|
1274
|
-
self._processing_task = None
|
1275
|
-
except asyncio.CancelledError:
|
1276
|
-
pass
|
1277
|
-
except TypeError as e:
|
1278
|
-
# Ignore the error if the task is already cancelled
|
1279
|
-
# For some reason we are getting None responses
|
1280
|
-
# TODO: This needs to be fixed - we need to think about why we are getting None responses.
|
1281
|
-
if "Received message 256:None" not in str(e):
|
1282
|
-
raise e
|
1283
|
-
|
1284
|
-
for context_id in list(self._context_queues.keys()):
|
1285
|
-
self._remove_context(context_id)
|
1286
|
-
|
1287
|
-
self._context_queues.clear()
|
1288
|
-
self._processing_task = None
|
1289
|
-
self.websocket = None
|
1290
|
-
|
1291
|
-
async def send(
|
1292
|
-
self,
|
1293
|
-
model_id: str,
|
1294
|
-
transcript: str,
|
1295
|
-
output_format: OutputFormat,
|
1296
|
-
voice_id: Optional[str] = None,
|
1297
|
-
voice_embedding: Optional[List[float]] = None,
|
1298
|
-
context_id: Optional[str] = None,
|
1299
|
-
duration: Optional[int] = None,
|
1300
|
-
language: Optional[str] = None,
|
1301
|
-
stream: bool = True,
|
1302
|
-
add_timestamps: bool = False,
|
1303
|
-
_experimental_voice_controls: Optional[VoiceControls] = None,
|
1304
|
-
) -> Union[bytes, AsyncGenerator[bytes, None]]:
|
1305
|
-
"""See :meth:`_WebSocket.send` for details."""
|
1306
|
-
if context_id is None:
|
1307
|
-
context_id = str(uuid.uuid4())
|
1308
|
-
|
1309
|
-
ctx = self.context(context_id)
|
1310
|
-
|
1311
|
-
await ctx.send(
|
1312
|
-
model_id=model_id,
|
1313
|
-
transcript=transcript,
|
1314
|
-
output_format=output_format,
|
1315
|
-
voice_id=voice_id,
|
1316
|
-
voice_embedding=voice_embedding,
|
1317
|
-
context_id=context_id,
|
1318
|
-
duration=duration,
|
1319
|
-
language=language,
|
1320
|
-
continue_=False,
|
1321
|
-
add_timestamps=add_timestamps,
|
1322
|
-
_experimental_voice_controls=_experimental_voice_controls,
|
1323
|
-
)
|
1324
|
-
|
1325
|
-
generator = ctx.receive()
|
1326
|
-
|
1327
|
-
if stream:
|
1328
|
-
return generator
|
1329
|
-
|
1330
|
-
chunks = []
|
1331
|
-
word_timestamps = defaultdict(list)
|
1332
|
-
async for chunk in generator:
|
1333
|
-
if "audio" in chunk:
|
1334
|
-
chunks.append(chunk["audio"])
|
1335
|
-
if add_timestamps and "word_timestamps" in chunk:
|
1336
|
-
for k, v in chunk["word_timestamps"].items():
|
1337
|
-
word_timestamps[k].extend(v)
|
1338
|
-
out = {"audio": b"".join(chunks), "context_id": context_id}
|
1339
|
-
if add_timestamps:
|
1340
|
-
out["word_timestamps"] = word_timestamps
|
1341
|
-
return out
|
1342
|
-
|
1343
|
-
async def _process_responses(self):
|
1344
|
-
try:
|
1345
|
-
while True:
|
1346
|
-
response = await self.websocket.receive_json()
|
1347
|
-
if response["context_id"]:
|
1348
|
-
context_id = response["context_id"]
|
1349
|
-
if context_id in self._context_queues:
|
1350
|
-
await self._context_queues[context_id].put(response)
|
1351
|
-
except Exception as e:
|
1352
|
-
self._error = e
|
1353
|
-
raise e
|
1354
|
-
|
1355
|
-
async def _get_message(self, context_id: str, timeout: float) -> Dict[str, Any]:
|
1356
|
-
if context_id not in self._context_queues:
|
1357
|
-
raise ValueError(f"Context ID {context_id} not found.")
|
1358
|
-
return await asyncio.wait_for(self._context_queues[context_id].get(), timeout=timeout)
|
1359
|
-
|
1360
|
-
def _remove_context(self, context_id: str):
|
1361
|
-
if context_id in self._context_queues:
|
1362
|
-
del self._context_queues[context_id]
|
1363
|
-
|
1364
|
-
def _dispatch_listener(self):
|
1365
|
-
if self._processing_task is None or self._processing_task.done():
|
1366
|
-
self._processing_task = asyncio.create_task(self._process_responses())
|
1367
|
-
|
1368
|
-
def context(self, context_id: Optional[str] = None) -> _AsyncTTSContext:
|
1369
|
-
if context_id in self._context_queues:
|
1370
|
-
raise ValueError(f"AsyncContext for context ID {context_id} already exists.")
|
1371
|
-
if context_id is None:
|
1372
|
-
context_id = str(uuid.uuid4())
|
1373
|
-
if context_id not in self._context_queues:
|
1374
|
-
self._context_queues[context_id] = asyncio.Queue()
|
1375
|
-
return _AsyncTTSContext(context_id, self, self.timeout)
|
1376
|
-
|
1377
|
-
|
1378
|
-
class AsyncTTS(TTS):
|
1379
|
-
def __init__(self, api_key, base_url, timeout, get_session):
|
1380
|
-
super().__init__(api_key, base_url, timeout)
|
1381
|
-
self._get_session = get_session
|
1382
|
-
self._sse_class = _AsyncSSE(self._http_url(), self.headers, self.timeout, get_session)
|
1383
|
-
self.sse = self._sse_class.send
|
1384
|
-
|
1385
|
-
async def websocket(self) -> _AsyncWebSocket:
|
1386
|
-
ws = _AsyncWebSocket(
|
1387
|
-
self._ws_url(), self.api_key, self.cartesia_version, self.timeout, self._get_session
|
1388
|
-
)
|
1389
|
-
await ws.connect()
|
1390
|
-
return ws
|