livekit-plugins-elevenlabs 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

livekit/plugins/elevenlabs/models.py

@@ -10,4 +10,12 @@ TTSModels = Literal[
     "eleven_flash_v2",
 ]
 
-TTSEncoding = Literal["mp3_44100",]
+TTSEncoding = Literal[
+    "mp3_22050_32",
+    "mp3_44100",
+    "mp3_44100_32",
+    "mp3_44100_64",
+    "mp3_44100_96",
+    "mp3_44100_128",
+    "mp3_44100_192",
+]

livekit/plugins/elevenlabs/tts.py

@@ -37,7 +37,9 @@ from livekit.agents import (
 from .log import logger
 from .models import TTSEncoding, TTSModels
 
-_DefaultEncoding: TTSEncoding = "mp3_44100"
+# by default, use 22.05kHz sample rate at 32kbps
+# in our testing,  reduce TTFB by about ~110ms
+_DefaultEncoding: TTSEncoding = "mp3_22050_32"
 
 
 def _sample_rate_from_format(output_format: TTSEncoding) -> int:
@@ -102,6 +104,7 @@ class TTS(tts.TTS):
         *,
         voice: Voice = DEFAULT_VOICE,
         model: TTSModels | str = "eleven_flash_v2_5",
+        encoding: TTSEncoding | None = None,
         api_key: str | None = None,
         base_url: str | None = None,
         streaming_latency: int = 0,
@@ -131,11 +134,14 @@ class TTS(tts.TTS):
             language (str | None): Language code for the TTS model, as of 10/24/24 only valid for "eleven_turbo_v2_5". Optional.
         """
 
+        if not encoding:
+            encoding = _DefaultEncoding
+
         super().__init__(
             capabilities=tts.TTSCapabilities(
                 streaming=True,
             ),
-            sample_rate=_sample_rate_from_format(_DefaultEncoding),
+            sample_rate=_sample_rate_from_format(encoding),
             num_channels=1,
         )
 
@@ -161,7 +167,7 @@ class TTS(tts.TTS):
             model=model,
             api_key=api_key,
             base_url=base_url or API_BASE_URL_V1,
-            encoding=_DefaultEncoding,
+            encoding=encoding,
             sample_rate=self.sample_rate,
             streaming_latency=streaming_latency,
             word_tokenizer=word_tokenizer,
@@ -171,36 +177,14 @@ class TTS(tts.TTS):
             inactivity_timeout=inactivity_timeout,
         )
         self._session = http_session
-        self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse](
-            connect_cb=self._connect_ws,
-            close_cb=self._close_ws,
-            max_session_duration=inactivity_timeout,
-            mark_refreshed_on_get=True,
-        )
         self._streams = weakref.WeakSet[SynthesizeStream]()
 
-    async def _connect_ws(self) -> aiohttp.ClientWebSocketResponse:
-        session = self._ensure_session()
-        return await asyncio.wait_for(
-            session.ws_connect(
-                _stream_url(self._opts),
-                headers={AUTHORIZATION_HEADER: self._opts.api_key},
-            ),
-            self._conn_options.timeout,
-        )
-
-    async def _close_ws(self, ws: aiohttp.ClientWebSocketResponse):
-        await ws.close()
-
     def _ensure_session(self) -> aiohttp.ClientSession:
         if not self._session:
             self._session = utils.http_context.http_session()
 
         return self._session
 
-    def prewarm(self) -> None:
-        self._pool.prewarm()
-
     async def list_voices(self) -> List[Voice]:
         async with self._ensure_session().get(
             f"{self._opts.base_url}/voices",
@@ -242,7 +226,12 @@ class TTS(tts.TTS):
     def stream(
         self, *, conn_options: Optional[APIConnectOptions] = None
     ) -> "SynthesizeStream":
-        stream = SynthesizeStream(tts=self, pool=self._pool, opts=self._opts)
+        stream = SynthesizeStream(
+            tts=self,
+            conn_options=conn_options,
+            opts=self._opts,
+            session=self._ensure_session(),
+        )
         self._streams.add(stream)
         return stream
 
@@ -250,7 +239,6 @@ class TTS(tts.TTS):
         for stream in list(self._streams):
             await stream.aclose()
         self._streams.clear()
-        await self._pool.aclose()
         await super().aclose()
 
 
@@ -338,11 +326,12 @@ class SynthesizeStream(tts.SynthesizeStream):
         self,
         *,
         tts: TTS,
-        pool: utils.ConnectionPool[aiohttp.ClientWebSocketResponse],
+        session: aiohttp.ClientSession,
         opts: _TTSOptions,
+        conn_options: Optional[APIConnectOptions] = None,
     ):
-        super().__init__(tts=tts)
-        self._opts, self._pool = opts, pool
+        super().__init__(tts=tts, conn_options=conn_options)
+        self._opts, self._session = opts, session
 
     async def _run(self) -> None:
         request_id = utils.shortuuid()
@@ -397,140 +386,143 @@ class SynthesizeStream(tts.SynthesizeStream):
         word_stream: tokenize.WordStream,
         request_id: str,
     ) -> None:
-        async with self._pool.connection() as ws_conn:
-            segment_id = utils.shortuuid()
-            expected_text = ""  # accumulate all tokens sent
+        ws_conn = await self._session.ws_connect(
+            _stream_url(self._opts),
+            headers={AUTHORIZATION_HEADER: self._opts.api_key},
+        )
 
-            decoder = utils.codecs.AudioStreamDecoder(
-                sample_rate=self._opts.sample_rate,
-                num_channels=1,
-            )
+        segment_id = utils.shortuuid()
+        decoder = utils.codecs.AudioStreamDecoder(
+            sample_rate=self._opts.sample_rate,
+            num_channels=1,
+        )
 
-            # 11labs protocol expects the first message to be an "init msg"
-            init_pkt = dict(
-                text=" ",
-                voice_settings=_strip_nones(
-                    dataclasses.asdict(self._opts.voice.settings)
-                )
-                if self._opts.voice.settings
-                else None,
-                generation_config=dict(
-                    chunk_length_schedule=self._opts.chunk_length_schedule
-                ),
+        # 11labs protocol expects the first message to be an "init msg"
+        init_pkt = dict(
+            text=" ",
+            voice_settings=_strip_nones(dataclasses.asdict(self._opts.voice.settings))
+            if self._opts.voice.settings
+            else None,
+            generation_config=dict(
+                chunk_length_schedule=self._opts.chunk_length_schedule
+            ),
+        )
+        await ws_conn.send_str(json.dumps(init_pkt))
+        eos_sent = False
+
+        @utils.log_exceptions(logger=logger)
+        async def send_task():
+            nonlocal eos_sent
+            xml_content = []
+            async for data in word_stream:
+                text = data.token
+                # send the xml phoneme in one go
+                if (
+                    self._opts.enable_ssml_parsing
+                    and data.token.startswith("<phoneme")
+                    or xml_content
+                ):
+                    xml_content.append(text)
+                    if data.token.find("</phoneme>") > -1:
+                        text = self._opts.word_tokenizer.format_words(xml_content)
+                        xml_content = []
+                    else:
+                        continue
+
+                data_pkt = dict(text=f"{text} ")  # must always end with a space
+                self._mark_started()
+                await ws_conn.send_str(json.dumps(data_pkt))
+            if xml_content:
+                logger.warning("11labs stream ended with incomplete xml content")
+
+            # no more token, mark eos
+            eos_pkt = dict(text="")
+            await ws_conn.send_str(json.dumps(eos_pkt))
+            eos_sent = True
+
+        # consumes from decoder and generates events
+        @utils.log_exceptions(logger=logger)
+        async def generate_task():
+            emitter = tts.SynthesizedAudioEmitter(
+                event_ch=self._event_ch,
+                request_id=request_id,
+                segment_id=segment_id,
             )
-            await ws_conn.send_str(json.dumps(init_pkt))
-
-            @utils.log_exceptions(logger=logger)
-            async def send_task():
-                nonlocal expected_text
-                xml_content = []
-                async for data in word_stream:
-                    text = data.token
-                    expected_text += text
-                    # send the xml phoneme in one go
-                    if (
-                        self._opts.enable_ssml_parsing
-                        and data.token.startswith("<phoneme")
-                        or xml_content
-                    ):
-                        xml_content.append(text)
-                        if text.find("</phoneme>") > -1:
-                            text = self._opts.word_tokenizer.format_words(xml_content)
-                            xml_content = []
-                        else:
-                            continue
-
-                    data_pkt = dict(text=f"{text} ")  # must always end with a space
-                    self._mark_started()
-                    await ws_conn.send_str(json.dumps(data_pkt))
-                if xml_content:
-                    logger.warning("11labs stream ended with incomplete xml content")
-                await ws_conn.send_str(json.dumps({"flush": True}))
-
-            # consumes from decoder and generates events
-            @utils.log_exceptions(logger=logger)
-            async def generate_task():
-                emitter = tts.SynthesizedAudioEmitter(
-                    event_ch=self._event_ch,
-                    request_id=request_id,
-                    segment_id=segment_id,
-                )
-                async for frame in decoder:
-                    emitter.push(frame)
-                emitter.flush()
+            async for frame in decoder:
+                emitter.push(frame)
+            emitter.flush()
 
-            # receives from ws and decodes audio
-            @utils.log_exceptions(logger=logger)
-            async def recv_task():
-                nonlocal expected_text
-                received_text = ""
-
-                while True:
-                    msg = await ws_conn.receive()
-                    if msg.type in (
-                        aiohttp.WSMsgType.CLOSED,
-                        aiohttp.WSMsgType.CLOSE,
-                        aiohttp.WSMsgType.CLOSING,
-                    ):
+        # receives from ws and decodes audio
+        @utils.log_exceptions(logger=logger)
+        async def recv_task():
+            nonlocal eos_sent
+
+            while True:
+                msg = await ws_conn.receive()
+                if msg.type in (
+                    aiohttp.WSMsgType.CLOSED,
+                    aiohttp.WSMsgType.CLOSE,
+                    aiohttp.WSMsgType.CLOSING,
+                ):
+                    if not eos_sent:
                         raise APIStatusError(
                             "11labs connection closed unexpectedly, not all tokens have been consumed",
                             request_id=request_id,
                         )
+                    return
 
-                    if msg.type != aiohttp.WSMsgType.TEXT:
-                        logger.warning("unexpected 11labs message type %s", msg.type)
-                        continue
-
-                    data = json.loads(msg.data)
-                    if data.get("audio"):
-                        b64data = base64.b64decode(data["audio"])
-                        decoder.push(b64data)
-
-                        if alignment := data.get("normalizedAlignment"):
-                            received_text += "".join(
-                                alignment.get("chars", [])
-                            ).replace(" ", "")
-                            if received_text == expected_text:
-                                decoder.end_input()
-                                break
-                    elif data.get("error"):
-                        raise APIStatusError(
-                            message=data["error"],
-                            status_code=500,
-                            request_id=request_id,
-                            body=None,
-                        )
-                    else:
-                        raise APIStatusError(
-                            message=f"unexpected 11labs message {data}",
-                            status_code=500,
-                            request_id=request_id,
-                            body=None,
-                        )
+                if msg.type != aiohttp.WSMsgType.TEXT:
+                    logger.warning("unexpected 11labs message type %s", msg.type)
+                    continue
+
+                data = json.loads(msg.data)
+                if data.get("audio"):
+                    b64data = base64.b64decode(data["audio"])
+                    decoder.push(b64data)
+
+                elif data.get("isFinal"):
+                    decoder.end_input()
+                    break
+                elif data.get("error"):
+                    raise APIStatusError(
+                        message=data["error"],
+                        status_code=500,
+                        request_id=request_id,
+                        body=None,
+                    )
+                else:
+                    raise APIStatusError(
+                        message=f"unexpected 11labs message {data}",
+                        status_code=500,
+                        request_id=request_id,
+                        body=None,
+                    )
 
-            tasks = [
-                asyncio.create_task(send_task()),
-                asyncio.create_task(recv_task()),
-                asyncio.create_task(generate_task()),
-            ]
-            try:
-                await asyncio.gather(*tasks)
-            except asyncio.TimeoutError as e:
-                raise APITimeoutError() from e
-            except aiohttp.ClientResponseError as e:
-                raise APIStatusError(
-                    message=e.message,
-                    status_code=e.status,
-                    request_id=request_id,
-                    body=None,
-                ) from e
-            except APIStatusError:
-                raise
-            except Exception as e:
-                raise APIConnectionError() from e
-            finally:
-                await utils.aio.gracefully_cancel(*tasks)
-                await decoder.aclose()
+        tasks = [
+            asyncio.create_task(send_task()),
+            asyncio.create_task(recv_task()),
+            asyncio.create_task(generate_task()),
+        ]
+        try:
+            await asyncio.gather(*tasks)
+        except asyncio.TimeoutError as e:
+            raise APITimeoutError() from e
+        except aiohttp.ClientResponseError as e:
+            raise APIStatusError(
+                message=e.message,
+                status_code=e.status,
+                request_id=request_id,
+                body=None,
+            ) from e
+        except APIStatusError:
+            raise
+        except Exception as e:
+            raise APIConnectionError() from e
+        finally:
+            await utils.aio.gracefully_cancel(*tasks)
+            await decoder.aclose()
+            if ws_conn is not None:
+                await ws_conn.close()
 
 
 def _dict_to_voices_list(data: dict[str, Any]):

livekit/plugins/elevenlabs/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.8.0"
+__version__ = "0.8.2"

{livekit_plugins_elevenlabs-0.8.0.dist-info → livekit_plugins_elevenlabs-0.8.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: livekit-plugins-elevenlabs
-Version: 0.8.0
+Version: 0.8.2
 Summary: Agent Framework plugin for voice synthesis with ElevenLabs' API.
 Home-page: https://github.com/livekit/agents
 License: Apache-2.0

livekit_plugins_elevenlabs-0.8.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+livekit/plugins/elevenlabs/__init__.py,sha256=YZVadomFq3JWiZN6GWXJbuE4vaNNWq1CmdH25du8qwg,1249
+livekit/plugins/elevenlabs/log.py,sha256=hIuXqDsEB5GBa7rQY3z4Uqi1oCqc_lRmCHZEmXz0LHw,73
+livekit/plugins/elevenlabs/models.py,sha256=p_wHEz15bdsNEqwzN831ysm70PNWQ-xeN__BKvGPZxA,401
+livekit/plugins/elevenlabs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/elevenlabs/tts.py,sha256=ipXJdSYMKBd8mzTL3JfvYdRc2sJJRASOPPh2Ppy8NBk,19529
+livekit/plugins/elevenlabs/version.py,sha256=qwktN8wnyHMjA3ewh43aDIBBwMd3jorNpCaoGTqBDrw,600
+livekit_plugins_elevenlabs-0.8.2.dist-info/METADATA,sha256=mOzHe4OynY-A7OK1hi1OK4eXTnMRvKGG0CYjW_kXz0s,1529
+livekit_plugins_elevenlabs-0.8.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+livekit_plugins_elevenlabs-0.8.2.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
+livekit_plugins_elevenlabs-0.8.2.dist-info/RECORD,,

{livekit_plugins_elevenlabs-0.8.0.dist-info → livekit_plugins_elevenlabs-0.8.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

livekit_plugins_elevenlabs-0.8.0.dist-info/RECORD

@@ -1,10 +0,0 @@
-livekit/plugins/elevenlabs/__init__.py,sha256=YZVadomFq3JWiZN6GWXJbuE4vaNNWq1CmdH25du8qwg,1249
-livekit/plugins/elevenlabs/log.py,sha256=hIuXqDsEB5GBa7rQY3z4Uqi1oCqc_lRmCHZEmXz0LHw,73
-livekit/plugins/elevenlabs/models.py,sha256=nB43wLS1ilzS7IxLYVSQxBjKPnbiPl4AHpHAOlG2i00,273
-livekit/plugins/elevenlabs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-livekit/plugins/elevenlabs/tts.py,sha256=KCZnuAngDZck4zIMMgp0BLV0GS31kKChMvdvXUVZ8vY,20491
-livekit/plugins/elevenlabs/version.py,sha256=fObgfvFfJb5Vj0qY1hgEiVKSo6z6atjrJvwAVl4KvR4,600
-livekit_plugins_elevenlabs-0.8.0.dist-info/METADATA,sha256=BwddENtvF9zqxTgjgIsHyavyRfA82TBISYEVwFfo2vs,1529
-livekit_plugins_elevenlabs-0.8.0.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-livekit_plugins_elevenlabs-0.8.0.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
-livekit_plugins_elevenlabs-0.8.0.dist-info/RECORD,,