livekit-plugins-elevenlabs 1.0.18__py3-none-any.whl → 1.0.20__py3-none-any.whl

livekit/plugins/elevenlabs/__init__.py

@@ -12,11 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""ElevenLabs plugin for LiveKit Agents
+
+See https://docs.livekit.io/agents/integrations/tts/elevenlabs/ for more information.
+"""
+
 from .models import TTSEncoding, TTSModels
+from .stt import STT
 from .tts import DEFAULT_VOICE_ID, TTS, Voice, VoiceSettings
 from .version import __version__
 
 __all__ = [
+    "STT",
     "TTS",
     "Voice",
     "VoiceSettings",

livekit/plugins/elevenlabs/stt.py

@@ -0,0 +1,127 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import asyncio
+import os
+from dataclasses import dataclass
+
+import aiohttp
+
+from livekit import rtc
+from livekit.agents import (
+    DEFAULT_API_CONNECT_OPTIONS,
+    APIConnectionError,
+    APIConnectOptions,
+    APIStatusError,
+    APITimeoutError,
+    stt,
+)
+from livekit.agents.stt import SpeechEventType, STTCapabilities
+from livekit.agents.types import NOT_GIVEN, NotGivenOr
+from livekit.agents.utils import AudioBuffer, http_context, is_given
+
+API_BASE_URL_V1 = "https://api.elevenlabs.io/v1"
+AUTHORIZATION_HEADER = "xi-api-key"
+
+
+@dataclass
+class _STTOptions:
+    api_key: str
+    base_url: str
+    language_code: str = "en"
+
+
+class STT(stt.STT):
+    def __init__(
+        self,
+        api_key: NotGivenOr[str] = NOT_GIVEN,
+        base_url: NotGivenOr[str] = NOT_GIVEN,
+        http_session: aiohttp.ClientSession | None = None,
+        language_code: NotGivenOr[str] = NOT_GIVEN,
+    ) -> None:
+        """
+        Create a new instance of ElevenLabs TTS.
+
+        Args:
+            api_key (NotGivenOr[str]): ElevenLabs API key. Can be set via argument or `ELEVEN_API_KEY` environment variable.
+            base_url (NotGivenOr[str]): Custom base URL for the API. Optional.
+            http_session (aiohttp.ClientSession | None): Custom HTTP session for API requests. Optional.
+            language (NotGivenOr[str]): Language code for the STT model. Optional.
+        """  # noqa: E501
+        super().__init__(capabilities=STTCapabilities(streaming=False, interim_results=True))
+
+        elevenlabs_api_key = api_key if is_given(api_key) else os.environ.get("ELEVEN_API_KEY")
+        if not elevenlabs_api_key:
+            raise ValueError(
+                "ElevenLabs API key is required, either as argument or "
+                "set ELEVEN_API_KEY environmental variable"
+            )
+        self._opts = _STTOptions(
+            api_key=elevenlabs_api_key,
+            base_url=base_url if is_given(base_url) else API_BASE_URL_V1,
+            language_code=language_code,
+        )
+        self._session = http_session
+
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if not self._session:
+            self._session = http_context.http_session()
+
+        return self._session
+
+    async def _recognize_impl(
+        self,
+        buffer: AudioBuffer,
+        *,
+        language: NotGivenOr[str] = NOT_GIVEN,
+        conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS,
+    ) -> stt.SpeechEvent:
+        if is_given(language):
+            self._opts.language_code = language
+
+        wav_bytes = rtc.combine_audio_frames(buffer).to_wav_bytes()
+        form = aiohttp.FormData()
+        form.add_field("file", wav_bytes, filename="audio.wav", content_type="audio/x-wav")
+        form.add_field("model_id", "scribe_v1")
+        form.add_field("language_code", self._opts.language_code)
+
+        try:
+            async with self._ensure_session().post(
+                f"{API_BASE_URL_V1}/speech-to-text",
+                data=form,
+                headers={AUTHORIZATION_HEADER: self._opts.api_key},
+            ) as response:
+                response_json = await response.json()
+                extracted_text = response_json.get("text")
+        except asyncio.TimeoutError as e:
+            raise APITimeoutError() from e
+        except aiohttp.ClientResponseError as e:
+            raise APIStatusError(
+                message=e.message,
+                status_code=e.status,
+                request_id=None,
+                body=None,
+            ) from e
+        except Exception as e:
+            raise APIConnectionError() from e
+
+        return self._transcription_to_speech_event(text=extracted_text)
+
+    def _transcription_to_speech_event(self, text: str) -> stt.SpeechEvent:
+        return stt.SpeechEvent(
+            type=SpeechEventType.FINAL_TRANSCRIPT,
+            alternatives=[stt.SpeechData(text=text, language=self._opts.language_code)],
+        )

livekit/plugins/elevenlabs/tts.py

@@ -88,7 +88,7 @@ class _TTSOptions:
     sample_rate: int
     streaming_latency: NotGivenOr[int]
     word_tokenizer: tokenize.WordTokenizer
-    chunk_length_schedule: list[int]
+    chunk_length_schedule: NotGivenOr[list[int]]
     enable_ssml_parsing: bool
     inactivity_timeout: int
 
@@ -99,7 +99,7 @@ class TTS(tts.TTS):
         *,
         voice_id: str = DEFAULT_VOICE_ID,
         voice_settings: NotGivenOr[VoiceSettings] = NOT_GIVEN,
-        model: TTSModels | str = "eleven_flash_v2_5",
+        model: TTSModels | str = "eleven_turbo_v2_5",
         encoding: NotGivenOr[TTSEncoding] = NOT_GIVEN,
         api_key: NotGivenOr[str] = NOT_GIVEN,
         base_url: NotGivenOr[str] = NOT_GIVEN,
@@ -124,14 +124,11 @@ class TTS(tts.TTS):
             inactivity_timeout (int): Inactivity timeout in seconds for the websocket connection. Defaults to 300.
             word_tokenizer (NotGivenOr[tokenize.WordTokenizer]): Tokenizer for processing text. Defaults to basic WordTokenizer.
             enable_ssml_parsing (bool): Enable SSML parsing for input text. Defaults to False.
-            chunk_length_schedule (NotGivenOr[list[int]]): Schedule for chunk lengths, ranging from 50 to 500. Defaults to [80, 120, 200, 260].
+            chunk_length_schedule (NotGivenOr[list[int]]): Schedule for chunk lengths, ranging from 50 to 500. Defaults are [120, 160, 250, 290].
             http_session (aiohttp.ClientSession | None): Custom HTTP session for API requests. Optional.
             language (NotGivenOr[str]): Language code for the TTS model, as of 10/24/24 only valid for "eleven_turbo_v2_5".
         """  # noqa: E501
 
-        if not is_given(chunk_length_schedule):
-            chunk_length_schedule = [80, 120, 200, 260]
-
         if not is_given(encoding):
             encoding = _DefaultEncoding
 
@@ -404,11 +401,13 @@ class SynthesizeStream(tts.SynthesizeStream):
         # 11labs protocol expects the first message to be an "init msg"
         init_pkt = {
             "text": " ",
-            "voice_settings": _strip_nones(dataclasses.asdict(self._opts.voice_settings))
-            if is_given(self._opts.voice_settings)
-            else None,
-            "generation_config": {"chunk_length_schedule": self._opts.chunk_length_schedule},
         }
+        if is_given(self._opts.chunk_length_schedule):
+            init_pkt["generation_config"] = {
+                "chunk_length_schedule": self._opts.chunk_length_schedule
+            }
+        if is_given(self._opts.voice_settings):
+            init_pkt["voice_settings"] = _strip_nones(dataclasses.asdict(self._opts.voice_settings))
         await ws_conn.send_str(json.dumps(init_pkt))
         eos_sent = False
 
@@ -418,20 +417,25 @@ class SynthesizeStream(tts.SynthesizeStream):
             xml_content = []
             async for data in word_stream:
                 text = data.token
-                # send the xml phoneme in one go
+                # send xml tags fully formed
+                xml_start_tokens = ["<phoneme", "<break"]
+                xml_end_tokens = ["</phoneme>", "/>"]
+
                 if (
                     self._opts.enable_ssml_parsing
-                    and data.token.startswith("<phoneme")
+                    and any(data.token.startswith(start) for start in xml_start_tokens)
                     or xml_content
                 ):
                     xml_content.append(text)
-                    if data.token.find("</phoneme>") > -1:
+
+                    if any(data.token.find(end) > -1 for end in xml_end_tokens):
                         text = self._opts.word_tokenizer.format_words(xml_content)
                         xml_content = []
                     else:
                         continue
 
                 data_pkt = {"text": f"{text} "}  # must always end with a space
+
                 self._mark_started()
                 await ws_conn.send_str(json.dumps(data_pkt))
             if xml_content:

livekit/plugins/elevenlabs/version.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "1.0.18"
+__version__ = "1.0.20"

{livekit_plugins_elevenlabs-1.0.18.dist-info → livekit_plugins_elevenlabs-1.0.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: livekit-plugins-elevenlabs
-Version: 1.0.18
+Version: 1.0.20
 Summary: Agent Framework plugin for voice synthesis with ElevenLabs' API.
 Project-URL: Documentation, https://docs.livekit.io
 Project-URL: Website, https://livekit.io/
@@ -18,12 +18,14 @@ Classifier: Topic :: Multimedia :: Sound/Audio
 Classifier: Topic :: Multimedia :: Video
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0
-Requires-Dist: livekit-agents[codecs]>=1.0.18
+Requires-Dist: livekit-agents[codecs]>=1.0.20
 Description-Content-Type: text/markdown
 
-# LiveKit Plugins Elevenlabs
+# ElevenLabs plugin for LiveKit Agents
 
-Agent Framework plugin for voice synthesis with [ElevenLabs](https://elevenlabs.io/) API.
+Support for voice synthesis with [ElevenLabs](https://elevenlabs.io/).
+
+See [https://docs.livekit.io/agents/integrations/tts/elevenlabs/](https://docs.livekit.io/agents/integrations/tts/elevenlabs/) for more information.
 
 ## Installation
 

livekit_plugins_elevenlabs-1.0.20.dist-info/RECORD

@@ -0,0 +1,10 @@
+livekit/plugins/elevenlabs/__init__.py,sha256=wDHAYf2M89U-wTdW8UjDPI_TQaO9XK03Y0mJpsXvobk,1419
+livekit/plugins/elevenlabs/log.py,sha256=hIuXqDsEB5GBa7rQY3z4Uqi1oCqc_lRmCHZEmXz0LHw,73
+livekit/plugins/elevenlabs/models.py,sha256=p_wHEz15bdsNEqwzN831ysm70PNWQ-xeN__BKvGPZxA,401
+livekit/plugins/elevenlabs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/elevenlabs/stt.py,sha256=1B8c7t_52GIbnPSFLq44Fkm0gnnFUZs7xX9nIWbsAQM,4528
+livekit/plugins/elevenlabs/tts.py,sha256=0Mfm6zhmTJKUQZTR_WfEF_p9TIzzrb3p_-VX2dqIJ10,20294
+livekit/plugins/elevenlabs/version.py,sha256=t4KmPVTpEy1pOJ2GRCA-GNJfCQq_-zHNDBxGj4GKfVk,601
+livekit_plugins_elevenlabs-1.0.20.dist-info/METADATA,sha256=kPmu-4R8ZWvXMnkCKdIrcQ6oAs8jpqdg2THHLVXBnz4,1455
+livekit_plugins_elevenlabs-1.0.20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+livekit_plugins_elevenlabs-1.0.20.dist-info/RECORD,,

livekit_plugins_elevenlabs-1.0.18.dist-info/RECORD

@@ -1,9 +0,0 @@
-livekit/plugins/elevenlabs/__init__.py,sha256=Va24UYTuuosmRuTcuzd_DIHYQOgV-wSYKJIXmOSB2Go,1255
-livekit/plugins/elevenlabs/log.py,sha256=hIuXqDsEB5GBa7rQY3z4Uqi1oCqc_lRmCHZEmXz0LHw,73
-livekit/plugins/elevenlabs/models.py,sha256=p_wHEz15bdsNEqwzN831ysm70PNWQ-xeN__BKvGPZxA,401
-livekit/plugins/elevenlabs/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-livekit/plugins/elevenlabs/tts.py,sha256=KPFf8845VCNG9z5mmwFocjgTilKk-uhA3uvRAnlAd8c,20142
-livekit/plugins/elevenlabs/version.py,sha256=cnPu9FVKZV9tFmmz7lEvftrO3B_nWJVFghi3j6UcJLs,601
-livekit_plugins_elevenlabs-1.0.18.dist-info/METADATA,sha256=fwQeB6XvUbWIgpZ8jkyN1eqBrgMXnb7HjJ5Ken8KgrQ,1314
-livekit_plugins_elevenlabs-1.0.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-livekit_plugins_elevenlabs-1.0.18.dist-info/RECORD,,