livekit-plugins-neuphonic 0.1.0__py3-none-any.whl

livekit/plugins/neuphonic/__init__.py

@@ -0,0 +1,39 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .tts import TTS, ChunkedStream
+from .version import __version__
+
+__all__ = ["TTS", "ChunkedStream", "__version__"]
+
+from livekit.agents import Plugin
+
+from .log import logger
+
+
+class NeuphonicPlugin(Plugin):
+    def __init__(self):
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(NeuphonicPlugin())
+
+# Cleanup docs of unexported modules
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit/plugins/neuphonic/log.py

@@ -0,0 +1,3 @@
+import logging
+
+logger = logging.getLogger("livekit.plugins.neuphonic")

livekit/plugins/neuphonic/models.py

@@ -0,0 +1,10 @@
+from typing import Literal
+
+TTSEncodings = Literal[
+    "pcm_linear",
+    "pcm_mulaw",
+]
+
+TTSModels = Literal["neu-fast", "neu-hq"]
+
+TTSLangCodes = Literal["en", "nl", "es", "de", "hi", "en-hi", "ar"]

livekit/plugins/neuphonic/tts.py

@@ -0,0 +1,435 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import json
+import os
+import weakref
+from dataclasses import dataclass
+from typing import Optional
+
+import aiohttp
+from livekit.agents import (
+    APIConnectionError,
+    APIConnectOptions,
+    APIStatusError,
+    APITimeoutError,
+    tts,
+    utils,
+)
+
+from .log import logger
+from .models import TTSEncodings, TTSLangCodes, TTSModels
+
+API_BASE_URL = "api.neuphonic.com"
+AUTHORIZATION_HEADER = "X-API-KEY"
+NUM_CHANNELS = 1
+
+
+@dataclass
+class _TTSOptions:
+    base_url: str
+    api_key: str
+    model: TTSModels | str
+    lang_code: TTSLangCodes | str
+    encoding: TTSEncodings | str
+    sampling_rate: int
+    speed: float
+    voice_id: str | None = None
+
+    @property
+    def model_params(self) -> dict:
+        """Returns a dict of all model parameters and their values."""
+        params = [
+            "voice_id",
+            "model",
+            "lang_code",
+            "encoding",
+            "sampling_rate",
+            "speed",
+        ]
+        values = {}
+
+        for param in params:
+            if hasattr(self, param) and getattr(self, param) is not None:
+                values[param] = getattr(self, param)
+
+        return values
+
+    def get_query_param_string(self):
+        """Forms the query parameter string from all model parameters."""
+        queries = []
+        for key, value in self.model_params.items():
+            queries.append(f"{key}={value}")
+
+        return "?" + "&".join(queries)
+
+
+def _parse_sse_message(message: str) -> dict:
+    """
+    Parse each response from the SSE endpoint.
+
+    The message will either be a string reading:
+    - `event: error`
+    - `event: message`
+    - `data: { "status_code": 200, "data": {"audio": ... } }`
+    """
+    message = message.strip()
+
+    if not message or "data" not in message:
+        return None
+
+    _, value = message.split(": ", 1)
+    message = json.loads(value)
+
+    if message.get("errors") is not None:
+        raise Exception(
+            f"Status {message.status_code} error received: {message.errors}."
+        )
+
+    return message
+
+
+class TTS(tts.TTS):
+    def __init__(
+        self,
+        *,
+        model: TTSModels | str = "neu_hq",
+        voice_id: str | None = None,
+        lang_code: TTSLangCodes | str = "en",
+        encoding: TTSEncodings | str = "pcm_linear",
+        speed: float = 1.0,
+        sample_rate: int = 22050,
+        api_key: str | None = None,
+        http_session: aiohttp.ClientSession | None = None,
+        base_url: str = API_BASE_URL,
+    ) -> None:
+        """
+        Create a new instance of the Neuphonic TTS.
+
+        See https://docs.neuphonic.com for more documentation on all of these options, or go to https://app.neuphonic.com/ to test out different options.
+
+        Args:
+            model (TTSModels | str, optional): The Neuphonic model to use. See Defaults to "neu_hq".
+            voice_id (str, optional): The voice ID for the desired voice. Defaults to None.
+            lang_code (TTSLanguages | str, optional): The language code for synthesis. Defaults to "en".
+            encoding (TTSEncodings | str, optional): The audio encoding format. Defaults to "pcm_mulaw".
+            speed (float, optional): The audio playback speed. Defaults to 1.0.
+            sample_rate (int, optional): The audio sample rate in Hz. Defaults to 22050.
+            api_key (str | None, optional): The Neuphonic API key. If not provided, it will be read from the NEUPHONIC_API_TOKEN environment variable.
+            http_session (aiohttp.ClientSession | None, optional): An existing aiohttp ClientSession to use. If not provided, a new session will be created.
+            base_url (str, optional): The base URL for the Neuphonic API. Defaults to "api.neuphonic.com".
+        """
+        super().__init__(
+            capabilities=tts.TTSCapabilities(streaming=True),
+            sample_rate=sample_rate,
+            num_channels=NUM_CHANNELS,
+        )
+
+        api_key = api_key or os.environ.get("NEUPHONIC_API_TOKEN")
+
+        if not api_key:
+            raise ValueError("NEUPHONIC_API_TOKEN must be set")
+
+        self._opts = _TTSOptions(
+            model=model,
+            voice_id=voice_id,
+            lang_code=lang_code,
+            encoding=encoding,
+            speed=speed,
+            sampling_rate=sample_rate,
+            api_key=api_key,
+            base_url=base_url,
+        )
+
+        self._session = http_session
+        self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse](
+            connect_cb=self._connect_ws,
+            close_cb=self._close_ws,
+            max_session_duration=90,
+            mark_refreshed_on_get=True,
+        )
+        self._streams = weakref.WeakSet[SynthesizeStream]()
+
+    async def _connect_ws(self) -> aiohttp.ClientWebSocketResponse:
+        session = self._ensure_session()
+        url = f"wss://{self._opts.base_url}/speak/{self._opts.lang_code}{self._opts.get_query_param_string()}"
+
+        return await asyncio.wait_for(
+            session.ws_connect(url, headers={AUTHORIZATION_HEADER: self._opts.api_key}),
+            self._conn_options.timeout,
+        )
+
+    async def _close_ws(self, ws: aiohttp.ClientWebSocketResponse):
+        await ws.close()
+
+    def _ensure_session(self) -> aiohttp.ClientSession:
+        if not self._session:
+            self._session = utils.http_context.http_session()
+
+        return self._session
+
+    def prewarm(self) -> None:
+        self._pool.prewarm()
+
+    def update_options(
+        self,
+        *,
+        model: TTSModels | str = None,
+        voice_id: str | None = None,
+        lang_code: TTSLangCodes | str | None = None,
+        encoding: TTSEncodings | str | None = None,
+        speed: float | None = None,
+        sample_rate: int | None = None,
+    ) -> None:
+        """
+        Update the Text-to-Speech (TTS) configuration options.
+
+        This method allows updating the TTS settings, including model type, voice_id, lang_code,
+        encoding, speed and sample_rate. If any parameter is not provided, the existing value will be
+        retained.
+
+        Args:
+            model (TTSModels | str, optional): The Neuphonic model to use.
+            voice_id (str, optional): The voice ID for the desired voice.
+            lang_code (TTSLanguages | str, optional): The language code for synthesis..
+            encoding (TTSEncodings | str, optional): The audio encoding format.
+            speed (float, optional): The audio playback speed.
+            sample_rate (int, optional): The audio sample rate in Hz.
+        """
+        self._opts.model = model or self._opts.model
+        self._opts.voice_id = voice_id or self._opts.voice_id
+        self._opts.lang_code = lang_code or self._opts.lang_code
+        self._opts.encoding = encoding or self._opts.encoding
+        self._opts.speed = speed or self._opts.speed
+        self._opts.sampling_rate = sample_rate or self._opts.sampling_rate
+        self._pool.invalidate()
+
+    def synthesize(
+        self,
+        text: str,
+        *,
+        conn_options: Optional[APIConnectOptions] = None,
+    ) -> ChunkedStream:
+        return ChunkedStream(
+            tts=self,
+            input_text=text,
+            conn_options=conn_options,
+            opts=self._opts,
+            session=self._ensure_session(),
+        )
+
+    def stream(
+        self, *, conn_options: Optional[APIConnectOptions] = None
+    ) -> SynthesizeStream:
+        stream = SynthesizeStream(
+            tts=self,
+            pool=self._pool,
+            opts=self._opts,
+        )
+        self._streams.add(stream)
+        return stream
+
+    async def aclose(self) -> None:
+        for stream in list(self._streams):
+            await stream.aclose()
+        self._streams.clear()
+        await self._pool.aclose()
+        await super().aclose()
+
+
+class ChunkedStream(tts.ChunkedStream):
+    """Synthesize chunked text using the SSE endpoint"""
+
+    def __init__(
+        self,
+        *,
+        tts: TTS,
+        input_text: str,
+        opts: _TTSOptions,
+        session: aiohttp.ClientSession,
+        conn_options: Optional[APIConnectOptions] = None,
+    ) -> None:
+        super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
+        self._opts, self._session = opts, session
+
+    async def _run(self) -> None:
+        request_id = utils.shortuuid()
+        bstream = utils.audio.AudioByteStream(
+            sample_rate=self._opts.sampling_rate, num_channels=NUM_CHANNELS
+        )
+
+        json_data = {
+            "text": self._input_text,
+            **self._opts.model_params,
+        }
+
+        headers = {
+            AUTHORIZATION_HEADER: self._opts.api_key,
+        }
+
+        try:
+            async with self._session.post(
+                f"https://{self._opts.base_url}/sse/speak/{self._opts.lang_code}",
+                headers=headers,
+                json=json_data,
+                timeout=aiohttp.ClientTimeout(
+                    total=30,
+                    sock_connect=self._conn_options.timeout,
+                ),
+                read_bufsize=10
+                * 1024
+                * 1024,  # large read_bufsize to avoid `ValueError: Chunk too big`
+            ) as response:
+                response.raise_for_status()
+                emitter = tts.SynthesizedAudioEmitter(
+                    event_ch=self._event_ch,
+                    request_id=request_id,
+                )
+
+                async for line in response.content:
+                    message = line.decode("utf-8").strip()
+                    if message:
+                        parsed_message = _parse_sse_message(message)
+
+                        if (
+                            parsed_message is not None
+                            and parsed_message.get("data", {}).get("audio") is not None
+                        ):
+                            audio_bytes = base64.b64decode(
+                                parsed_message["data"]["audio"]
+                            )
+
+                            for frame in bstream.write(audio_bytes):
+                                emitter.push(frame)
+
+                for frame in bstream.flush():
+                    emitter.push(frame)
+                emitter.flush()
+        except asyncio.TimeoutError as e:
+            raise APITimeoutError() from e
+        except aiohttp.ClientResponseError as e:
+            raise APIStatusError(
+                message=e.message,
+                status_code=e.status,
+                request_id=None,
+                body=None,
+            ) from e
+        except Exception as e:
+            raise APIConnectionError() from e
+
+
+class SynthesizeStream(tts.SynthesizeStream):
+    def __init__(
+        self,
+        *,
+        tts: TTS,
+        opts: _TTSOptions,
+        pool: utils.ConnectionPool[aiohttp.ClientWebSocketResponse],
+    ):
+        super().__init__(tts=tts)
+        self._opts, self._pool = opts, pool
+
+    async def _run(self) -> None:
+        request_id = utils.shortuuid()
+        request_data = {request_id: {"sent": "", "recv": ""}}
+
+        def _is_all_audio_recv():
+            """Check whether all audio has been recieved."""
+            recv_text = (
+                request_data[request_id]["recv"]
+                .lower()
+                .replace(" ", "")
+                .replace("\n", "")
+                .replace("<stop>", "")
+            )
+            sent_text = (
+                request_data[request_id]["sent"]
+                .lower()
+                .replace(" ", "")
+                .replace("\n", "")
+                .replace("<stop>", "")
+            )
+
+            return sent_text == recv_text
+
+        async def _send_task(ws: aiohttp.ClientWebSocketResponse):
+            """Stream text to the websocket."""
+            async for data in self._input_ch:
+                self._mark_started()
+
+                if isinstance(data, self._FlushSentinel):
+                    await ws.send_str(json.dumps({"text": "<STOP>"}))
+                    continue
+
+                request_data[request_id]["sent"] += data
+                await ws.send_str(json.dumps({"text": data}))
+
+        async def _recv_task(ws: aiohttp.ClientWebSocketResponse):
+            audio_bstream = utils.audio.AudioByteStream(
+                sample_rate=self._opts.sampling_rate,
+                num_channels=NUM_CHANNELS,
+            )
+            emitter = tts.SynthesizedAudioEmitter(
+                event_ch=self._event_ch,
+                request_id=request_id,
+            )
+
+            while True:
+                msg = await ws.receive()
+                if msg.type in (
+                    aiohttp.WSMsgType.CLOSED,
+                    aiohttp.WSMsgType.CLOSE,
+                    aiohttp.WSMsgType.CLOSING,
+                ):
+                    raise APIStatusError(
+                        "Neuphonic connection closed unexpectedly",
+                        request_id=request_id,
+                    )
+
+                if msg.type != aiohttp.WSMsgType.TEXT:
+                    logger.warning("Unexpected Neuphonic message type %s", msg.type)
+                    continue
+
+                data = json.loads(msg.data)
+
+                if data.get("data"):
+                    b64data = base64.b64decode(data["data"]["audio"])
+                    recv_text = data["data"]["text"]
+                    for frame in audio_bstream.write(b64data):
+                        emitter.push(frame)
+
+                    request_data[request_id]["recv"] += recv_text
+                else:
+                    logger.error("Unexpected Neuphonic message %s", data)
+
+                if _is_all_audio_recv():
+                    for frame in audio_bstream.flush():
+                        emitter.push(frame)
+                    emitter.flush()
+                    break  # we are not going to receive any more audio
+
+        async with self._pool.connection() as ws:
+            tasks = [
+                asyncio.create_task(_send_task(ws)),
+                asyncio.create_task(_recv_task(ws)),
+            ]
+
+            try:
+                await asyncio.gather(*tasks)
+            finally:
+                await utils.aio.gracefully_cancel(*tasks)

livekit/plugins/neuphonic/version.py

@@ -0,0 +1,15 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__version__ = "0.1.0"

livekit_plugins_neuphonic-0.1.0.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.2
+Name: livekit-plugins-neuphonic
+Version: 0.1.0
+Summary: LiveKit Agents Plugin for Neuphonic
+Home-page: https://github.com/livekit/agents
+License: Apache-2.0
+Project-URL: Documentation, https://docs.livekit.io
+Project-URL: Website, https://livekit.io/
+Project-URL: Source, https://github.com/livekit/agents
+Keywords: webrtc,realtime,audio,video,livekit
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Topic :: Multimedia :: Sound/Audio
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.9.0
+Description-Content-Type: text/markdown
+Requires-Dist: livekit-agents<1.0.0,>=0.12.16
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: project-url
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# LiveKit Plugins Neuphonic
+
+Agent Framework plugin for voice synthesis with [Neuphonic](https://neuphonic.com) API.
+
+## Installation
+
+```bash
+pip install livekit-plugins-neuphonic
+```
+
+## Pre-requisites
+
+You'll need an API key from Neuphonic. It can be set as an environment variable: `NEUPHONIC_API_TOKEN`

livekit_plugins_neuphonic-0.1.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+livekit/plugins/neuphonic/__init__.py,sha256=mJnPVLsKAdUkdWuHWd16A0n2vsVBi3GjgNmB8gv9jjI,1097
+livekit/plugins/neuphonic/log.py,sha256=rAHz71IcbvPkixndXBVffPQsmWUKTLqRaYRuPIxO29w,72
+livekit/plugins/neuphonic/models.py,sha256=Svfn_sWA3Q2ZXsPBXY-K5hslq5FE62hvyXBES2C3aSc,201
+livekit/plugins/neuphonic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+livekit/plugins/neuphonic/tts.py,sha256=FFLDghnve_Mrx-0qfEYb4o_Bs8VNnZqAsQNI-M6Zxkw,14736
+livekit/plugins/neuphonic/version.py,sha256=vQH9cItKAVYAmrLbOntkbLqmxrUZrPiKb1TjkZ8jRKQ,600
+livekit_plugins_neuphonic-0.1.0.dist-info/METADATA,sha256=Q7Skn-28cnC318qr28oepZyaWnQ9etIO-H2c7D-M9jo,1480
+livekit_plugins_neuphonic-0.1.0.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+livekit_plugins_neuphonic-0.1.0.dist-info/top_level.txt,sha256=OoDok3xUmXbZRvOrfvvXB-Juu4DX79dlq188E19YHoo,8
+livekit_plugins_neuphonic-0.1.0.dist-info/RECORD,,

livekit_plugins_neuphonic-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (76.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

livekit_plugins_neuphonic-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+livekit