sinapsis-speech 0.1.0__py3-none-any.whl

sinapsis_elevenlabs/src/sinapsis_elevenlabs/helpers/env_var_keys.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+from typing import Any
+
+from pydantic import BaseModel
+from sinapsis_core.utils.env_var_keys import EnvVarEntry, doc_str, return_docs_for_vars
+
+
+class _ElevenlabsKeys(BaseModel):
+    """
+    Env vars for Elevenlabs
+    """
+
+    ELEVENLABS_API_KEY: EnvVarEntry = EnvVarEntry(
+        var_name="ELEVENLABS_API_KEY",
+        default_value=" ",
+        allowed_values=None,
+        description="set api key for Elevenlabs",
+    )
+
+
+ElevenlabsEnvVars = _ElevenlabsKeys()
+
+doc_str = return_docs_for_vars(ElevenlabsEnvVars, docs=doc_str, string_for_doc="""Elevenlabs env vars available: \n""")
+__doc__ = doc_str
+
+
+def __getattr__(name: str) -> Any:
+    """to use as an import, when updating the value is not important"""
+    if name in ElevenlabsEnvVars.model_fields:
+        return ElevenlabsEnvVars.model_fields[name].default.value
+
+    raise AttributeError(f"Agent does not have `{name}` env var")
+
+
+_all__ = (*list(ElevenlabsEnvVars.model_fields.keys()), "ElevenlabsEnvVars")

sinapsis_elevenlabs/src/sinapsis_elevenlabs/helpers/voice_utils.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+from elevenlabs import VoiceSettings
+from elevenlabs.client import DEFAULT_VOICE, ElevenLabs, VoiceId, VoiceName
+from sinapsis_core.data_containers.data_packet import TextPacket
+from sinapsis_core.utils.logging_utils import sinapsis_logger
+
+
+def create_voice_settings(settings: VoiceSettings) -> VoiceSettings | None:
+    """
+    Creates or updates a `VoiceSettings` object based on the provided settings.
+
+    This function attempts to create or update a `VoiceSettings` object using the provided
+    `VoiceSettings` instance. If any of the fields in the settings contain `None`,
+    the corresponding field is populated with a default value from `DEFAULT_VOICE.settings`.
+    If all fields are valid (i.e., none are `None`), the provided `settings` object is returned unchanged.
+
+    If the settings argument is `None` or if no valid settings are provided, the function
+    returns `None`.
+
+    Args:
+        settings (VoiceSettings): An instance of `VoiceSettings` containing the settings to be applied.
+            This object may have fields with `None` values that should be replaced with default values.
+
+    Returns:
+        VoiceSettings: A `VoiceSettings` object created or updated with the provided settings. If any field
+                       was `None`, it is updated with default values. If the settings are invalid or empty,
+                       `None` is returned.
+    """
+    if settings:
+        settings_dict = settings.model_dump()
+        if any(value is None for value in settings_dict.values()):
+            for field, value in settings_dict.items():
+                if value is None:
+                    settings_dict[field] = getattr(DEFAULT_VOICE.settings, field)
+
+            return VoiceSettings(**settings_dict)
+        else:
+            return settings
+    return None
+
+
+def get_voice_id(client: ElevenLabs, voice: VoiceId | VoiceName) -> VoiceId:
+    """
+    Resolves the voice ID for a given voice name or ID.
+
+    This function searches through available voices from the ElevenLabs API
+    to match the provided voice name or ID. If the specified voice is not found,
+    it logs the error and returns the first available voice ID as a fallback.
+
+    Args:
+        client (ElevenLabs): The ElevenLabs API client instance.
+        voice (VoiceId | VoiceName): The name or ID of the desired voice.
+
+    Returns:
+        VoiceId: The resolved voice ID.
+
+    Raises:
+        ValueError: If no voices are available to resolve.
+    """
+    try:
+        voices = client.voices.get_all().voices
+        for v in voices:
+            if voice == v.name or voice == v.voice_id:
+                sinapsis_logger.debug("Voice '%s' resolved to ID: %s", voice, v.voice_id)
+                return v.voice_id
+
+        sinapsis_logger.error("Voice '%s' is not available.", voice)
+        if voices:
+            sinapsis_logger.info("Returning default voice ID: %s", voices[0].voice_id)
+            return voices[0].voice_id
+
+        raise ValueError("No available voices to resolve. Ensure the client is configured correctly.")
+    except Exception as e:
+        sinapsis_logger.error("Error resolving voice ID: %s", e)
+        raise
+
+
+def load_input_text(input_data: list[TextPacket]) -> str:
+    """Loads and concatenates the text content from a list of TextPacket objects."""
+    return "".join([item.content for item in input_data])

sinapsis_elevenlabs/src/sinapsis_elevenlabs/templates/__init__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+import importlib
+from typing import Callable
+
+_root_lib_path = "sinapsis_elevenlabs.templates"
+
+_template_lookup = {
+    "ElevenLabsTTS": f"{_root_lib_path}.elevenlabs_tts",
+    "ElevenLabsVoiceGeneration": f"{_root_lib_path}.elevenlabs_voice_generation",
+}
+
+
+def __getattr__(name: str) -> Callable:
+    if name in _template_lookup:
+        module = importlib.import_module(_template_lookup[name])
+        return getattr(module, name)
+
+    raise AttributeError(f"template `{name}` not found in {_root_lib_path}")
+
+
+__all__ = list(_template_lookup.keys())

sinapsis_elevenlabs/src/sinapsis_elevenlabs/templates/elevenlabs_base.py

@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+"""Base template for ElevenLabs speech synthesis"""
+
+import abc
+import os
+import uuid
+from io import BytesIO
+from typing import IO, Iterator, Literal
+
+from elevenlabs import Voice, VoiceSettings, save
+from elevenlabs.client import ElevenLabs, VoiceId, VoiceName
+from sinapsis_core.data_containers.data_packet import AudioPacket, DataContainer, Packet
+from sinapsis_core.template_base.template import (
+    Template,
+    TemplateAttributes,
+    TemplateAttributeType,
+)
+from sinapsis_core.utils.env_var_keys import SINAPSIS_CACHE_DIR
+
+from sinapsis_elevenlabs.helpers.env_var_keys import ELEVENLABS_API_KEY
+
+RESPONSE_TYPE = Iterator[bytes] | list[bytes] | list[Iterator[bytes]] | None
+
+
+class ElevenLabsBase(Template, abc.ABC):
+    """
+    Base template to perform audio generation tasks using the Elevenlabs package.
+
+    The template takes as attributes the elevenlabs api key, the voice for the generated audio,
+    settings associated with the audio (such as stability, style, etc.), the model to be used,
+    the format for the audio, the path, etc. It implements methods to process the
+    DataContainer, initialize the Elevenlabs client, perform the inference,
+    and store the audio.
+
+    """
+
+    PACKET_TYPE_NAME: str = "texts"
+
+    class AttributesBaseModel(TemplateAttributes):
+        """
+        Attributes for ElevenLabs Base Class.
+        Args:
+            api_key (str): The API key to authenticate with ElevenLabs' API.
+            voice (str|elevenlabs.Voice): The voice to use for speech synthesis. This can be a voice ID (str),
+                a voice name (str) or an elevenlabs voice object (Voice).
+            voice_settings (VoiceSettings): A dictionary of settings that control the behavior of the voice.
+                - stability (float)
+                - similarity_boost (float)
+                - style (float)
+                - use_speaker_boost (bool)
+            model (Literal): The model identifier to use for speech synthesis.
+            output_format (Literal): The output audio format and quality. Options include:
+                ["mp3_22050_32", "mp3_44100_32", "mp3_44100_64", "mp3_44100_96", "mp3_44100_128",
+                "mp3_44100_192", "pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100", "ulaw_8000"]
+            output_folder (str): The folder where generated audio files will be saved.
+            stream (bool): If True, the audio is returned as a stream; otherwise, saved to a file.
+        """
+
+        api_key: str | None = None
+        voice: VoiceId | VoiceName | Voice = None
+        voice_settings: VoiceSettings | None = None
+        model: Literal[
+            "eleven_turbo_v2_5",
+            "eleven_multilingual_v2",
+            "eleven_turbo_v2",
+            "eleven_monolingual_v1",
+            "eleven_multilingual_v1",
+            "eleven_english_sts_v2",
+            "eleven_multilingual_sts_v2",
+        ] = "eleven_turbo_v2_5"
+        output_format: Literal[
+            "mp3_22050_32",
+            "mp3_44100_32",
+            "mp3_44100_64",
+            "mp3_44100_96",
+            "mp3_44100_128",
+            "mp3_44100_192",
+            "pcm_16000",
+            "pcm_22050",
+            "pcm_24000",
+            "pcm_44100",
+            "ulaw_8000",
+        ] = "mp3_44100_128"
+        output_folder: str = os.path.join(SINAPSIS_CACHE_DIR, "elevenlabs", "audios")
+        stream: bool = False
+
+    def __init__(self, attributes: TemplateAttributeType) -> None:
+        """Initializes the ElevenLabs API client with the given attributes."""
+        super().__init__(attributes)
+        os.makedirs(self.attributes.output_folder, exist_ok=True)
+        self.client = self.init_elevenlabs_client()
+
+    def init_elevenlabs_client(self) -> ElevenLabs:
+        """Resets client object"""
+        key = self.attributes.api_key if self.attributes.api_key else ELEVENLABS_API_KEY
+        return ElevenLabs(api_key=key)
+
+    def reset_state(self) -> None:
+        """Resets state of model"""
+        self.client = self.init_elevenlabs_client()
+
+    @abc.abstractmethod
+    def synthesize_speech(self, input_data: list[Packet]) -> RESPONSE_TYPE:
+        """Abstract method for ElevenLabs speech synthesis."""
+
+    def _save_audio(self, response: Iterator[bytes] | bytes, file_format: str) -> str:
+        """Saves the audio to a file and returns the file path."""
+        output_file = os.path.join(self.attributes.output_folder, f"{uuid.uuid4()}.{file_format}")
+        try:
+            save(response, output_file)
+            self.logger.info(f"Audio saved to: {output_file}")
+            return output_file
+        except OSError as e:
+            self.logger.error(f"File system error while saving speech to file: {e}")
+            raise
+
+    def _generate_audio_stream(self, response: Iterator[bytes] | bytes) -> IO[bytes]:
+        """Generates and returns the audio stream."""
+        audio_stream = BytesIO()
+        try:
+            if isinstance(response, Iterator):
+                for chunk in response:
+                    if chunk:
+                        audio_stream.write(chunk)
+            elif isinstance(response, bytes):
+                audio_stream.write(response)
+            else:
+                raise TypeError(f"Unsupported response type: {type(response)}")
+
+            audio_stream.seek(0)
+            self.logger.info("Returning audio stream")
+            return audio_stream
+        except IOError as e:
+            self.logger.error(f"I/O error while processing the audio stream: {e}")
+            raise
+        except ValueError as e:
+            self.logger.error(f"Value error while processing audio chunks: {e}")
+            raise
+
+    def _process_audio_output(self, response: Iterator[bytes] | bytes) -> str | IO[bytes]:
+        """Processes a single audio output (either stream or file)."""
+        if self.attributes.stream:
+            return self._generate_audio_stream(response)
+        else:
+            file_format = "mp3" if "mp3" in self.attributes.output_format else "wav"
+            return self._save_audio(response, file_format)
+
+    def generate_speech(self, input_data: list[Packet]) -> list[str | IO[bytes]] | None:
+        """Generates speech and saves it to a file."""
+        responses: RESPONSE_TYPE = self.synthesize_speech(input_data)
+        if not responses:
+            return None
+
+        if isinstance(responses, Iterator):
+            responses = [responses]
+
+        audio_outputs = [self._process_audio_output(response) for response in responses]
+        return audio_outputs
+
+    def _handle_streaming_output(self, audio_outputs: list[str | IO[bytes]]) -> list[AudioPacket]:
+        """Handles audio stream output by adding it to the container as AudioPackets."""
+        generated_audios: list[AudioPacket] = []
+        sample_rate = int(self.attributes.output_format.split("_")[1])
+        for audio_output in audio_outputs:
+            audio_packet = AudioPacket(
+                content=audio_output,
+                sample_rate=sample_rate,
+            )
+            generated_audios.append(audio_packet)
+        return generated_audios
+
+    def _handle_audio_outputs(self, audio_outputs: list[str | IO[bytes]], container: DataContainer) -> None:
+        """Handles the audio outputs by appending to the container based on the output type (stream or file)."""
+        if self.attributes.stream:
+            container.audios = container.audios or []
+            container.audios.extend(self._handle_streaming_output(audio_outputs))
+        else:
+            self._set_generic_data(container, audio_outputs)
+
+    def execute(self, container: DataContainer) -> DataContainer:
+        """
+        Processes the input data and generates a speech output.
+        Depending on the configuration, either a file or a stream of audio is
+        generated and added to the provided `container`.
+        """
+
+        if ELEVENLABS_API_KEY is None and self.attributes.api_key is None:
+            self.logger.error("Api key was not provided")
+            return container
+
+        data_packet = getattr(container, self.PACKET_TYPE_NAME)
+
+        if not data_packet:
+            self.logger.debug("No query to enter")
+            return container
+
+        audio_outputs = self.generate_speech(data_packet)
+        if not audio_outputs:
+            self.logger.error("Unable to generate speech")
+            return container
+
+        self._handle_audio_outputs(audio_outputs, container)
+
+        return container

sinapsis_elevenlabs/src/sinapsis_elevenlabs/templates/elevenlabs_tts.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""Text-To-Speech template for ElevenLabs"""
+
+from typing import Iterator, Literal
+
+from sinapsis_core.data_containers.data_packet import TextPacket
+
+from sinapsis_elevenlabs.helpers.voice_utils import (
+    create_voice_settings,
+    load_input_text,
+)
+from sinapsis_elevenlabs.templates.elevenlabs_base import ElevenLabsBase
+
+
+class ElevenLabsTTS(ElevenLabsBase):
+    """Template to interact with ElevenLabs text-to-speech API.
+
+    This class provides an implementation to generate speech from text using the
+    ElevenLabs text-to-speech API. It allows customization of voice, model settings,
+    and audio output format.
+
+    Usage example:
+
+    agent:
+      name: my_test_agent
+    templates:
+    - template_name: InputTemplate
+      class_name: InputTemplate
+      attributes: {}
+    - template_name: ElevenLabsTTS
+      class_name: ElevenLabsTTS
+      template_input: InputTemplate
+      attributes:
+        voice: null
+        voice_settings: null
+        model: eleven_turbo_v2_5
+        output_format: mp3_44100_128
+        output_folder: /sinapsis/cache/dir/elevenlabs/audios
+        stream: false
+
+    """
+
+    class AttributesBaseModel(ElevenLabsBase.AttributesBaseModel):
+        """Attributes specific to ElevenLabs TTS API interaction.
+
+        This class overrides the base attributes of `ElevenLabsBase` to define
+        default models specific to the ElevenLabs TTS system.
+        """
+
+        model: Literal[
+            "eleven_turbo_v2_5",
+            "eleven_multilingual_v2",
+            "eleven_turbo_v2",
+            "eleven_monolingual_v1",
+            "eleven_multilingual_v1",
+        ] = "eleven_turbo_v2_5"
+
+    def synthesize_speech(self, input_data: list[TextPacket]) -> Iterator[bytes]:
+        """
+        Sends the text to ElevenLabs API to generate speech.
+
+        This method communicates with the ElevenLabs API to generate the audio
+        response based on the provided text, voice, and model settings.
+        """
+        input_text: str = load_input_text(input_data)
+        try:
+            response: Iterator[bytes] = self.client.generate(
+                text=input_text,
+                voice=self.attributes.voice,
+                model=self.attributes.model,
+                voice_settings=create_voice_settings(self.attributes.voice_settings),
+                output_format=self.attributes.output_format,
+                stream=self.attributes.stream,
+            )
+
+            return response
+        except ValueError as e:
+            self.logger.error(f"Value error synthesizing speech: {e}")
+            raise
+        except TypeError as e:
+            self.logger.error(f"Type error in input data or parameters: {e}")
+            raise
+        except KeyError as e:
+            self.logger.error(f"Missing key in input data or settings: {e}")
+            raise

sinapsis_elevenlabs/src/sinapsis_elevenlabs/templates/elevenlabs_voice_generation.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+"""Text-To-Speech template for ElevenLabs"""
+
+import base64
+
+from sinapsis_core.data_containers.data_packet import TextPacket
+
+from sinapsis_elevenlabs.helpers.voice_utils import load_input_text
+from sinapsis_elevenlabs.templates.elevenlabs_base import ElevenLabsBase
+
+
+class ElevenLabsVoiceGeneration(ElevenLabsBase):
+    """
+    Template to generate a voice using ElevenLabs API.
+
+    The template takes the voice description as an attribute and
+    the prompt for the audio as a TextPacket stored in the DataContainer
+    and stores the generated audio in the DataContainer.
+
+    Usage example:
+
+    agent:
+      name: my_test_agent
+    templates:
+    - template_name: InputTemplate
+      class_name: InputTemplate
+      attributes: {}
+    - template_name: ElevenLabsVoiceGeneration
+      class_name: ElevenLabsVoiceGeneration
+      template_input: InputTemplate
+      attributes:
+        voice: null
+        voice_settings: null
+        model: eleven_turbo_v2_5
+        output_format: mp3_44100_128
+        output_folder: /sinapsis/cache/dir/elevenlabs/audios
+        stream: false
+        voice_description: An old British male with a raspy, deep voice. Professional,
+          relaxed and assertive
+    """
+
+    class AttributesBaseModel(ElevenLabsBase.AttributesBaseModel):
+        """
+        Attributes for voice generation in ElevenLabs API.
+        """
+
+        voice_description: str = "An old British male with a raspy, deep voice. Professional, relaxed and assertive"
+
+    def synthesize_speech(self, input_data: list[TextPacket]) -> list[bytes] | None:
+        """
+        Sends the text to ElevenLabs API to generate speech.
+
+        This method communicates with the ElevenLabs API to generate the audio
+        response based on the provided text, voice, and model settings.
+        """
+
+        input_text: str = load_input_text(input_data)
+        if len(input_text) < 100:
+            self.logger.error("The text to be spoken must be at least 100 characters long.")
+            return None
+        try:
+            voice_previews = self.client.text_to_voice.create_previews(
+                voice_description=self.attributes.voice_description,
+                text=input_text,
+            )
+
+            responses: list[bytes] = [base64.b64decode(preview.audio_base_64) for preview in voice_previews.previews]
+
+            return responses
+        except ValueError as e:
+            self.logger.error(f"Value error with voice description or input text: {e}")
+            raise
+        except TypeError as e:
+            self.logger.error(f"Type error with input data or voice preview parameters: {e}")
+            raise
+        except KeyError as e:
+            self.logger.error(f"Missing expected key in voice preview response: {e}")
+            raise