autobyteus 1.1.5__py3-none-any.whl → 1.1.7__py3-none-any.whl

autobyteus/multimedia/audio/api/gemini_audio_client.py

@@ -0,0 +1,219 @@
+import asyncio
+import base64
+import logging
+import os
+import uuid
+import wave
+from typing import Optional, Dict, Any, TYPE_CHECKING, List
+
+# Old/legacy Gemini SDK (as requested)
+import google.generativeai as genai
+
+from autobyteus.multimedia.audio.base_audio_client import BaseAudioClient
+from autobyteus.multimedia.utils.response_types import SpeechGenerationResponse
+
+if TYPE_CHECKING:
+    from autobyteus.multimedia.audio.audio_model import AudioModel
+    from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+
+logger = logging.getLogger(__name__)
+
+
+def _save_audio_bytes_to_wav(
+    pcm_bytes: bytes,
+    channels: int = 1,
+    rate: int = 24000,
+    sample_width: int = 2,
+) -> str:
+    """
+    Save raw PCM (s16le) audio bytes to a temporary WAV file and return the file path.
+
+    Gemini TTS models output mono, 24 kHz, 16-bit PCM by default.
+    """
+    temp_dir = "/tmp/autobyteus_audio"
+    os.makedirs(temp_dir, exist_ok=True)
+    file_path = os.path.join(temp_dir, f"{uuid.uuid4()}.wav")
+
+    try:
+        with wave.open(file_path, "wb") as wf:
+            wf.setnchannels(channels)
+            wf.setsampwidth(sample_width)  # 2 bytes => 16-bit
+            wf.setframerate(rate)
+            wf.writeframes(pcm_bytes)
+        logger.info("Successfully saved generated audio to %s", file_path)
+        return file_path
+    except Exception as e:
+        logger.error("Failed to save audio to WAV file at %s: %s", file_path, e)
+        raise
+
+
+def _extract_inline_audio_bytes(response) -> bytes:
+    """
+    Extract inline audio bytes from a google.generativeai response.
+
+    The legacy SDK returns a Response object with candidates -> content -> parts[0].inline_data.data.
+    Depending on version, `.data` can be bytes or base64-encoded str.
+    """
+    try:
+        # Access the first candidate's first part's inline_data
+        part = response.candidates[0].content.parts[0]
+        inline = getattr(part, "inline_data", None)
+        if not inline or not hasattr(inline, "data"):
+            raise ValueError("No inline audio data found in response.")
+        data = inline.data
+        if isinstance(data, bytes):
+            return data
+        if isinstance(data, str):
+            return base64.b64decode(data)
+        raise TypeError(f"Unexpected inline_data.data type: {type(data)}")
+    except Exception as e:
+        logger.error("Failed to extract audio from response: %s", e)
+        raise
+
+
+class GeminiAudioClient(BaseAudioClient):
+    """
+    An audio client that uses Google's Gemini models for TTS via the *legacy* SDK
+    (`google.generativeai`).
+
+    Usage notes:
+      - Ensure your model value is a TTS-capable model (e.g. "gemini-2.5-flash-preview-tts"
+        or "gemini-2.5-pro-preview-tts").
+      - Single-speaker is default. For simple usage, provide `voice_name` (e.g. "Kore", "Puck")
+        in MultimediaConfig or generation_config.
+      - Multi-speaker preview exists in the API; if you want it, pass:
+          generation_config = {
+              "mode": "multi-speaker",
+              "speakers": [
+                  {"speaker": "Alice", "voice_name": "Kore"},
+                  {"speaker": "Bob",   "voice_name": "Puck"},
+              ]
+          }
+        and make sure your prompt contains lines for each named speaker.
+    """
+
+    def __init__(self, model: "AudioModel", config: "MultimediaConfig"):
+        super().__init__(model, config)
+        api_key = os.getenv("GEMINI_API_KEY")
+        if not api_key:
+            raise ValueError("Please set the GEMINI_API_KEY environment variable.")
+
+        try:
+            # Legacy library uses a global configure call
+            genai.configure(api_key=api_key)
+            # Create a GenerativeModel handle
+            self._model = genai.GenerativeModel(self.model.value or "gemini-2.5-flash-preview-tts")
+            logger.info("GeminiAudioClient (legacy SDK) configured for model '%s'.", self.model.value)
+        except Exception as e:
+            logger.error("Failed to configure Gemini client: %s", e)
+            raise RuntimeError(f"Failed to configure Gemini client: {e}")
+
+    @staticmethod
+    def _build_single_speaker_generation_config(voice_name: str) -> Dict[str, Any]:
+        """
+        Build generation_config for single-speaker TTS in the legacy SDK.
+        Key bits:
+          - response_mime_type => request audio
+          - speech_config.voice_config.prebuilt_voice_config.voice_name => set the voice
+        """
+        return {
+            "response_mime_type": "audio/pcm",
+            "speech_config": {
+                "voice_config": {
+                    "prebuilt_voice_config": {
+                        "voice_name": voice_name,
+                    }
+                }
+            },
+        }
+
+    @staticmethod
+    def _build_multi_speaker_generation_config(speakers: List[Dict[str, str]]) -> Dict[str, Any]:
+        """
+        Build generation_config for multi-speaker TTS (preview).
+        `speakers` = [{"speaker": "...", "voice_name": "..."}, ...]
+        """
+        speaker_voice_configs = []
+        for s in speakers:
+            spk = s.get("speaker")
+            vname = s.get("voice_name")
+            if not spk or not vname:
+                raise ValueError("Each speaker must include 'speaker' and 'voice_name'.")
+            speaker_voice_configs.append(
+                {
+                    "speaker": spk,
+                    "voice_config": {
+                        "prebuilt_voice_config": {
+                            "voice_name": vname,
+                        }
+                    },
+                }
+            )
+        return {
+            "response_mime_type": "audio/pcm",
+            "speech_config": {
+                "multi_speaker_voice_config": {
+                    "speaker_voice_configs": speaker_voice_configs
+                }
+            },
+        }
+
+    async def generate_speech(
+        self,
+        prompt: str,
+        generation_config: Optional[Dict[str, Any]] = None
+    ) -> SpeechGenerationResponse:
+        """
+        Generates spoken audio from text using a Gemini TTS model through the legacy SDK.
+
+        Implementation details:
+          - We call `GenerativeModel.generate_content(...)` with a `generation_config`
+            that asks for AUDIO and sets the voice settings.
+          - The legacy SDK call is synchronous; we offload to a worker thread.
+        """
+        try:
+            logger.info("Generating speech with Gemini TTS (legacy SDK) model '%s'...", self.model.value)
+
+            # Merge base config with per-call overrides
+            final_cfg = self.config.to_dict().copy()
+            if generation_config:
+                final_cfg.update(generation_config or {})
+
+            # Style instructions: prepend if provided
+            style_instructions = final_cfg.get("style_instructions")
+            final_prompt = f"{style_instructions}: {prompt}" if style_instructions else prompt
+            logger.debug("Final prompt for TTS (truncated): '%s...'", final_prompt[:160])
+
+            # Mode & voice
+            mode = final_cfg.get("mode", "single-speaker")
+            default_voice = final_cfg.get("voice_name", "Kore")
+
+            if mode == "multi-speaker":
+                speakers = final_cfg.get("speakers")
+                if not speakers or not isinstance(speakers, list):
+                    raise ValueError(
+                        "For multi-speaker mode, provide generation_config['speakers'] "
+                        "as a list of {'speaker': <name>, 'voice_name': <prebuilt voice>}."
+                    )
+                gen_config = self._build_multi_speaker_generation_config(speakers)
+            else:
+                gen_config = self._build_single_speaker_generation_config(default_voice)
+
+            # Run the blocking gen call in a thread so this coroutine stays non-blocking
+            response = await asyncio.to_thread(
+                self._model.generate_content,
+                final_prompt,
+                generation_config=gen_config,
+            )
+
+            audio_pcm = _extract_inline_audio_bytes(response)
+            audio_path = _save_audio_bytes_to_wav(audio_pcm)
+
+            return SpeechGenerationResponse(audio_urls=[audio_path])
+
+        except Exception as e:
+            logger.error("Error during Google Gemini speech generation (legacy SDK): %s", str(e))
+            raise ValueError(f"Google Gemini speech generation failed: {str(e)}")
+
+    async def cleanup(self):
+        logger.debug("GeminiAudioClient cleanup called (legacy SDK; nothing to release).")

autobyteus/multimedia/audio/audio_client_factory.py

@@ -0,0 +1,120 @@
+import logging
+from typing import Dict, Optional
+
+from autobyteus.multimedia.audio.base_audio_client import BaseAudioClient
+from autobyteus.multimedia.audio.audio_model import AudioModel
+from autobyteus.multimedia.providers import MultimediaProvider
+from autobyteus.multimedia.audio.api.gemini_audio_client import GeminiAudioClient
+from autobyteus.multimedia.audio.autobyteus_audio_provider import AutobyteusAudioModelProvider
+from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+from autobyteus.utils.singleton import SingletonMeta
+
+logger = logging.getLogger(__name__)
+
+GEMINI_TTS_VOICES = [
+    "Zephyr", "Puck", "Charon", "Kore", "Fenrir", "Leda", "Orus", "Aoede", 
+    "Callirrhoe", "Autonoe", "Enceladus", "Iapetus", "Umbriel", "Algieba", 
+    "Despina", "Erinome", "Algenib", "Rasalgethi", "Laomedeia", "Achernar", 
+    "Alnilam", "Schedar", "Gacrux", "Pulcherrima", "Achird", "Zubenelgenubi", 
+    "Vindemiatrix", "Sadachbia", "Sadaltager", "Sulafat"
+]
+
+class AudioClientFactory(metaclass=SingletonMeta):
+    """
+    A factory for creating instances of audio clients based on registered AudioModels.
+    """
+    _models_by_identifier: Dict[str, AudioModel] = {}
+    _initialized = False
+
+    @staticmethod
+    def ensure_initialized():
+        """Ensures the factory is initialized before use."""
+        if not AudioClientFactory._initialized:
+            AudioClientFactory._initialize_registry()
+            AudioClientFactory._initialized = True
+
+    @staticmethod
+    def reinitialize():
+        """Reinitializes the model registry, clearing all models and re-discovering them."""
+        logger.info("Reinitializing Audio model registry...")
+        AudioClientFactory._initialized = False
+        AudioClientFactory._models_by_identifier.clear()
+        AudioClientFactory.ensure_initialized()
+        logger.info("Audio model registry reinitialized successfully.")
+
+    @staticmethod
+    def _initialize_registry():
+        """Initializes the registry with built-in audio models."""
+        
+        # Google Gemini Audio Models
+        gemini_tts_model = AudioModel(
+            name="gemini-2.5-flash-tts",
+            value="gemini-2.5-flash-preview-tts",
+            provider=MultimediaProvider.GOOGLE,
+            client_class=GeminiAudioClient,
+            parameter_schema={
+                "mode": { 
+                    "type": "string", 
+                    "default": "single-speaker", 
+                    "allowed_values": ["single-speaker", "multi-speaker"],
+                    "description": "The speech generation mode. 'single-speaker' for a consistent voice, or 'multi-speaker' to assign different voices to speakers identified in the prompt."
+                },
+                "voice_name": { 
+                    "type": "string", 
+                    "default": "Kore",
+                    "allowed_values": GEMINI_TTS_VOICES,
+                    "description": "The voice to use for single-speaker generation."
+                },
+                "style_instructions": {
+                    "type": "string",
+                    "description": "Optional instructions on the style of speech, e.g., 'Say this in a dramatic whisper'."
+                },
+                "speaker_mapping": {
+                    "type": "object",
+                    "description": "Required for multi-speaker mode. An object mapping speaker names from the prompt (e.g., 'Joe') to a voice name (e.g., 'Puck')."
+                }
+            }
+        )
+
+        models_to_register = [
+            gemini_tts_model,
+        ]
+        
+        for model in models_to_register:
+            AudioClientFactory.register_model(model)
+        
+        logger.info("Default API-based audio models registered.")
+        
+        # Discover models from remote Autobyteus servers
+        AutobyteusAudioModelProvider.discover_and_register()
+
+    @staticmethod
+    def register_model(model: AudioModel):
+        """Registers a new audio model."""
+        identifier = model.model_identifier
+        if identifier in AudioClientFactory._models_by_identifier:
+            logger.warning(f"Audio model '{identifier}' is already registered. Overwriting.")
+        
+        if not isinstance(model.provider, MultimediaProvider):
+            try:
+                model.provider = MultimediaProvider(model.provider)
+            except ValueError:
+                logger.error(f"Cannot register model '{identifier}' with unknown provider '{model.provider}'.")
+                return
+
+        AudioClientFactory._models_by_identifier[identifier] = model
+
+    @staticmethod
+    def create_audio_client(model_identifier: str, config_override: Optional[MultimediaConfig] = None) -> BaseAudioClient:
+        """Creates an instance of a registered audio client for a specific model."""
+        AudioClientFactory.ensure_initialized()
+        
+        model = AudioClientFactory._models_by_identifier.get(model_identifier)
+        if not model:
+            raise ValueError(f"No audio model registered with the name '{model_identifier}'. "
+                             f"Available models: {list(AudioClientFactory._models_by_identifier.keys())}")
+        
+        logger.info(f"Creating instance of audio client for model '{model_identifier}'.")
+        return model.create_client(config_override)
+
+audio_client_factory = AudioClientFactory()

autobyteus/multimedia/audio/audio_model.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+import logging
+from typing import TYPE_CHECKING, Type, Optional, Iterator, Dict, Any
+from urllib.parse import urlparse
+
+from autobyteus.multimedia.providers import MultimediaProvider
+from autobyteus.multimedia.runtimes import MultimediaRuntime
+from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+
+if TYPE_CHECKING:
+    from autobyteus.multimedia.audio.base_audio_client import BaseAudioClient
+
+logger = logging.getLogger(__name__)
+
+class AudioModelMeta(type):
+    """
+    Metaclass for AudioModel to allow discovery and access like an Enum.
+    """
+    def __iter__(cls) -> Iterator[AudioModel]:
+        from autobyteus.multimedia.audio.audio_client_factory import AudioClientFactory
+        AudioClientFactory.ensure_initialized()
+        for model in AudioClientFactory._models_by_identifier.values():
+            yield model
+
+    def __getitem__(cls, name_or_identifier: str) -> AudioModel:
+        from autobyteus.multimedia.audio.audio_client_factory import AudioClientFactory
+        AudioClientFactory.ensure_initialized()
+        model = AudioClientFactory._models_by_identifier.get(name_or_identifier)
+        if model:
+            return model
+        available_models = list(AudioClientFactory._models_by_identifier.keys())
+        raise KeyError(f"Audio model '{name_or_identifier}' not found. Available models: {available_models}")
+
+    def __len__(cls) -> int:
+        from autobyteus.multimedia.audio.audio_client_factory import AudioClientFactory
+        AudioClientFactory.ensure_initialized()
+        return len(AudioClientFactory._models_by_identifier)
+
+
+class AudioModel(metaclass=AudioModelMeta):
+    """
+    Represents a single audio model's metadata.
+    """
+    def __init__(
+        self,
+        name: str,
+        value: str,
+        provider: MultimediaProvider,
+        client_class: Type["BaseAudioClient"],
+        parameter_schema: Optional[Dict[str, Any]] = None,
+        runtime: MultimediaRuntime = MultimediaRuntime.API,
+        host_url: Optional[str] = None
+    ):
+        self.name = name
+        self.value = value
+        self.provider = provider
+        self.client_class = client_class
+        self.runtime = runtime
+        self.host_url = host_url
+        self.parameter_schema = parameter_schema if parameter_schema else {}
+
+        # Automatically build default_config from the schema's default values
+        default_params = {
+            key: meta.get("default")
+            for key, meta in self.parameter_schema.items()
+            if "default" in meta
+        }
+        self.default_config = MultimediaConfig(params=default_params)
+
+    @property
+    def model_identifier(self) -> str:
+        """Returns the unique identifier for the model."""
+        if self.runtime == MultimediaRuntime.AUTOBYTEUS and self.host_url:
+            try:
+                host = urlparse(self.host_url).hostname
+                return f"{self.name}@{host}"
+            except Exception:
+                return f"{self.name}@{self.host_url}" # Fallback
+        return self.name
+
+    def create_client(self, config_override: Optional[MultimediaConfig] = None) -> "BaseAudioClient":
+        """
+        Instantiates the client class for this model.
+        """
+        config_to_use = self.default_config
+        if config_override:
+            from copy import deepcopy
+            config_to_use = deepcopy(self.default_config)
+            config_to_use.merge_with(config_override)
+        
+        return self.client_class(model=self, config=config_to_use)
+
+    def __repr__(self):
+        return (
+            f"AudioModel(identifier='{self.model_identifier}', "
+            f"provider='{self.provider.name}', runtime='{self.runtime.value}')"
+        )

autobyteus/multimedia/audio/autobyteus_audio_provider.py

@@ -0,0 +1,108 @@
+import logging
+from typing import List
+import os
+from urllib.parse import urlparse
+
+from autobyteus_llm_client import AutobyteusClient
+from autobyteus.multimedia.audio.api.autobyteus_audio_client import AutobyteusAudioClient
+from autobyteus.multimedia.audio.audio_model import AudioModel
+from autobyteus.multimedia.providers import MultimediaProvider
+from autobyteus.multimedia.runtimes import MultimediaRuntime
+
+logger = logging.getLogger(__name__)
+
+class AutobyteusAudioModelProvider:
+    """
+    Discovers and registers audio models from remote Autobyteus server instances.
+    """
+    DEFAULT_SERVER_URL = 'http://localhost:8000'
+
+    @staticmethod
+    def _get_hosts() -> List[str]:
+        """Gets Autobyteus server hosts from env vars."""
+        hosts_str = os.getenv('AUTOBYTEUS_LLM_SERVER_HOSTS')
+        if hosts_str:
+            return [host.strip() for host in hosts_str.split(',')]
+        
+        legacy_host = os.getenv('AUTOBYTEUS_LLM_SERVER_URL')
+        if legacy_host:
+            return [legacy_host]
+            
+        return [AutobyteusAudioModelProvider.DEFAULT_SERVER_URL]
+
+    @staticmethod
+    def discover_and_register():
+        """Discover and register audio models from all configured hosts."""
+        try:
+            from autobyteus.multimedia.audio.audio_client_factory import AudioClientFactory
+
+            hosts = AutobyteusAudioModelProvider._get_hosts()
+            total_registered_count = 0
+
+            for host_url in hosts:
+                if not AutobyteusAudioModelProvider.is_valid_url(host_url):
+                    logger.error(f"Invalid Autobyteus host URL for audio model discovery: {host_url}, skipping.")
+                    continue
+                
+                logger.info(f"Discovering audio models from host: {host_url}")
+                client = None
+                try:
+                    client = AutobyteusClient(server_url=host_url)
+                    response = client.get_available_audio_models_sync()
+                except Exception as e:
+                    logger.warning(f"Could not fetch audio models from Autobyteus server at {host_url}: {e}")
+                    continue
+                finally:
+                    if client:
+                        client.sync_client.close()
+
+                if not response.get('models'):
+                    logger.info(f"No audio models found on host {host_url}.")
+                    continue
+
+                models = response.get('models', [])
+                host_registered_count = 0
+                for model_info in models:
+                    try:
+                        if not all(k in model_info for k in ["name", "value", "provider"]):
+                            logger.warning(f"Skipping malformed audio model from {host_url}: {model_info}")
+                            continue
+
+                        if "parameter_schema" not in model_info:
+                             logger.debug(f"Skipping model from {host_url} as it lacks a parameter schema, likely not an audio model: {model_info.get('name')}")
+                             continue
+
+                        audio_model = AudioModel(
+                            name=model_info["name"],
+                            value=model_info["value"],
+                            provider=MultimediaProvider(model_info["provider"]),
+                            client_class=AutobyteusAudioClient,
+                            runtime=MultimediaRuntime.AUTOBYTEUS,
+                            host_url=host_url,
+                            parameter_schema=model_info.get("parameter_schema")
+                        )
+                        
+                        AudioClientFactory.register_model(audio_model)
+                        host_registered_count += 1
+                        
+                    except Exception as e:
+                        logger.error(f"Failed to register audio model '{model_info.get('name')}' from {host_url}: {e}")
+                
+                if host_registered_count > 0:
+                    logger.info(f"Registered {host_registered_count} audio models from Autobyteus host {host_url}")
+                total_registered_count += host_registered_count
+            
+            if total_registered_count > 0:
+                 logger.info(f"Finished Autobyteus audio model discovery. Total models registered: {total_registered_count}")
+
+        except Exception as e:
+            logger.error(f"An unexpected error occurred during Autobyteus audio model discovery: {e}", exc_info=True)
+
+    @staticmethod
+    def is_valid_url(url: str) -> bool:
+        """Validate URL format"""
+        try:
+            result = urlparse(url)
+            return all([result.scheme, result.netloc])
+        except Exception:
+            return False

autobyteus/multimedia/audio/base_audio_client.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any, List, TYPE_CHECKING
+from autobyteus.multimedia.utils.response_types import SpeechGenerationResponse
+
+if TYPE_CHECKING:
+    from autobyteus.multimedia.audio.audio_model import AudioModel
+    from autobyteus.multimedia.utils.multimedia_config import MultimediaConfig
+
+
+class BaseAudioClient(ABC):
+    """
+    Abstract base class for audio clients that connect to models for audio generation.
+    """
+    def __init__(self, model: "AudioModel", config: "MultimediaConfig"):
+        self.model = model
+        self.config = config
+
+    @abstractmethod
+    async def generate_speech(
+        self,
+        prompt: str,
+        generation_config: Optional[Dict[str, Any]] = None
+    ) -> SpeechGenerationResponse:
+        """
+        Generates spoken audio from text (Text-to-Speech).
+
+        Args:
+            prompt (str): The text to be converted to speech.
+            generation_config (Optional[Dict[str, Any]]): Provider-specific parameters
+                                                        (e.g., voice_name, speaker_mapping).
+
+        Returns:
+            SpeechGenerationResponse: An object containing URLs or paths to the generated audio files.
+        """
+        pass
+
+    async def cleanup(self):
+        """Optional cleanup method for resources like network clients."""
+        pass

autobyteus/multimedia/image/__init__.py

@@ -0,0 +1,11 @@
+from .image_client_factory import image_client_factory, ImageClientFactory
+from .image_model import ImageModel
+from .base_image_client import BaseImageClient
+from .api import *
+
+__all__ = [
+    "image_client_factory",
+    "ImageClientFactory",
+    "ImageModel",
+    "BaseImageClient",
+]

autobyteus/multimedia/image/api/__init__.py

@@ -0,0 +1,9 @@
+from .autobyteus_image_client import AutobyteusImageClient
+from .gemini_image_client import GeminiImageClient
+from .openai_image_client import OpenAIImageClient
+
+__all__ = [
+    "AutobyteusImageClient",
+    "GeminiImageClient",
+    "OpenAIImageClient",
+]