roomkit 0.1.1__py3-none-any.whl → 0.2.1__py3-none-any.whl

roomkit/voice/stt/deepgram.py

@@ -0,0 +1,214 @@
+"""Deepgram speech-to-text provider."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+import httpx
+
+from roomkit.voice.base import AudioChunk, TranscriptionResult
+from roomkit.voice.stt.base import STTProvider
+
+if TYPE_CHECKING:
+    from roomkit.models.event import AudioContent
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class DeepgramConfig:
+    """Configuration for Deepgram STT provider."""
+
+    api_key: str
+    model: str = "nova-2"
+    language: str = "en"
+    punctuate: bool = True
+    diarize: bool = False
+    smart_format: bool = True
+    filler_words: bool = False
+    # Real-time streaming options
+    interim_results: bool = True
+    endpointing: int = 300  # ms of silence to end utterance
+    vad_events: bool = True
+
+
+class DeepgramSTTProvider(STTProvider):
+    """Deepgram speech-to-text provider with streaming support."""
+
+    def __init__(self, config: DeepgramConfig) -> None:
+        self._config = config
+        self._client: httpx.AsyncClient | None = None
+
+    @property
+    def name(self) -> str:
+        return "DeepgramSTT"
+
+    def _get_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(
+                base_url="https://api.deepgram.com/v1",
+                headers={
+                    "Authorization": f"Token {self._config.api_key}",
+                    "Content-Type": "audio/wav",
+                },
+                timeout=60.0,
+            )
+        return self._client
+
+    def _build_query_params(self) -> dict[str, Any]:
+        """Build query parameters for Deepgram API."""
+        params: dict[str, Any] = {
+            "model": self._config.model,
+            "language": self._config.language,
+            "punctuate": self._config.punctuate,
+            "diarize": self._config.diarize,
+            "smart_format": self._config.smart_format,
+            "filler_words": self._config.filler_words,
+        }
+        return {k: str(v).lower() if isinstance(v, bool) else v for k, v in params.items()}
+
+    async def transcribe(self, audio: AudioContent | AudioChunk) -> str:
+        """Transcribe complete audio to text.
+
+        Args:
+            audio: Audio content (URL) or raw audio chunk.
+
+        Returns:
+            Transcribed text.
+        """
+        client = self._get_client()
+        params = self._build_query_params()
+
+        # Handle AudioContent (URL-based)
+        if hasattr(audio, "url"):
+            # Fetch audio from URL
+            async with httpx.AsyncClient() as fetch_client:
+                resp = await fetch_client.get(audio.url)
+                resp.raise_for_status()
+                audio_data = resp.content
+                content_type = resp.headers.get("content-type", "audio/wav")
+        else:
+            # Handle AudioChunk (raw bytes)
+            audio_data = audio.data
+            audio_format = getattr(audio, "format", "wav")
+
+            # For raw PCM formats, set encoding params for Deepgram
+            if audio_format in ("pcm_s16le", "linear16", "raw"):
+                content_type = "audio/raw"
+                params["encoding"] = "linear16"
+                params["sample_rate"] = getattr(audio, "sample_rate", 16000)
+                params["channels"] = getattr(audio, "channels", 1)
+            else:
+                content_type = f"audio/{audio_format}"
+
+        # Call Deepgram API
+        response = await client.post(
+            "/listen",
+            params=params,
+            content=audio_data,
+            headers={"Content-Type": content_type},
+        )
+        response.raise_for_status()
+        result = response.json()
+
+        # Extract transcript
+        try:
+            transcript: str = result["results"]["channels"][0]["alternatives"][0]["transcript"]
+            return transcript.strip()
+        except (KeyError, IndexError):
+            logger.warning("No transcript in Deepgram response: %s", result)
+            return ""
+
+    async def transcribe_stream(
+        self, audio_stream: AsyncIterator[AudioChunk]
+    ) -> AsyncIterator[TranscriptionResult]:
+        """Stream transcription with partial results using WebSocket.
+
+        Args:
+            audio_stream: Async iterator of audio chunks.
+
+        Yields:
+            TranscriptionResult with partial and final transcripts.
+        """
+        import websockets
+
+        # Build WebSocket URL with query params
+        params = self._build_query_params()
+        params["interim_results"] = str(self._config.interim_results).lower()
+        params["endpointing"] = self._config.endpointing
+        params["vad_events"] = str(self._config.vad_events).lower()
+        params["encoding"] = "linear16"
+        params["sample_rate"] = "16000"
+
+        query_string = "&".join(f"{k}={v}" for k, v in params.items())
+        ws_url = f"wss://api.deepgram.com/v1/listen?{query_string}"
+
+        headers = [("Authorization", f"Token {self._config.api_key}")]
+
+        async with websockets.connect(ws_url, additional_headers=headers) as ws:
+            # Start sender task
+            async def send_audio() -> None:
+                try:
+                    async for chunk in audio_stream:
+                        if chunk.data:
+                            await ws.send(chunk.data)
+                        if chunk.is_final:
+                            # Send close frame to signal end of audio
+                            await ws.send(b"")
+                            break
+                except Exception as e:
+                    logger.error("Error sending audio to Deepgram: %s", e)
+
+            sender_task = asyncio.create_task(send_audio())
+
+            try:
+                async for message in ws:
+                    if isinstance(message, bytes):
+                        continue
+
+                    import json
+
+                    data = json.loads(message)
+
+                    # Handle transcription results
+                    if data.get("type") == "Results":
+                        channel = data.get("channel", {})
+                        alternatives = channel.get("alternatives", [])
+                        if alternatives:
+                            alt = alternatives[0]
+                            transcript = alt.get("transcript", "")
+                            confidence = alt.get("confidence")
+                            words = alt.get("words", [])
+                            is_final = data.get("is_final", False)
+
+                            if transcript:
+                                yield TranscriptionResult(
+                                    text=transcript,
+                                    is_final=is_final,
+                                    confidence=confidence,
+                                    language=data.get("channel", {}).get("detected_language"),
+                                    words=words,
+                                )
+
+                    # Handle speech events
+                    elif data.get("type") == "SpeechStarted":
+                        logger.debug("Speech started")
+                    elif data.get("type") == "UtteranceEnd":
+                        logger.debug("Utterance ended")
+
+            finally:
+                sender_task.cancel()
+                import contextlib
+
+                with contextlib.suppress(asyncio.CancelledError):
+                    await sender_task
+
+    async def close(self) -> None:  # noqa: B027
+        """Release resources."""
+        if self._client:
+            await self._client.aclose()
+            self._client = None

roomkit/voice/stt/mock.py

@@ -0,0 +1,40 @@
+"""Mock speech-to-text provider for testing."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from typing import TYPE_CHECKING
+
+from roomkit.voice.base import AudioChunk, TranscriptionResult
+from roomkit.voice.stt.base import STTProvider
+
+if TYPE_CHECKING:
+    from roomkit.models.event import AudioContent
+
+
+class MockSTTProvider(STTProvider):
+    """Mock speech-to-text for testing."""
+
+    def __init__(self, transcripts: list[str] | None = None) -> None:
+        self.transcripts = transcripts or ["Hello", "How can I help you?"]
+        self.calls: list[AudioContent | AudioChunk] = []
+        self._index = 0
+
+    async def transcribe(self, audio: AudioContent | AudioChunk) -> str:
+        self.calls.append(audio)
+        result = self.transcripts[self._index % len(self.transcripts)]
+        self._index += 1
+        return result
+
+    async def transcribe_stream(
+        self, audio_stream: AsyncIterator[AudioChunk]
+    ) -> AsyncIterator[TranscriptionResult]:
+        chunks = []
+        async for chunk in audio_stream:
+            chunks.append(chunk)
+
+        text = self.transcripts[self._index % len(self.transcripts)]
+        self._index += 1
+        self.calls.extend(chunks)
+
+        yield TranscriptionResult(text=text, is_final=True)

roomkit/voice/tts/__init__.py

@@ -0,0 +1 @@
+"""Text-to-speech providers."""

roomkit/voice/tts/base.py

@@ -0,0 +1,58 @@
+"""Text-to-speech provider ABC."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import AsyncIterator
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from roomkit.models.event import AudioContent
+    from roomkit.voice.base import AudioChunk
+
+
+class TTSProvider(ABC):
+    """Text-to-speech provider."""
+
+    @property
+    def name(self) -> str:
+        """Provider name (e.g. 'elevenlabs', 'openai')."""
+        return self.__class__.__name__
+
+    @property
+    def default_voice(self) -> str | None:
+        """Default voice ID. Override in subclasses."""
+        return None
+
+    @abstractmethod
+    async def synthesize(
+        self, text: str, *, voice: str | None = None
+    ) -> AudioContent:
+        """Synthesize text to audio.
+
+        Args:
+            text: Text to synthesize.
+            voice: Voice ID (uses default_voice if not specified).
+
+        Returns:
+            AudioContent with URL to generated audio.
+        """
+        ...
+
+    async def synthesize_stream(
+        self, text: str, *, voice: str | None = None
+    ) -> AsyncIterator[AudioChunk]:
+        """Stream audio chunks as they're generated.
+
+        Override for providers that support streaming.
+        Default: synthesizes full audio and yields single chunk.
+        """
+        raise NotImplementedError(
+            f"{self.name} does not support streaming synthesis. "
+            "Use synthesize() instead."
+        )
+        # Make this an async generator (unreachable, but required for type)
+        yield  # pragma: no cover
+
+    async def close(self) -> None:  # noqa: B027
+        """Release resources. Override in subclasses if needed."""

roomkit/voice/tts/elevenlabs.py

@@ -0,0 +1,329 @@
+"""ElevenLabs text-to-speech provider."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncIterator
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+import httpx
+
+from roomkit.voice.base import AudioChunk
+from roomkit.voice.tts.base import TTSProvider
+
+if TYPE_CHECKING:
+    from roomkit.models.event import AudioContent
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ElevenLabsConfig:
+    """Configuration for ElevenLabs TTS provider."""
+
+    api_key: str
+    voice_id: str = "21m00Tcm4TlvDq8ikWAM"  # Rachel (default)
+    model_id: str = "eleven_multilingual_v2"
+    stability: float = 0.5
+    similarity_boost: float = 0.75
+    style: float = 0.0
+    use_speaker_boost: bool = True
+    output_format: str = "mp3_44100_128"  # mp3, pcm_16000, pcm_22050, etc.
+    # Streaming options
+    optimize_streaming_latency: int = 3  # 0-4, higher = lower latency
+
+
+@dataclass
+class ElevenLabsVoice:
+    """Voice metadata from ElevenLabs."""
+
+    voice_id: str
+    name: str
+    category: str = "premade"
+    labels: dict[str, str] = field(default_factory=dict)
+
+
+class ElevenLabsTTSProvider(TTSProvider):
+    """ElevenLabs text-to-speech provider with streaming support."""
+
+    def __init__(self, config: ElevenLabsConfig) -> None:
+        self._config = config
+        self._client: httpx.AsyncClient | None = None
+        self._voices_cache: dict[str, ElevenLabsVoice] | None = None
+
+    @property
+    def name(self) -> str:
+        return "ElevenLabsTTS"
+
+    @property
+    def default_voice(self) -> str:
+        return self._config.voice_id
+
+    def _get_client(self) -> httpx.AsyncClient:
+        if self._client is None:
+            self._client = httpx.AsyncClient(
+                base_url="https://api.elevenlabs.io/v1",
+                headers={
+                    "xi-api-key": self._config.api_key,
+                    "Content-Type": "application/json",
+                },
+                timeout=60.0,
+            )
+        return self._client
+
+    def _build_voice_settings(self) -> dict[str, float | bool]:
+        """Build voice settings for synthesis."""
+        return {
+            "stability": self._config.stability,
+            "similarity_boost": self._config.similarity_boost,
+            "style": self._config.style,
+            "use_speaker_boost": self._config.use_speaker_boost,
+        }
+
+    async def list_voices(self) -> list[ElevenLabsVoice]:
+        """List available voices from ElevenLabs."""
+        if self._voices_cache is not None:
+            return list(self._voices_cache.values())
+
+        client = self._get_client()
+        response = await client.get("/voices")
+        response.raise_for_status()
+        data = response.json()
+
+        self._voices_cache = {}
+        for voice in data.get("voices", []):
+            v = ElevenLabsVoice(
+                voice_id=voice["voice_id"],
+                name=voice["name"],
+                category=voice.get("category", "premade"),
+                labels=voice.get("labels", {}),
+            )
+            self._voices_cache[v.voice_id] = v
+
+        return list(self._voices_cache.values())
+
+    async def synthesize(
+        self, text: str, *, voice: str | None = None
+    ) -> AudioContent:
+        """Synthesize text to audio.
+
+        Args:
+            text: Text to synthesize.
+            voice: Voice ID (uses default_voice if not specified).
+
+        Returns:
+            AudioContent with URL to generated audio.
+        """
+        from roomkit.models.event import AudioContent as AudioContentModel
+
+        voice_id = voice or self._config.voice_id
+        client = self._get_client()
+
+        response = await client.post(
+            f"/text-to-speech/{voice_id}",
+            json={
+                "text": text,
+                "model_id": self._config.model_id,
+                "voice_settings": self._build_voice_settings(),
+            },
+            params={"output_format": self._config.output_format},
+        )
+        response.raise_for_status()
+
+        # ElevenLabs returns raw audio bytes
+        # We need to save/upload this somewhere to get a URL
+        # For now, return a data URL (base64 encoded)
+        import base64
+
+        audio_bytes = response.content
+        mime_type = self._get_mime_type()
+        data_url = f"data:{mime_type};base64,{base64.b64encode(audio_bytes).decode()}"
+
+        # Estimate duration (rough: ~150 words/minute, ~5 chars/word)
+        words = len(text.split())
+        duration = words / 150 * 60  # seconds
+
+        return AudioContentModel(
+            url=data_url,
+            mime_type=mime_type,
+            transcript=text,
+            duration_seconds=duration,
+        )
+
+    async def synthesize_stream(
+        self, text: str, *, voice: str | None = None
+    ) -> AsyncIterator[AudioChunk]:
+        """Stream audio chunks as they're generated.
+
+        Uses ElevenLabs streaming API for low-latency synthesis.
+
+        Args:
+            text: Text to synthesize.
+            voice: Voice ID (uses default_voice if not specified).
+
+        Yields:
+            AudioChunk with raw audio data.
+        """
+        voice_id = voice or self._config.voice_id
+        client = self._get_client()
+
+        # Use streaming endpoint
+        async with client.stream(
+            "POST",
+            f"/text-to-speech/{voice_id}/stream",
+            json={
+                "text": text,
+                "model_id": self._config.model_id,
+                "voice_settings": self._build_voice_settings(),
+            },
+            params={
+                "output_format": self._config.output_format,
+                "optimize_streaming_latency": self._config.optimize_streaming_latency,
+            },
+        ) as response:
+            response.raise_for_status()
+
+            chunk_index = 0
+            async for chunk in response.aiter_bytes(chunk_size=4096):
+                if chunk:
+                    yield AudioChunk(
+                        data=chunk,
+                        sample_rate=self._get_sample_rate(),
+                        format=self._get_audio_format(),
+                        is_final=False,
+                    )
+                    chunk_index += 1
+
+            # Send final chunk marker
+            yield AudioChunk(
+                data=b"",
+                sample_rate=self._get_sample_rate(),
+                format=self._get_audio_format(),
+                is_final=True,
+            )
+
+    async def synthesize_stream_input(
+        self, text_stream: AsyncIterator[str], *, voice: str | None = None
+    ) -> AsyncIterator[AudioChunk]:
+        """Stream audio from streaming text input.
+
+        Uses ElevenLabs WebSocket API for real-time text-to-speech.
+
+        Args:
+            text_stream: Async iterator of text chunks.
+            voice: Voice ID (uses default_voice if not specified).
+
+        Yields:
+            AudioChunk with raw audio data.
+        """
+        import asyncio
+        import json
+
+        import websockets
+
+        voice_id = voice or self._config.voice_id
+        model_id = self._config.model_id
+
+        ws_url = (
+            f"wss://api.elevenlabs.io/v1/text-to-speech/{voice_id}/stream-input"
+            f"?model_id={model_id}"
+            f"&output_format={self._config.output_format}"
+            f"&optimize_streaming_latency={self._config.optimize_streaming_latency}"
+        )
+
+        async with websockets.connect(
+            ws_url,
+            additional_headers=[("xi-api-key", self._config.api_key)],
+        ) as ws:
+            # Send initial BOS (beginning of stream) message
+            await ws.send(
+                json.dumps(
+                    {
+                        "text": " ",
+                        "voice_settings": self._build_voice_settings(),
+                        "xi_api_key": self._config.api_key,
+                    }
+                )
+            )
+
+            # Start text sender task
+            async def send_text() -> None:
+                try:
+                    async for text_chunk in text_stream:
+                        if text_chunk:
+                            await ws.send(json.dumps({"text": text_chunk}))
+                    # Send EOS (end of stream) message
+                    await ws.send(json.dumps({"text": ""}))
+                except Exception as e:
+                    logger.error("Error sending text to ElevenLabs: %s", e)
+
+            sender_task = asyncio.create_task(send_text())
+
+            try:
+                async for message in ws:
+                    if isinstance(message, str):
+                        data = json.loads(message)
+                        if "audio" in data:
+                            import base64
+
+                            audio_bytes = base64.b64decode(data["audio"])
+                            yield AudioChunk(
+                                data=audio_bytes,
+                                sample_rate=self._get_sample_rate(),
+                                format=self._get_audio_format(),
+                                is_final=data.get("isFinal", False),
+                            )
+                        elif data.get("isFinal"):
+                            yield AudioChunk(
+                                data=b"",
+                                sample_rate=self._get_sample_rate(),
+                                format=self._get_audio_format(),
+                                is_final=True,
+                            )
+                            break
+            finally:
+                sender_task.cancel()
+                import contextlib
+
+                with contextlib.suppress(asyncio.CancelledError):
+                    await sender_task
+
+    def _get_mime_type(self) -> str:
+        """Get MIME type from output format."""
+        fmt = self._config.output_format
+        if fmt.startswith("mp3"):
+            return "audio/mpeg"
+        elif fmt.startswith("pcm"):
+            return "audio/pcm"
+        elif fmt.startswith("ulaw"):
+            return "audio/basic"
+        return "audio/mpeg"
+
+    def _get_sample_rate(self) -> int:
+        """Get sample rate from output format."""
+        fmt = self._config.output_format
+        if "44100" in fmt:
+            return 44100
+        elif "22050" in fmt:
+            return 22050
+        elif "16000" in fmt:
+            return 16000
+        return 44100
+
+    def _get_audio_format(self) -> str:
+        """Get audio format string."""
+        fmt = self._config.output_format
+        if fmt.startswith("mp3"):
+            return "mp3"
+        elif fmt.startswith("pcm"):
+            return "pcm_s16le"
+        elif fmt.startswith("ulaw"):
+            return "ulaw"
+        return "mp3"
+
+    async def close(self) -> None:  # noqa: B027
+        """Release resources."""
+        if self._client:
+            await self._client.aclose()
+            self._client = None

roomkit/voice/tts/mock.py

@@ -0,0 +1,51 @@
+"""Mock text-to-speech provider for testing."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator
+from typing import TYPE_CHECKING
+from uuid import uuid4
+
+from roomkit.voice.base import AudioChunk
+from roomkit.voice.tts.base import TTSProvider
+
+if TYPE_CHECKING:
+    from roomkit.models.event import AudioContent
+
+
+class MockTTSProvider(TTSProvider):
+    """Mock text-to-speech for testing."""
+
+    def __init__(self, voice: str = "mock-voice") -> None:
+        self._default_voice = voice
+        self.calls: list[dict[str, str | None]] = []
+
+    @property
+    def default_voice(self) -> str:
+        return self._default_voice
+
+    async def synthesize(
+        self, text: str, *, voice: str | None = None
+    ) -> AudioContent:
+        from roomkit.models.event import AudioContent as AudioContentModel
+
+        self.calls.append({"text": text, "voice": voice or self._default_voice})
+        return AudioContentModel(
+            url=f"https://mock.test/audio/{uuid4().hex}.mp3",
+            mime_type="audio/mpeg",
+            transcript=text,
+            duration_seconds=len(text) * 0.05,  # ~50ms per char
+        )
+
+    async def synthesize_stream(
+        self, text: str, *, voice: str | None = None
+    ) -> AsyncIterator[AudioChunk]:
+        self.calls.append({"text": text, "voice": voice or self._default_voice})
+        # Simulate streaming with small chunks
+        words = text.split()
+        for i, word in enumerate(words):
+            yield AudioChunk(
+                data=f"mock-audio-{word}".encode(),
+                sample_rate=16000,
+                is_final=(i == len(words) - 1),
+            )