openspeechapi 0.1.0__py3-none-any.whl

openspeech/providers/stt/elevenlabs.py

@@ -0,0 +1,385 @@
+"""ElevenLabs STT provider adapter (batch + realtime WS)."""
+from __future__ import annotations
+
+import asyncio
+import base64
+import inspect
+import json
+from urllib.parse import urlencode
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from openspeech.core.base import STTProvider
+from openspeech.core.enums import AudioFormat, Capability, ExecMode, ProviderType
+from openspeech.core.models import AudioData, STTOptions, Transcription, Word
+from openspeech.core.settings import BaseSettings
+from openspeech.logging_config import logger
+
+
+def _ws_connect_with_headers(websockets_mod, url: str, headers: dict[str, str]):
+    """Compatible connect kwargs across websockets versions.
+
+    websockets>=15 uses ``additional_headers``; older versions use ``extra_headers``.
+    """
+    try:
+        sig = inspect.signature(websockets_mod.connect)
+        if "additional_headers" in sig.parameters:
+            return websockets_mod.connect(url, additional_headers=headers)
+    except Exception:
+        pass
+    return websockets_mod.connect(url, extra_headers=headers)
+
+
+@dataclass
+class ElevenLabsSTTSettings(BaseSettings):
+    api_key: str = ""
+    model_id: str = "scribe_v2"
+    realtime_model_id: str = "scribe_v2_realtime"
+    language_code: str = ""
+    diarize: bool = False
+    tag_audio_events: bool = True
+    num_speakers: int | None = None
+    timestamps_granularity: str = "word"
+    diarization_threshold: float | None = None
+    use_multi_channel: bool = False
+    no_verbatim: bool = False
+    detect_speaker_roles: bool = False
+    temperature: float | None = None
+    entity_detection: str = ""
+    realtime_audio_format: str = "pcm_16000"
+    realtime_commit_strategy: str = "vad"  # manual | vad
+    realtime_include_timestamps: bool = True
+    realtime_include_language_detection: bool = True
+    realtime_vad_silence_threshold_secs: float = 1.0
+
+
+class ElevenLabsSTT(STTProvider):
+    name = "elevenlabs-stt"
+    provider_type = ProviderType.STT
+    execution_mode = ExecMode.REMOTE
+    settings_cls = ElevenLabsSTTSettings
+    capabilities = {Capability.BATCH, Capability.STREAMING, Capability.MULTILINGUAL}
+    field_options = {
+        "model_id": ["scribe_v2", "scribe_v1"],
+        "realtime_model_id": ["scribe_v2_realtime"],
+        "realtime_audio_format": ["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"],
+        "realtime_commit_strategy": ["vad", "manual"],
+        "timestamps_granularity": ["none", "word", "character"],
+        "language_code": [
+            "",
+            "en", "zh", "ja", "ko", "es", "fr", "de", "pt", "it", "hi",
+            "id", "nl", "tr", "pl", "sv", "ar", "ru", "uk", "vi", "hu",
+            "no", "da", "fi", "cs", "el", "ro", "bg", "hr", "sk", "ms",
+            "ta", "fil",
+        ],
+        "diarize": [False, True],
+        "tag_audio_events": [True, False],
+        "no_verbatim": [False, True],
+        "detect_speaker_roles": [False, True],
+        "use_multi_channel": [False, True],
+    }
+
+    def __init__(self, settings: ElevenLabsSTTSettings | None = None) -> None:
+        self.settings = settings or ElevenLabsSTTSettings()
+        self._client: httpx.AsyncClient | None = None
+        self._owns_client: bool = True
+
+    def set_http_client(self, client) -> None:
+        self._client = client
+        self._owns_client = False
+
+    async def start(self) -> None:
+        if self._client is None:
+            self._client = httpx.AsyncClient(timeout=60.0, trust_env=False)
+            self._owns_client = True
+        logger.info("{} provider started", self.name)
+
+    async def stop(self) -> None:
+        if self._client and self._owns_client:
+            await self._client.aclose()
+        self._client = None
+        logger.info("{} provider stopped", self.name)
+
+    async def health_check(self) -> bool:
+        return bool(self.settings.api_key)
+
+    async def transcribe(
+        self, audio: AudioData, opts: STTOptions | None = None
+    ) -> Transcription:
+        if self._client is None:
+            raise RuntimeError("Provider not started — call start() first")
+
+        opts = opts or STTOptions()
+        language_code = (opts.language or self.settings.language_code or "").strip()
+
+        data: dict[str, str] = {
+            "model_id": self.settings.model_id,
+            "diarize": "true" if self.settings.diarize else "false",
+            "tag_audio_events": "true" if self.settings.tag_audio_events else "false",
+            "timestamps_granularity": self.settings.timestamps_granularity,
+            "no_verbatim": "true" if self.settings.no_verbatim else "false",
+            "detect_speaker_roles": "true" if self.settings.detect_speaker_roles else "false",
+            "use_multi_channel": "true" if self.settings.use_multi_channel else "false",
+        }
+        if language_code:
+            data["language_code"] = language_code
+        if self.settings.num_speakers is not None:
+            data["num_speakers"] = str(self.settings.num_speakers)
+        if self.settings.diarization_threshold is not None:
+            data["diarization_threshold"] = str(self.settings.diarization_threshold)
+        if self.settings.temperature is not None:
+            data["temperature"] = str(self.settings.temperature)
+        if self.settings.entity_detection:
+            data["entity_detection"] = self.settings.entity_detection
+
+        files = {
+            "file": (
+                "audio",
+                audio.data,
+                self._mime_for_format(audio.format),
+            )
+        }
+        headers = {"xi-api-key": self.settings.api_key}
+
+        resp = await self._client.post(
+            "https://api.elevenlabs.io/v1/speech-to-text",
+            headers=headers,
+            data=data,
+            files=files,
+        )
+        if resp.status_code != 200:
+            raise RuntimeError(
+                f"ElevenLabs STT API error {resp.status_code}: {resp.text}"
+            )
+
+        payload = resp.json()
+        text = str(payload.get("text", "") or "")
+        if not text and isinstance(payload.get("transcripts"), list):
+            parts: list[str] = []
+            for item in payload.get("transcripts", []):
+                channel_text = str((item or {}).get("text", "") or "").strip()
+                if channel_text:
+                    parts.append(channel_text)
+            text = "\n".join(parts)
+        response_lang = payload.get("language_code")
+
+        words: list[Word] = []
+        source_word_lists = []
+        if isinstance(payload.get("words"), list):
+            source_word_lists.append(payload.get("words", []))
+        if isinstance(payload.get("transcripts"), list):
+            for item in payload.get("transcripts", []):
+                if isinstance((item or {}).get("words"), list):
+                    source_word_lists.append((item or {}).get("words", []))
+        for word_items in source_word_lists:
+            for item in word_items:
+                wtxt = str(item.get("text", "") or "").strip()
+                if not wtxt:
+                    continue
+                start_ms = int(float(item.get("start", 0.0) or 0.0) * 1000)
+                end_ms = int(float(item.get("end", 0.0) or 0.0) * 1000)
+                words.append(
+                    Word(
+                        text=wtxt,
+                        start_ms=start_ms,
+                        end_ms=end_ms,
+                    )
+                )
+
+        return Transcription(
+            text=text,
+            language=response_lang or (language_code or None),
+            confidence=float(payload.get("language_probability", 0.0) or 0.0)
+            if payload.get("language_probability") is not None
+            else None,
+            words=words if words else None,
+        )
+
+    async def transcribe_stream(
+        self, stream: AsyncIterator[bytes]
+    ) -> AsyncIterator[Any]:
+        """Realtime STT over ElevenLabs WebSocket.
+
+        Protocol summary (official docs):
+        - Connect: ``wss://api.elevenlabs.io/v1/speech-to-text/realtime``
+        - Send audio frames as ``input_audio_chunk`` with base64 payload
+        - Receive ``partial_transcript`` and ``committed_transcript*`` events
+        """
+        if self._client is None:
+            raise RuntimeError("Provider not started — call start() first")
+
+        import websockets
+
+        params = {
+            "model_id": self.settings.realtime_model_id,
+            "audio_format": self.settings.realtime_audio_format,
+            "commit_strategy": self.settings.realtime_commit_strategy,
+            "include_timestamps": "true" if self.settings.realtime_include_timestamps else "false",
+            "include_language_detection": "true" if self.settings.realtime_include_language_detection else "false",
+            "vad_silence_threshold_secs": str(self.settings.realtime_vad_silence_threshold_secs),
+        }
+        language_code = (self.settings.language_code or "").strip()
+        if language_code:
+            params["language_code"] = language_code
+
+        url = "wss://api.elevenlabs.io/v1/speech-to-text/realtime?" + urlencode(params)
+        headers = {"xi-api-key": self.settings.api_key}
+
+        logger.debug("{}: connecting realtime WS...", self.name)
+        _frames_sent = 0
+        _sender_done = asyncio.Event()
+        _confirmed_parts: list[str] = []
+        _last_partial: str = ""
+        results: asyncio.Queue[Transcription | None] = asyncio.Queue()
+
+        async with _ws_connect_with_headers(websockets, url, headers) as ws:
+            async def _sender() -> None:
+                nonlocal _frames_sent
+                try:
+                    async for chunk in stream:
+                        if not chunk:
+                            continue
+                        payload = {
+                            "message_type": "input_audio_chunk",
+                            "audio_base_64": base64.b64encode(chunk).decode("ascii"),
+                            "sample_rate": 16000,
+                        }
+                        # In manual mode, we explicitly commit each frame.
+                        if self.settings.realtime_commit_strategy == "manual":
+                            payload["commit"] = True
+                        await ws.send(json.dumps(payload))
+                        _frames_sent += 1
+                finally:
+                    # Trigger a final commit after stream ends to flush any buffered text.
+                    if self.settings.realtime_commit_strategy != "manual":
+                        try:
+                            await ws.send(json.dumps({
+                                "message_type": "input_audio_chunk",
+                                "audio_base_64": "",
+                                "sample_rate": 16000,
+                                "commit": True,
+                            }))
+                        except Exception:
+                            pass
+                    _sender_done.set()
+
+            async def _receiver() -> None:
+                nonlocal _last_partial
+                try:
+                    async for raw in ws:
+                        data = json.loads(raw)
+                        mtype = str(data.get("message_type", "") or "")
+                        if mtype == "session_started":
+                            continue
+
+                        if mtype.startswith("scribe") or mtype == "error":
+                            detail = data.get("message") or data.get("detail") or data
+                            raise RuntimeError(f"ElevenLabs realtime STT error: {detail}")
+
+                        if mtype == "partial_transcript":
+                            partial = str(data.get("text", "") or "").strip()
+                            if not partial:
+                                continue
+                            _last_partial = partial
+                            snapshot = self._snapshot_text(_confirmed_parts, _last_partial)
+                            if not snapshot:
+                                continue
+                            await results.put(Transcription(
+                                text=snapshot,
+                                language=data.get("language_code") or language_code or None,
+                                confidence=data.get("confidence"),
+                                is_partial=True,
+                            ))
+                            continue
+
+                        if mtype in {"committed_transcript", "committed_transcript_with_timestamps"}:
+                            committed = str(data.get("text", "") or "").strip()
+                            if committed:
+                                if not _confirmed_parts or _confirmed_parts[-1] != committed:
+                                    _confirmed_parts.append(committed)
+                            _last_partial = ""
+                            snapshot = self._snapshot_text(_confirmed_parts, _last_partial)
+                            if not snapshot:
+                                continue
+                            words = None
+                            if mtype == "committed_transcript_with_timestamps":
+                                words = self._parse_stream_words(data.get("words") or [])
+                            await results.put(Transcription(
+                                text=snapshot,
+                                language=data.get("language_code") or language_code or None,
+                                confidence=data.get("confidence"),
+                                words=words,
+                                is_partial=False,
+                            ))
+                finally:
+                    await results.put(None)
+
+            sender_task = asyncio.create_task(_sender())
+            receiver_task = asyncio.create_task(_receiver())
+
+            while True:
+                try:
+                    timeout = 1.2 if _sender_done.is_set() else None
+                    item = await asyncio.wait_for(results.get(), timeout=timeout)
+                except asyncio.TimeoutError:
+                    break
+                if item is None:
+                    break
+                yield item
+
+            if not sender_task.done():
+                sender_task.cancel()
+            if not receiver_task.done():
+                receiver_task.cancel()
+            for task in (sender_task, receiver_task):
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+
+        logger.debug("{}: realtime WS finished, sent {} frames", self.name, _frames_sent)
+
+    @staticmethod
+    def _mime_for_format(fmt: AudioFormat) -> str:
+        if fmt == AudioFormat.WAV:
+            return "audio/wav"
+        if fmt == AudioFormat.MP3:
+            return "audio/mpeg"
+        if fmt == AudioFormat.FLAC:
+            return "audio/flac"
+        if fmt == AudioFormat.OGG:
+            return "audio/ogg"
+        if fmt == AudioFormat.OPUS:
+            return "audio/opus"
+        if fmt in (AudioFormat.PCM_16K, AudioFormat.PCM_44K):
+            return "audio/wav"
+        return "application/octet-stream"
+
+    @staticmethod
+    def _snapshot_text(confirmed_parts: list[str], partial: str) -> str:
+        base = " ".join([p for p in confirmed_parts if p]).strip()
+        p = (partial or "").strip()
+        if not base:
+            return p
+        if not p:
+            return base
+        if p.startswith(base):
+            return p
+        return f"{base} {p}".strip()
+
+    @staticmethod
+    def _parse_stream_words(words_payload: list[dict]) -> list[Word] | None:
+        words: list[Word] = []
+        for item in words_payload:
+            if str(item.get("type", "word")) != "word":
+                continue
+            txt = str(item.get("text", "") or "").strip()
+            if not txt:
+                continue
+            start_ms = int(float(item.get("start", 0.0) or 0.0) * 1000)
+            end_ms = int(float(item.get("end", 0.0) or 0.0) * 1000)
+            words.append(Word(text=txt, start_ms=start_ms, end_ms=end_ms))
+        return words or None

openspeech/providers/stt/faster_whisper.py

@@ -0,0 +1,211 @@
+"""Faster-Whisper STT provider adapter (local, subprocess mode)."""
+from __future__ import annotations
+
+from openspeech.logging_config import logger
+import os
+from pathlib import Path
+import tempfile
+import time
+from collections.abc import AsyncIterator
+from dataclasses import dataclass
+from typing import Any
+
+from openspeech.core.base import STTProvider
+
+from openspeech.core.enums import Capability, ExecMode, ProviderType
+from openspeech.core.models import AudioData, STTOptions, Transcription, Word
+from openspeech.core.settings import BaseSettings
+
+@dataclass
+class FasterWhisperSTTSettings(BaseSettings):
+    model_size: str = "base"
+    device: str = "auto"
+    compute_type: str = "default"
+    download_root: str | None = None
+    beam_size: int = 5
+
+class FasterWhisperSTT(STTProvider):
+    name = "faster-whisper"
+    provider_type = ProviderType.STT
+    execution_mode = ExecMode.SUBPROCESS
+    settings_cls = FasterWhisperSTTSettings
+    capabilities = {
+        Capability.BATCH,
+        Capability.MULTILINGUAL,
+        Capability.WORD_TIMESTAMPS,
+    }
+    field_options = {"device": ["auto", "cpu", "cuda", "mps"], "compute_type": ["default", "auto", "float16", "float32", "int8", "int8_float16"], "model_size": ["tiny", "base", "small", "medium", "large-v2", "large-v3", "large-v3-turbo"]}
+
+    def __init__(self, settings: FasterWhisperSTTSettings | None = None) -> None:
+        self.settings = settings or FasterWhisperSTTSettings()
+        self._client: Any = None
+        self._model: Any = None
+        self._loaded_model_size = self.settings.model_size
+        self._loaded_device = self.settings.device
+        self._loaded_compute_type = self.settings.compute_type
+
+    async def start(self) -> None:
+        await self._ensure_model(
+            model_size=self.settings.model_size,
+            device=self.settings.device,
+            compute_type=self.settings.compute_type,
+        )
+        logger.info("{} provider started", self.name)
+
+    async def _ensure_model(
+        self,
+        *,
+        model_size: str,
+        device: str,
+        compute_type: str,
+    ) -> None:
+        try:
+            from faster_whisper import WhisperModel
+        except ImportError:
+            raise ImportError(
+                "Install faster-whisper: pip install openspeech[faster-whisper]"
+            )
+
+        normalized_model_size = self._resolve_model_dir(model_size)
+
+        if (
+            self._model is not None
+            and self._loaded_model_size == normalized_model_size
+            and self._loaded_device == device
+            and self._loaded_compute_type == compute_type
+        ):
+            return
+
+        self._model = WhisperModel(
+            normalized_model_size,
+            device=device,
+            compute_type=compute_type,
+            download_root=self.settings.download_root,
+        )
+        self._client = self._model  # for health_check
+        self._loaded_model_size = normalized_model_size
+        self._loaded_device = device
+        self._loaded_compute_type = compute_type
+
+    @staticmethod
+    def _resolve_model_dir(model_size: str) -> str:
+        raw = (model_size or "").strip()
+        p = Path(raw).expanduser()
+        if not p.is_absolute() or not p.exists() or not p.is_dir():
+            return raw
+        if (p / "model.bin").exists():
+            return str(p)
+        snapshots = p / "snapshots"
+        if snapshots.is_dir():
+            for d in sorted(snapshots.iterdir()):
+                if d.is_dir() and (d / "model.bin").exists():
+                    return str(d)
+        return str(p)
+
+    async def stop(self) -> None:
+        self._client = None
+        self._model = None
+        logger.info("{} provider stopped", self.name)
+
+    async def health_check(self) -> bool:
+        return self._client is not None
+
+    async def transcribe(
+        self, audio: AudioData, opts: STTOptions | None = None
+    ) -> Transcription:
+        if self._client is None:
+            raise RuntimeError("Provider not started — call start() first")
+        logger.info("{}: request received, audio={} bytes", self.name, len(audio.data))
+        _t0 = time.perf_counter()
+
+        opts = opts or STTOptions()
+        requested_model = (opts.model or self.settings.model_size).strip()
+        requested_device = (opts.device or self.settings.device).strip()
+        requested_compute_type = (opts.compute_type or self.settings.compute_type).strip()
+        await self._ensure_model(
+            model_size=requested_model,
+            device=requested_device,
+            compute_type=requested_compute_type,
+        )
+
+        # Ensure proper WAV headers — raw PCM from TTS providers needs wrapping
+        import io
+        import wave
+
+        is_wav = len(audio.data) > 4 and audio.data[:4] == b"RIFF"
+        with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
+            if is_wav:
+                f.write(audio.data)
+            else:
+                buf = io.BytesIO()
+                with wave.open(buf, "wb") as wf:
+                    wf.setnchannels(audio.channels)
+                    wf.setsampwidth(2)
+                    wf.setframerate(audio.sample_rate)
+                    wf.writeframes(audio.data)
+                f.write(buf.getvalue())
+            tmp_path = f.name
+
+        try:
+            beam_size = int(opts.beam_size) if opts.beam_size is not None else int(self.settings.beam_size)
+            kwargs: dict[str, Any] = {"beam_size": beam_size}
+            if opts.language:
+                kwargs["language"] = opts.language
+
+            segments, info = self._model.transcribe(tmp_path, **kwargs)
+
+            text_parts = []
+            words_list: list[Word] = []
+            max_end_s = 0.0
+            for segment in segments:
+                text_parts.append(segment.text)
+                seg_end = getattr(segment, "end", None)
+                if seg_end is not None:
+                    try:
+                        max_end_s = max(max_end_s, float(seg_end))
+                    except Exception:
+                        pass
+                if hasattr(segment, "words") and segment.words:
+                    for w in segment.words:
+                        words_list.append(
+                            Word(
+                                text=w.word,
+                                start_ms=int(w.start * 1000),
+                                end_ms=int(w.end * 1000),
+                                confidence=getattr(w, "probability", None),
+                            )
+                        )
+                        try:
+                            max_end_s = max(max_end_s, float(w.end))
+                        except Exception:
+                            pass
+
+            duration_ms: int | None = None
+            info_duration = getattr(info, "duration", None)
+            if info_duration is not None:
+                try:
+                    duration_ms = int(float(info_duration) * 1000)
+                except Exception:
+                    duration_ms = None
+            if duration_ms is None and max_end_s > 0:
+                duration_ms = int(max_end_s * 1000)
+
+            result = Transcription(
+                text=" ".join(text_parts).strip(),
+                language=info.language if hasattr(info, "language") else None,
+                confidence=getattr(info, "language_probability", None),
+                words=words_list if words_list else None,
+                duration_ms=duration_ms,
+            )
+            logger.info("{}: completed in {:.0f}ms, result={} chars", self.name, (time.perf_counter() - _t0) * 1000, len(result.text))
+            return result
+        finally:
+            os.unlink(tmp_path)
+
+    async def transcribe_stream(
+        self, stream: AsyncIterator[bytes]
+    ) -> AsyncIterator[Any]:
+        raise NotImplementedError(
+            "FasterWhisperSTT does not support streaming transcription"
+        )
+        yield  # pragma: no cover