tau-coding-agent 0.1.0__py3-none-any.whl

tau/inference/api/audio/elevenlabs_audio.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import time
+from typing import Any
+
+import httpx
+
+from tau.inference.api.audio.base import BaseAudioAPI as BaseAPI
+from tau.inference.model.types import Model
+from tau.inference.types import (
+    AudioFormat,
+    AudioOptions,
+    AudioStopReason,
+    STTContext,
+    SynthesizedAudio,
+    TTSContext,
+    TimestampGranularity,
+    TranscribedAudio,
+    WordTimestamp,
+)
+
+_BASE_URL = "https://api.elevenlabs.io"
+
+# Maps our AudioFormat to ElevenLabs output_format query param values
+_FORMAT_MAP: dict[AudioFormat, str] = {
+    AudioFormat.MP3:  "mp3_44100_128",
+    AudioFormat.WAV:  "pcm_44100",
+    AudioFormat.PCM:  "pcm_44100",
+    AudioFormat.OPUS: "mp3_44100_128",
+    AudioFormat.AAC:  "mp3_44100_128",
+    AudioFormat.FLAC: "mp3_44100_128",
+}
+
+# ElevenLabs PCM output_format values are actually raw PCM, not WAV
+_PCM_FORMATS = {"pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"}
+
+
+class ElevenLabsAudioAPI(BaseAPI):
+    """
+    ElevenLabs audio API — proprietary REST endpoints.
+    TTS: POST /v1/text-to-speech/{voice_id} — voice_id in URL path,
+         output_format as query param, returns raw audio bytes.
+    STT: POST /v1/speech-to-text — multipart, returns JSON transcript.
+    Auth via xi-api-key header.
+    """
+
+    def __init__(self, options: AudioOptions) -> None:
+        super().__init__(options)
+
+    def _new_client(self) -> httpx.AsyncClient:
+        # Per-call client (used inside `async with`) so its connection pool is
+        # always closed — no persistent client left unclosed for the GC.
+        return httpx.AsyncClient(
+            base_url=self.options.base_url or _BASE_URL,
+            timeout=self.options.timeout.total_seconds(),
+        )
+
+    def _auth_headers(self) -> dict[str, str]:
+        headers: dict[str, str] = {}
+        if self.options.api_key:
+            headers["xi-api-key"] = self.options.api_key
+        if self.options.headers:
+            headers.update(self.options.headers)
+        return headers
+
+    async def synthesize(self, model: Model, context: TTSContext) -> SynthesizedAudio:
+        el_format = _FORMAT_MAP.get(context.response_format, "mp3_44100_128")
+        resolved_format = (
+            AudioFormat.PCM if el_format in _PCM_FORMATS else context.response_format
+        )
+
+        body: dict[str, Any] = {
+            "text": context.input,
+            "model_id": model.id,
+            "voice_settings": {"speed": context.speed},
+        }
+        if context.language:
+            body["language_code"] = context.language
+
+        payload = {"voice_id": context.voice, "output_format": el_format, "body": body}
+
+        if self.options.on_payload:
+            modified = self.options.on_payload(payload)
+            if modified is not None:
+                payload = modified
+
+        voice_id = payload.pop("voice_id")
+        output_format = payload.pop("output_format")
+        body = payload.pop("body", body)
+
+        try:
+            async with self._new_client() as http:
+                response = await http.post(
+                    f"/v1/text-to-speech/{voice_id}",
+                    params={"output_format": output_format},
+                    json=body,
+                    headers=self._auth_headers(),
+                )
+                response.raise_for_status()
+
+            if self.options.on_response:
+                self.options.on_response(response)
+
+            return SynthesizedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                audio=response.content,
+                format=resolved_format,
+                stop_reason=AudioStopReason.Stop,
+                timestamp=time.time(),
+            )
+        except Exception as exc:
+            return SynthesizedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                audio=b"",
+                format=context.response_format,
+                stop_reason=AudioStopReason.Error,
+                error=str(exc),
+                timestamp=time.time(),
+            )
+
+    async def transcribe(self, model: Model, context: STTContext) -> TranscribedAudio:
+        filename = f"audio.{context.format.value}"
+        files = {"file": (filename, context.audio, f"audio/{context.format.value}")}
+
+        data: dict[str, Any] = {"model_id": model.id}
+        if context.language:
+            data["language_code"] = context.language
+        if context.temperature != 0.0:
+            data["temperature"] = str(context.temperature)
+        if context.timestamp_granularities:
+            # ElevenLabs uses a single value: "word" or "character"
+            data["timestamps_granularity"] = "word"
+
+        payload = {**data, "_files": files}
+
+        if self.options.on_payload:
+            modified = self.options.on_payload(payload)
+            if modified is not None:
+                files = modified.pop("_files", files)
+                data = {k: v for k, v in modified.items() if not k.startswith("_")}
+
+        try:
+            async with self._new_client() as http:
+                response = await http.post(
+                    "/v1/speech-to-text",
+                    data=data,
+                    files=files,
+                    headers=self._auth_headers(),
+                )
+                response.raise_for_status()
+                result = response.json()
+
+            if self.options.on_response:
+                self.options.on_response(result)
+
+            words: list[WordTimestamp] = [
+                WordTimestamp(word=w["text"], start=w["start"], end=w["end"])
+                for w in result.get("words", [])
+                if w.get("type") == "word"
+            ]
+
+            return TranscribedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                text=result.get("text", ""),
+                language=result.get("language_code"),
+                duration=result.get("audio_duration_secs"),
+                words=words,
+                stop_reason=AudioStopReason.Stop,
+                usage={"language_probability": result.get("language_probability")},
+                timestamp=time.time(),
+            )
+        except Exception as exc:
+            return TranscribedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                text="",
+                stop_reason=AudioStopReason.Error,
+                error=str(exc),
+                timestamp=time.time(),
+            )

tau/inference/api/audio/gemini_audio.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import time
+
+from google import genai
+from google.genai import types as genai_types
+
+from tau.inference.api.audio.base import BaseAudioAPI as BaseAPI
+from tau.inference.model.types import Model
+from tau.inference.types import (
+    AudioFormat,
+    AudioOptions,
+    AudioStopReason,
+    STTContext,
+    SynthesizedAudio,
+    TTSContext,
+    TranscribedAudio,
+)
+
+
+class GeminiAudioAPI(BaseAPI):
+    """
+    Gemini TTS via generate_content with response_modalities=["AUDIO"].
+    Output is always raw PCM (s16le, 24 kHz, mono) regardless of the
+    requested format — Gemini does not support format selection.
+    STT is not available through the Gemini generative API.
+    """
+
+    def __init__(self, options: AudioOptions) -> None:
+        super().__init__(options)
+        self._client: genai.Client | None = None
+        if options.api_key:
+            self._client = genai.Client(api_key=options.api_key)
+
+    def _get_client(self) -> genai.Client:
+        if self._client is None or self.options.api_key != getattr(self._client._api_client, 'api_key', None):
+            self._client = genai.Client(api_key=self.options.api_key)
+        return self._client
+
+    async def synthesize(self, model: Model, context: TTSContext) -> SynthesizedAudio:
+        """Synthesize audio from text using Gemini API."""
+        client = self._get_client()
+
+        config = genai_types.GenerateContentConfig(
+            response_modalities=["AUDIO"],
+            speech_config=genai_types.SpeechConfig(
+                voice_config=genai_types.VoiceConfig(
+                    prebuilt_voice_config=genai_types.PrebuiltVoiceConfig(
+                        voice_name=context.voice,
+                    )
+                )
+            ),
+        )
+
+        payload = {"model": model.id, "contents": context.input, "config": config}
+
+        if self.options.on_payload:
+            modified = self.options.on_payload(payload)
+            if modified is not None:
+                payload = modified
+
+        try:
+            response = await client.aio.models.generate_content(**payload)
+
+            if self.options.on_response:
+                self.options.on_response(response)
+
+            part = response.candidates[0].content.parts[0]
+            audio_bytes: bytes = part.inline_data.data
+
+            return SynthesizedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                audio=audio_bytes,
+                format=AudioFormat.PCM,
+                stop_reason=AudioStopReason.Stop,
+                timestamp=time.time(),
+            )
+        except Exception as exc:
+            return SynthesizedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                audio=b"",
+                format=AudioFormat.PCM,
+                stop_reason=AudioStopReason.Error,
+                error=str(exc),
+                timestamp=time.time(),
+            )
+
+    async def transcribe(self, model: Model, context: STTContext) -> TranscribedAudio:
+        """Transcribe audio to text (not supported by Gemini API)."""
+        raise NotImplementedError(
+            "Gemini generative API does not support speech-to-text transcription. "
+            "Use a Whisper-based model (openai or groq provider) for STT."
+        )

tau/inference/api/audio/openai_audio.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import time
+from pathlib import Path
+from typing import Any
+
+from openai import AsyncOpenAI
+
+from tau.inference.api.audio.base import BaseAudioAPI as BaseAPI
+from tau.inference.model.types import Model
+from tau.inference.types import (
+    AudioFormat,
+    AudioOptions,
+    AudioStopReason,
+    STTContext,
+    SegmentTimestamp,
+    SynthesizedAudio,
+    TTSContext,
+    TranscribedAudio,
+    WordTimestamp,
+)
+
+_STOP_REASON: dict[str, AudioStopReason] = {
+    "stop": AudioStopReason.Stop,
+    "error": AudioStopReason.Error,
+}
+
+
+class OpenAIAudioAPI(BaseAPI):
+    """
+    OpenAI-compatible audio API — works with OpenAI, Groq, and any provider
+    that implements the /v1/audio/speech and /v1/audio/transcriptions endpoints.
+    Point at a different provider by setting options.base_url and options.api_key.
+    """
+
+    def __init__(self, options: AudioOptions) -> None:
+        super().__init__(options)
+        self._client = AsyncOpenAI(
+            api_key=options.api_key or "placeholder",
+            base_url=options.base_url,
+            default_headers=options.headers,
+            max_retries=options.max_retries,
+            timeout=options.timeout.total_seconds(),
+        )
+
+    async def synthesize(self, model: Model, context: TTSContext) -> SynthesizedAudio:
+        """Synthesize audio from text using OpenAI API."""
+        if self.options.api_key:
+            self._client.api_key = self.options.api_key
+
+        params: dict[str, Any] = {
+            "model": model.id,
+            "input": context.input,
+            "voice": context.voice,
+            "response_format": context.response_format.value,
+            "speed": context.speed,
+        }
+        if context.instructions:
+            params["instructions"] = context.instructions
+
+        if self.options.on_payload:
+            modified = self.options.on_payload(params)
+            if modified is not None:
+                params = modified
+
+        try:
+            response = await self._client.audio.speech.create(**params)
+            audio_bytes = response.content
+
+            if self.options.on_response:
+                self.options.on_response(response)
+
+            return SynthesizedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                audio=audio_bytes,
+                format=context.response_format,
+                stop_reason=AudioStopReason.Stop,
+                timestamp=time.time(),
+            )
+        except Exception as exc:
+            return SynthesizedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                audio=b"",
+                format=context.response_format,
+                stop_reason=AudioStopReason.Error,
+                error=str(exc),
+                timestamp=time.time(),
+            )
+
+    async def transcribe(self, model: Model, context: STTContext) -> TranscribedAudio:
+        """Transcribe audio to text using OpenAI API."""
+        if self.options.api_key:
+            self._client.api_key = self.options.api_key
+
+        # OpenAI SDK expects a file-like object with a name; wrap raw bytes
+        filename = f"audio.{context.format.value}"
+        audio_file = (filename, context.audio, f"audio/{context.format.value}")
+
+        params: dict[str, Any] = {
+            "model": model.id,
+            "file": audio_file,
+            "response_format": "verbose_json",
+        }
+        if context.language:
+            params["language"] = context.language
+        if context.temperature != 0.0:
+            params["temperature"] = context.temperature
+        if context.timestamp_granularities:
+            params["timestamp_granularities"] = [g.value for g in context.timestamp_granularities]
+        if context.prompt:
+            params["prompt"] = context.prompt
+
+        if self.options.on_payload:
+            modified = self.options.on_payload(params)
+            if modified is not None:
+                params = modified
+
+        try:
+            response = await self._client.audio.transcriptions.create(**params)
+
+            if self.options.on_response:
+                self.options.on_response(response)
+
+            words: list[WordTimestamp] = []
+            segments: list[SegmentTimestamp] = []
+
+            if hasattr(response, "words") and response.words:
+                words = [
+                    WordTimestamp(word=w.word, start=w.start, end=w.end)
+                    for w in response.words
+                ]
+            if hasattr(response, "segments") and response.segments:
+                segments = [
+                    SegmentTimestamp(id=s.id, text=s.text, start=s.start, end=s.end)
+                    for s in response.segments
+                ]
+
+            return TranscribedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                text=response.text,
+                language=getattr(response, "language", None),
+                duration=getattr(response, "duration", None),
+                words=words,
+                segments=segments,
+                stop_reason=AudioStopReason.Stop,
+                timestamp=time.time(),
+            )
+        except Exception as exc:
+            return TranscribedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                text="",
+                stop_reason=AudioStopReason.Error,
+                error=str(exc),
+                timestamp=time.time(),
+            )

tau/inference/api/audio/registry.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+from tau.inference.api.registry import BaseAPIRegistry
+from tau.inference.api.audio.base import BaseAudioAPI
+
+
+class AudioAPIRegistry(BaseAPIRegistry[BaseAudioAPI]):
+    """Registry for audio API implementations."""
+
+    @classmethod
+    def from_builtins(cls) -> AudioAPIRegistry:
+        from tau.inference.api.audio.builtins import AUDIO_APIS
+        instance = cls()
+        for name, api in AUDIO_APIS:
+            instance.register(name, api)
+        return instance

tau/inference/api/audio/sarvam_audio.py

@@ -0,0 +1,163 @@
+from __future__ import annotations
+
+import base64
+import time
+from typing import Any
+
+import httpx
+
+from tau.inference.api.audio.base import BaseAudioAPI as BaseAPI
+from tau.inference.model.types import Model
+from tau.inference.types import (
+    AudioFormat,
+    AudioOptions,
+    AudioStopReason,
+    STTContext,
+    SegmentTimestamp,
+    SynthesizedAudio,
+    TTSContext,
+    TranscribedAudio,
+    WordTimestamp,
+)
+
+_BASE_URL = "https://api.sarvam.ai"
+_TTS_ENDPOINT = "/text-to-speech"
+_STT_ENDPOINT = "/speech-to-text"
+
+# Sarvam returns WAV by default inside the base64 payload
+_OUTPUT_FORMAT = AudioFormat.WAV
+
+
+class SarvamAudioAPI(BaseAPI):
+    """
+    Sarvam AI audio API — proprietary REST endpoints for Indian language TTS and STT.
+    Auth via custom header: api-subscription-key.
+    TTS response: JSON with base64-encoded audio in audios[].
+    STT response: JSON with transcript, language_code, language_probability.
+    """
+
+    def __init__(self, options: AudioOptions) -> None:
+        super().__init__(options)
+
+    def _new_client(self) -> httpx.AsyncClient:
+        # Per-call client (used inside `async with`) so its connection pool is
+        # always closed — no persistent client left unclosed for the GC.
+        return httpx.AsyncClient(
+            base_url=self.options.base_url or _BASE_URL,
+            timeout=self.options.timeout.total_seconds(),
+        )
+
+    def _auth_headers(self) -> dict[str, str]:
+        headers: dict[str, str] = {}
+        if self.options.api_key:
+            headers["api-subscription-key"] = self.options.api_key
+        if self.options.headers:
+            headers.update(self.options.headers)
+        return headers
+
+    async def synthesize(self, model: Model, context: TTSContext) -> SynthesizedAudio:
+        payload: dict[str, Any] = {
+            "model": model.id,
+            "text": context.input,
+            "speaker": context.voice,
+            "pace": context.speed,
+        }
+        if context.language:
+            payload["target_language_code"] = context.language
+
+        if self.options.on_payload:
+            modified = self.options.on_payload(payload)
+            if modified is not None:
+                payload = modified
+
+        try:
+            async with self._new_client() as http:
+                response = await http.post(
+                    _TTS_ENDPOINT,
+                    json=payload,
+                    headers=self._auth_headers(),
+                )
+                response.raise_for_status()
+                data = response.json()
+
+            if self.options.on_response:
+                self.options.on_response(data)
+
+            audio_bytes = base64.b64decode(data["audios"][0])
+
+            return SynthesizedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                audio=audio_bytes,
+                format=_OUTPUT_FORMAT,
+                stop_reason=AudioStopReason.Stop,
+                timestamp=time.time(),
+            )
+        except Exception as exc:
+            return SynthesizedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                audio=b"",
+                format=_OUTPUT_FORMAT,
+                stop_reason=AudioStopReason.Error,
+                error=str(exc),
+                timestamp=time.time(),
+            )
+
+    async def transcribe(self, model: Model, context: STTContext) -> TranscribedAudio:
+        filename = f"audio.{context.format.value}"
+        files = {"file": (filename, context.audio, f"audio/{context.format.value}")}
+        data: dict[str, Any] = {"model": model.id}
+
+        if context.language:
+            data["language_code"] = context.language
+        if context.prompt:
+            # prompt field carries Sarvam's mode: transcribe/translate/verbatim/translit/codemix
+            data["mode"] = context.prompt
+
+        if self.options.on_payload:
+            modified = self.options.on_payload({**data, "_files": files})
+            if modified is not None:
+                files = modified.pop("_files", files)
+                data = modified
+
+        try:
+            async with self._new_client() as http:
+                response = await http.post(
+                    _STT_ENDPOINT,
+                    data=data,
+                    files=files,
+                    headers=self._auth_headers(),
+                )
+                response.raise_for_status()
+                result = response.json()
+
+            if self.options.on_response:
+                self.options.on_response(result)
+
+            words: list[WordTimestamp] = []
+            if ts := result.get("timestamps"):
+                words = [
+                    WordTimestamp(word=w["word"], start=w["start"], end=w["end"])
+                    for w in ts.get("words", [])
+                ]
+
+            return TranscribedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                text=result.get("transcript", ""),
+                language=result.get("language_code"),
+                words=words,
+                stop_reason=AudioStopReason.Stop,
+                usage={"language_probability": result.get("language_probability")},
+                timestamp=time.time(),
+            )
+        except Exception as exc:
+            return TranscribedAudio(
+                model_id=model.id,
+                provider=model.provider,
+                text="",
+                stop_reason=AudioStopReason.Error,
+                error=str(exc),
+                timestamp=time.time(),
+            )