codex-autorunner 0.1.0__py3-none-any.whl

codex_autorunner/voice/providers/openai_whisper.py

@@ -0,0 +1,345 @@
+from __future__ import annotations
+
+import dataclasses
+import json
+import logging
+import os
+import time
+from io import BytesIO
+from typing import Any, Callable, Dict, Iterable, Mapping, Optional, cast
+
+import httpx
+
+from ..provider import (
+    AudioChunk,
+    SpeechProvider,
+    SpeechSessionMetadata,
+    TranscriptionEvent,
+    TranscriptionStream,
+)
+
+RequestFn = Callable[[bytes, Mapping[str, Any]], Dict[str, Any]]
+
+_EXT_TO_CONTENT_TYPE: dict[str, str] = {
+    # Keep these aligned with OpenAI's documented accepted formats for /audio/transcriptions.
+    "webm": "audio/webm",
+    "ogg": "audio/ogg",
+    "wav": "audio/wav",
+    "mp3": "audio/mpeg",
+    "mpeg": "audio/mpeg",
+    "mpga": "audio/mpeg",
+    "m4a": "audio/mp4",
+    "mp4": "audio/mp4",
+}
+
+
+def _normalize_content_type(raw: Optional[str]) -> Optional[str]:
+    """
+    Normalize potentially noisy MIME types.
+
+    - Browsers may include codec parameters (e.g. "audio/webm;codecs=opus")
+    - Python's mimetypes may emit unusual values (e.g. "audio/mp4a-latm" for .m4a)
+    """
+
+    if not raw:
+        return None
+    base = raw.split(";", 1)[0].strip().lower()
+    if not base:
+        return None
+
+    # Map common-but-unhelpful values to canonical ones OpenAI reliably accepts.
+    if base == "video/webm":
+        return "audio/webm"
+    if base in ("audio/mp4a-latm", "audio/x-m4a"):
+        return "audio/mp4"
+    if base == "audio/x-wav":
+        return "audio/wav"
+    if base == "video/mp4":
+        return "audio/mp4"
+
+    return base
+
+
+def _content_type_from_filename(filename: str) -> str:
+    lower = (filename or "").lower()
+    if "." in lower:
+        ext = lower.rsplit(".", 1)[-1]
+        if ext in _EXT_TO_CONTENT_TYPE:
+            return _EXT_TO_CONTENT_TYPE[ext]
+    return "application/octet-stream"
+
+
+def _pick_upload_content_type(filename: str, provided: Optional[str]) -> str:
+    normalized = _normalize_content_type(provided)
+    return normalized or _content_type_from_filename(filename)
+
+
+def _extract_http_error_detail(
+    exc: Exception,
+) -> tuple[Optional[int], Optional[str]]:
+    if not isinstance(exc, httpx.HTTPStatusError) or exc.response is None:
+        return None, None
+
+    status_code = exc.response.status_code
+    detail: Optional[str] = None
+    try:
+        payload = exc.response.json()
+        # OpenAI typically returns {"error": {"message": "...", "type": "...", ...}}
+        if isinstance(payload, dict):
+            err = payload.get("error")
+            if isinstance(err, dict) and err.get("message"):
+                detail = str(err["message"])
+            else:
+                detail = json.dumps(payload, ensure_ascii=False)
+        else:
+            detail = json.dumps(payload, ensure_ascii=False)
+    except Exception:
+        try:
+            detail = exc.response.text
+        except Exception:
+            detail = None
+
+    if detail is not None:
+        detail = detail.strip()
+        if len(detail) > 600:
+            detail = f"{detail[:600]}…"
+    return status_code, detail
+
+
+@dataclasses.dataclass
+class OpenAIWhisperSettings:
+    api_key_env: str = "OPENAI_API_KEY"
+    model: str = "whisper-1"
+    base_url: Optional[str] = None
+    temperature: float = 0.0
+    language: Optional[str] = None
+    redact_request: bool = True
+    timeout_s: float = 60.0
+
+    @classmethod
+    def from_mapping(cls, raw: Mapping[str, Any]) -> "OpenAIWhisperSettings":
+        return cls(
+            api_key_env=str(raw.get("api_key_env", "OPENAI_API_KEY")),
+            model=str(raw.get("model", "whisper-1")),
+            base_url=raw.get("base_url"),
+            temperature=float(raw.get("temperature", 0.0)),
+            language=raw.get("language"),
+            redact_request=bool(raw.get("redact_request", True)),
+            timeout_s=float(raw.get("timeout_s", 60.0)),
+        )
+
+
+class OpenAIWhisperProvider(SpeechProvider):
+    """
+    Whisper transcription provider behind the SpeechProvider abstraction.
+
+    This keeps raw audio in-memory only and redacts request metadata by default.
+    """
+
+    name = "openai_whisper"
+    supports_streaming = (
+        False  # OpenAI Whisper is request/response; we buffer chunks locally.
+    )
+
+    def __init__(
+        self,
+        settings: OpenAIWhisperSettings,
+        env: Optional[Mapping[str, str]] = None,
+        warn_on_remote_api: bool = True,
+        logger: Optional[logging.Logger] = None,
+        request_fn: Optional[RequestFn] = None,
+    ) -> None:
+        self._settings = settings
+        self._env = env or os.environ
+        self._warn_on_remote_api = warn_on_remote_api
+        self._logger = logger or logging.getLogger(__name__)
+        self._request_fn: RequestFn = request_fn or self._default_request
+
+    def start_stream(self, session: SpeechSessionMetadata) -> TranscriptionStream:
+        api_key = self._env.get(self._settings.api_key_env)
+        if api_key:
+            # Defensive normalization: .env / launchd / shells sometimes introduce
+            # trailing newlines or quoting that can yield 401s.
+            api_key = api_key.strip().strip('"').strip("'").strip("`").strip()
+        if not api_key:
+            raise ValueError(
+                f"OpenAI Whisper provider requires API key env '{self._settings.api_key_env}' to be set"
+            )
+        return _OpenAIWhisperStream(
+            api_key=api_key,
+            settings=self._settings,
+            session=session,
+            warn_on_remote_api=self._warn_on_remote_api,
+            logger=self._logger,
+            request_fn=self._request_fn,
+        )
+
+    def _default_request(
+        self, audio_bytes: bytes, payload: Mapping[str, Any]
+    ) -> Dict[str, Any]:
+        headers = {"Authorization": f"Bearer {payload['api_key']}"}
+        url = f"{payload['base_url'].rstrip('/')}/v1/audio/transcriptions"
+        data: Dict[str, Any] = {
+            "model": payload["model"],
+            "temperature": payload["temperature"],
+        }
+        if payload.get("language"):
+            data["language"] = payload["language"]
+
+        filename = payload.get("filename", "audio.webm")
+        content_type = _pick_upload_content_type(filename, payload.get("content_type"))
+        files = {
+            "file": (
+                filename,
+                BytesIO(audio_bytes),
+                content_type,
+            )
+        }
+
+        timeout_s = float(payload.get("timeout_s", 60.0))
+        response = httpx.post(
+            url, headers=headers, data=data, files=files, timeout=timeout_s
+        )
+        response.raise_for_status()
+        return cast(Dict[str, Any], response.json())
+
+
+class _OpenAIWhisperStream(TranscriptionStream):
+    def __init__(
+        self,
+        api_key: str,
+        settings: OpenAIWhisperSettings,
+        session: SpeechSessionMetadata,
+        warn_on_remote_api: bool,
+        logger: logging.Logger,
+        request_fn: RequestFn,
+    ) -> None:
+        self._api_key = api_key
+        self._settings = settings
+        self._session = session
+        self._warn_on_remote_api = warn_on_remote_api
+        self._logger = logger
+        self._request_fn = request_fn
+        self._started_at = time.monotonic()
+        self._chunks: list[bytes] = []
+        self._aborted = False
+
+    def send_chunk(self, chunk: AudioChunk) -> Iterable[TranscriptionEvent]:
+        # Only retain raw bytes in-memory until the final request to avoid persistence.
+        if self._aborted:
+            return []
+        self._chunks.append(chunk.data)
+        return []
+
+    def flush_final(self) -> Iterable[TranscriptionEvent]:
+        if self._aborted:
+            return []
+        if not self._chunks:
+            return []
+
+        audio_bytes = b"".join(self._chunks)
+        if self._warn_on_remote_api:
+            self._logger.warning(
+                "Sending audio to OpenAI Whisper (%s); audio bytes are not logged or persisted.",
+                self._settings.model,
+            )
+
+        payload = self._build_payload()
+        status_code: Optional[int] = None
+        error_detail: Optional[str] = None
+        try:
+            started = time.monotonic()
+            result = self._request_fn(audio_bytes, payload)
+            latency_ms = int((time.monotonic() - started) * 1000)
+            text = (result or {}).get("text", "") if isinstance(result, Mapping) else ""
+            return [TranscriptionEvent(text=text, is_final=True, latency_ms=latency_ms)]
+        except Exception as exc:
+            status_code, error_detail = _extract_http_error_detail(exc)
+            if status_code is None and isinstance(exc, httpx.HTTPStatusError):
+                status_code = (
+                    exc.response.status_code if exc.response is not None else None
+                )
+
+            if error_detail:
+                self._logger.error(
+                    "OpenAI Whisper transcription failed (HTTP %s): %s",
+                    status_code if status_code is not None else "n/a",
+                    error_detail,
+                    exc_info=False,
+                )
+            else:
+                self._logger.error(
+                    "OpenAI Whisper transcription failed: %s", exc, exc_info=False
+                )
+            # Avoid retry loops for credential errors; surface explicit reasons.
+            if status_code == 401:
+                return [
+                    TranscriptionEvent(text="", is_final=True, error="unauthorized")
+                ]
+            if status_code == 403:
+                return [TranscriptionEvent(text="", is_final=True, error="forbidden")]
+            if status_code == 400:
+                # Usually indicates invalid/unsupported audio format or malformed params.
+                return [
+                    TranscriptionEvent(text="", is_final=True, error="invalid_audio")
+                ]
+            if status_code == 413:
+                return [
+                    TranscriptionEvent(text="", is_final=True, error="audio_too_large")
+                ]
+            if status_code == 429:
+                return [
+                    TranscriptionEvent(text="", is_final=True, error="rate_limited")
+                ]
+            return [TranscriptionEvent(text="", is_final=True, error="provider_error")]
+        finally:
+            # Release buffered bytes to avoid accidental reuse.
+            self._chunks = []
+
+    def abort(self, reason: Optional[str] = None) -> None:
+        self._aborted = True
+        self._chunks = []
+        if reason:
+            self._logger.info("OpenAI Whisper stream aborted: %s", reason)
+
+    def _build_payload(self) -> Dict[str, Any]:
+        base_url = self._settings.base_url or "https://api.openai.com"
+        payload = {
+            "api_key": self._api_key,
+            "base_url": base_url,
+            "model": self._settings.model,
+            "temperature": self._settings.temperature,
+            "language": self._settings.language or self._session.language,
+            "timeout_s": self._settings.timeout_s,
+        }
+        if self._session.filename:
+            payload["filename"] = self._session.filename
+        if self._session.content_type:
+            payload["content_type"] = self._session.content_type
+
+        if not self._settings.redact_request:
+            payload.update(
+                {
+                    "client": self._session.client,
+                    "session_id": self._session.session_id,
+                }
+            )
+        return payload
+
+
+def build_speech_provider(
+    config: Mapping[str, Any],
+    warn_on_remote_api: bool = True,
+    env: Optional[Mapping[str, str]] = None,
+    logger: Optional[logging.Logger] = None,
+) -> OpenAIWhisperProvider:
+    """
+    Factory used by voice resolver to construct the Whisper provider from config mappings.
+    """
+    settings = OpenAIWhisperSettings.from_mapping(config)
+    return OpenAIWhisperProvider(
+        settings=settings,
+        env=env,
+        warn_on_remote_api=warn_on_remote_api,
+        logger=logger,
+    )

codex_autorunner/voice/resolver.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import logging
+import os
+from typing import Mapping, Optional
+
+from .config import VoiceConfig
+from .provider import SpeechProvider
+from .providers import OpenAIWhisperProvider, build_speech_provider
+
+
+def resolve_speech_provider(
+    voice_config: VoiceConfig,
+    logger: Optional[logging.Logger] = None,
+    env: Optional[Mapping[str, str]] = None,
+) -> SpeechProvider:
+    """
+    Resolve the configured speech provider. Raises when disabled or unknown.
+    """
+    if not voice_config.enabled:
+        raise ValueError("Voice features are disabled in config")
+
+    provider_name = voice_config.provider
+    provider_configs = voice_config.providers or {}
+    if not provider_name:
+        raise ValueError("No voice provider configured")
+
+    if provider_name == OpenAIWhisperProvider.name:
+        return build_speech_provider(
+            provider_configs.get(provider_name, {}),
+            warn_on_remote_api=voice_config.warn_on_remote_api,
+            env=env or os.environ,
+            logger=logger,
+        )
+
+    raise ValueError(f"Unsupported voice provider '{provider_name}'")

codex_autorunner/voice/service.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import logging
+import os
+import uuid
+from typing import Callable, Optional
+
+from .capture import CaptureCallbacks, CaptureState, PushToTalkCapture
+from .config import VoiceConfig
+from .provider import SpeechSessionMetadata
+from .resolver import resolve_speech_provider
+
+
+class VoiceServiceError(Exception):
+    """Raised when voice transcription fails at the service boundary."""
+
+    def __init__(self, reason: str, detail: str):
+        super().__init__(detail)
+        self.reason = reason
+        self.detail = detail
+
+
+class VoiceService:
+    """
+    Thin wrapper that wires the shared PushToTalkCapture into HTTP handlers.
+    This keeps raw audio in-memory only and centralizes provider wiring/error mapping.
+    """
+
+    def __init__(
+        self,
+        config: VoiceConfig,
+        logger: Optional[logging.Logger] = None,
+        provider_resolver: Callable[[VoiceConfig], object] = resolve_speech_provider,
+        provider: Optional[object] = None,
+        env: Optional[dict] = None,
+    ) -> None:
+        self.config = config
+        self._logger = logger or logging.getLogger(__name__)
+        self._provider_resolver = provider_resolver
+        self._provider = provider
+        self._env = env if env is not None else os.environ
+
+    def config_payload(self) -> dict:
+        """Expose safe config fields to the UI."""
+        # Check if API key is configured for status display
+        provider_cfg = self.config.providers.get(
+            self.config.provider or "openai_whisper", {}
+        )
+        api_key_env = provider_cfg.get("api_key_env", "OPENAI_API_KEY")
+        has_api_key = bool(self._env.get(api_key_env))
+
+        return {
+            "enabled": self.config.enabled,
+            "provider": self.config.provider,
+            "latency_mode": self.config.latency_mode,
+            "chunk_ms": self.config.chunk_ms,
+            "sample_rate": self.config.sample_rate,
+            "warn_on_remote_api": self.config.warn_on_remote_api,
+            "has_api_key": has_api_key,
+            "api_key_env": api_key_env,
+            "push_to_talk": {
+                "max_ms": self.config.push_to_talk.max_ms,
+                "silence_auto_stop_ms": self.config.push_to_talk.silence_auto_stop_ms,
+                "min_hold_ms": self.config.push_to_talk.min_hold_ms,
+            },
+        }
+
+    def transcribe(
+        self,
+        audio_bytes: bytes,
+        *,
+        client: str = "web",
+        user_agent: Optional[str] = None,
+        language: Optional[str] = None,
+        filename: Optional[str] = None,
+        content_type: Optional[str] = None,
+    ) -> dict:
+        if not self.config.enabled:
+            raise VoiceServiceError("disabled", "Voice is disabled")
+        if not audio_bytes:
+            raise VoiceServiceError("empty_audio", "No audio received")
+
+        provider = self._resolve_provider()
+        buffer = _TranscriptionBuffer()
+        capture = PushToTalkCapture(
+            provider=provider,
+            config=self.config,
+            callbacks=buffer.callbacks,
+            permission_requester=lambda: True,
+            client=client,
+            logger=self._logger,
+            session_builder=lambda: self._build_session_metadata(
+                provider_name=provider.name,
+                language=language,
+                client=client,
+                user_agent=user_agent,
+                filename=filename,
+                content_type=content_type,
+            ),
+        )
+
+        capture.begin_capture()
+        if capture.state == CaptureState.ERROR:
+            reason = buffer.error_reason or "capture_failed"
+            raise VoiceServiceError(reason, reason.replace("_", " "))
+
+        try:
+            capture.handle_chunk(audio_bytes)
+            capture.end_capture("client_stop")
+        except Exception as exc:
+            raise VoiceServiceError("provider_error", str(exc)) from exc
+
+        if buffer.error_reason:
+            if buffer.error_reason in ("unauthorized", "forbidden"):
+                provider_cfg = self.config.providers.get(
+                    self.config.provider or "openai_whisper", {}
+                )
+                api_key_env = provider_cfg.get("api_key_env", "OPENAI_API_KEY")
+                raise VoiceServiceError(
+                    buffer.error_reason,
+                    f"OpenAI API key rejected ({buffer.error_reason}); check {api_key_env}",
+                )
+            if buffer.error_reason == "invalid_audio":
+                meta = ""
+                if filename or content_type:
+                    meta = f" (file={filename or 'audio'}, type={content_type or 'unknown'})"
+                raise VoiceServiceError(
+                    "invalid_audio",
+                    "OpenAI rejected the audio upload (bad request). "
+                    f"Try re-recording or switching formats/browsers{meta}.",
+                )
+            if buffer.error_reason == "audio_too_large":
+                raise VoiceServiceError(
+                    "audio_too_large",
+                    "Audio upload too large; record a shorter clip and try again.",
+                )
+            if buffer.error_reason == "rate_limited":
+                raise VoiceServiceError(
+                    "rate_limited",
+                    "OpenAI rate limited the request; wait a moment and try again.",
+                )
+            raise VoiceServiceError(
+                buffer.error_reason, buffer.error_reason.replace("_", " ")
+            )
+
+        transcript = buffer.final_text or buffer.partial_text or ""
+        return {
+            "text": transcript,
+            "warnings": buffer.warnings,
+        }
+
+    def _resolve_provider(self):
+        if self._provider is None:
+            try:
+                self._provider = self._provider_resolver(
+                    self.config, logger=self._logger
+                )
+            except TypeError:
+                self._provider = self._provider_resolver(self.config)
+        return self._provider
+
+    def _build_session_metadata(
+        self,
+        *,
+        provider_name: str,
+        language: Optional[str],
+        client: Optional[str],
+        user_agent: Optional[str],
+        filename: Optional[str] = None,
+        content_type: Optional[str] = None,
+    ) -> SpeechSessionMetadata:
+        return SpeechSessionMetadata(
+            session_id=str(uuid.uuid4()),
+            provider=provider_name,
+            latency_mode=self.config.latency_mode,
+            language=language,
+            client=client,
+            user_agent=user_agent,
+            filename=filename,
+            content_type=content_type,
+        )
+
+
+class _TranscriptionBuffer:
+    def __init__(self) -> None:
+        self.partial_text = ""
+        self.final_text = ""
+        self.warnings: list[str] = []
+        self.error_reason: Optional[str] = None
+        self.callbacks = CaptureCallbacks(
+            on_partial=self._on_partial,
+            on_final=self._on_final,
+            on_warning=self._on_warning,
+            on_error=self._on_error,
+        )
+
+    def _on_partial(self, text: str) -> None:
+        if text:
+            self.partial_text = text
+
+    def _on_final(self, text: str) -> None:
+        if text:
+            self.final_text = text
+
+    def _on_warning(self, message: str) -> None:
+        if message:
+            self.warnings.append(message)
+
+    def _on_error(self, reason: str) -> None:
+        self.error_reason = reason

codex_autorunner/web/__init__.py

@@ -0,0 +1 @@
+"""Web server components."""