voice-vibecoder 2.20.3__tar.gz → 2.22.0__tar.gz

{voice_vibecoder-2.20.3 → voice_vibecoder-2.22.0}/CHANGELOG.md

@@ -2,6 +2,22 @@
 
 <!-- version list -->
 
+## v2.22.0 (2026-06-30)
+
+### Features
+
+- **realtime**: Support the Azure GA realtime API (gpt-realtime / gpt-realtime-2)
+  ([`93703c3`](https://github.com/snokam/voice-vibecoder/commit/93703c322410e882a796b9415d663a3c446398c0))
+
+
+## v2.21.0 (2026-04-12)
+
+### Features
+
+- **session**: Narrate background tool events to voice user
+  ([`9c28a4f`](https://github.com/snokam/voice-vibecoder/commit/9c28a4f2f6a3f2bc6e6243f11a629ab0e369d7ee))
+
+
 ## v2.20.3 (2026-02-23)
 
 ### Bug Fixes

{voice_vibecoder-2.20.3 → voice_vibecoder-2.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: voice-vibecoder
-Version: 2.20.3
+Version: 2.22.0
 Summary: Olaf The Vibecoder — a voice-controlled coding assistant using OpenAI Realtime API + Claude Code
 Project-URL: Homepage, https://github.com/snokam/voice-vibecoder
 Project-URL: Repository, https://github.com/snokam/voice-vibecoder

{voice_vibecoder-2.20.3 → voice_vibecoder-2.22.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "voice-vibecoder"
-version = "2.20.3"
+version = "2.22.0"
 description = "Olaf The Vibecoder — a voice-controlled coding assistant using OpenAI Realtime API + Claude Code"
 readme = "README.md"
 requires-python = ">=3.10"

{voice_vibecoder-2.20.3 → voice_vibecoder-2.22.0}/src/voice_vibecoder/__init__.py

@@ -3,7 +3,7 @@
 A voice-controlled coding assistant using OpenAI Realtime API + Claude Code.
 """
 
-__version__ = "2.20.3"
+__version__ = "2.22.0"
 
 from voice_vibecoder.app_config import VoiceConfig
 from voice_vibecoder.ui.screen import VoiceCodingScreen

{voice_vibecoder-2.20.3 → voice_vibecoder-2.22.0}/src/voice_vibecoder/config.py

@@ -354,10 +354,12 @@ class RealtimeSettings:
     @property
     def ws_url(self) -> str:
         if self.provider == "azure":
+            # GA realtime API: /openai/v1/realtime?model=<deployment>, no
+            # api-version. The dated preview api-versions reject GA-only models
+            # (e.g. gpt-realtime / gpt-realtime-2 → "only available on the GA API").
             return (
-                f"wss://{self.azure.endpoint}/openai/realtime"
-                f"?api-version={AZURE_API_VERSION}"
-                f"&deployment={self.azure.deployment}"
+                f"wss://{self.azure.endpoint}/openai/v1/realtime"
+                f"?model={self.azure.deployment}"
             )
         if self.provider == "gemini":
             return f"{GEMINI_WS_URL}?key={self.active_api_key}"

{voice_vibecoder-2.20.3 → voice_vibecoder-2.22.0}/src/voice_vibecoder/session.py

@@ -13,6 +13,7 @@ via the voice provider so ChatGPT can summarize it to the user.
 
 import asyncio
 import logging
+import time
 from pathlib import Path
 from typing import Any, Callable
 
@@ -49,6 +50,70 @@ logger = logging.getLogger(__name__)
 RECONNECT_MAX_ATTEMPTS = 5
 RECONNECT_DELAYS = [2, 4, 8, 16, 30]
 
+# Debounce between spoken tool-progress narrations. Prevents the model from
+# interrupting itself mid-filler when many tool events fire in quick succession.
+TOOL_NARRATION_DEBOUNCE_SEC = 6.0
+
+
+# Markers used by code-provider runners to tag the kind of tool event:
+#   🔧  tool invocation (tool_call)
+#   ✓   tool completed successfully (tool_result)
+#   ❌  tool failed (tool_result)
+#   ↳   streaming output preview, NOT a completion — skip narration
+_TOOL_CALL_MARKER = "\U0001f527"  # 🔧
+_TOOL_OK_MARKER = "\u2713"        # ✓
+_TOOL_ERR_MARKER = "\u274c"       # ❌
+
+
+def _friendly_tool_hint(line: str) -> str | None:
+    """Turn a registry tool-event line into a short natural-language hint.
+
+    Lines look like `"tool_call:🔧 jira_search({...})"` or
+    `"tool_result:✓ jira_search: 3 issues"`. Returns None for lines that
+    aren't recognized tool starts/completions (e.g. streaming previews)
+    so the caller skips narration.
+    """
+    if not line or ":" not in line:
+        return None
+    category, rest = line.split(":", 1)
+    if category not in ("tool_call", "tool_result"):
+        return None
+    rest = rest.strip()
+    # Must be one of the known markers — otherwise it's a preview or
+    # something we don't want to narrate.
+    if category == "tool_call":
+        if not rest.startswith(_TOOL_CALL_MARKER):
+            return None
+        verb = "using"
+    else:
+        if rest.startswith(_TOOL_OK_MARKER):
+            verb = "finished"
+        elif rest.startswith(_TOOL_ERR_MARKER):
+            verb = "had trouble with"
+        else:
+            return None
+    rest = rest[1:].lstrip()
+    for sep in ("(", ":", " "):
+        if sep in rest:
+            rest = rest.split(sep, 1)[0]
+    raw = rest.strip()
+    if not raw:
+        return None
+    name = raw.split(".", 1)[-1].replace("_", " ").replace("-", " ").strip()
+    if not name:
+        return None
+    if verb == "using":
+        return f"using {name}"
+    if verb == "finished":
+        return f"finished {name}, preparing answer"
+    return f"had trouble with {name}, trying another way"
+
+
+_TOOL_NARRATION_PROMPT = (
+    "[agent background status: {hint}. Let the user know naturally in "
+    "whatever language they are speaking. Don't guess at the results.]"
+)
+
 
 class RealtimeSession:
     """Manages a single voice session with multi-instance support."""
@@ -89,6 +154,7 @@ class RealtimeSession:
         self._response_pending = False
         self._fallback_task: asyncio.Task | None = None
         self._loop: asyncio.AbstractEventLoop | None = None
+        self._last_tool_narration: float = 0.0
 
         # Create instance registry and wire callbacks
         self._registry = InstanceRegistry(user_email=user_email, user_jwt=user_jwt)
@@ -322,6 +388,38 @@ class RealtimeSession:
     def _on_registry_output(self, instance_id: str, line: str) -> None:
         if self._on_agent_output:
             self._on_agent_output(instance_id, line)
+        self._maybe_narrate_tool_event(line)
+
+    def _maybe_narrate_tool_event(self, line: str) -> None:
+        """Speak a short filler phrase when a background tool event fires.
+
+        Without this, a caller hears 30–60s of silence while the agent runs
+        MCP tools. Debounced to TOOL_NARRATION_DEBOUNCE_SEC so many rapid
+        tool events coalesce into at most one spoken hint. Called from
+        background threads, so the injection is bounced to the session's
+        asyncio loop.
+        """
+        if self.text_only or not self._running:
+            return
+        if not isinstance(line, str):
+            return
+        hint = _friendly_tool_hint(line)
+        if not hint:
+            return
+        now = time.monotonic()
+        if (now - self._last_tool_narration) < TOOL_NARRATION_DEBOUNCE_SEC:
+            return
+        loop = self._loop
+        if loop is None:
+            return
+        self._last_tool_narration = now
+        text = _TOOL_NARRATION_PROMPT.format(hint=hint)
+        try:
+            asyncio.run_coroutine_threadsafe(
+                self._provider.inject_message(text), loop,
+            )
+        except Exception as e:
+            logger.warning("Tool narration inject failed: %s", e)
 
     def _on_registry_status_change(self, instance_id: str, status: InstanceStatus) -> None:
         if self._on_instance_status:

{voice_vibecoder-2.20.3 → voice_vibecoder-2.22.0}/src/voice_vibecoder/voice_providers/__init__.py

@@ -142,4 +142,7 @@ def create_provider(settings) -> VoiceProvider:
     return OpenAIVoiceProvider(
         ws_url=settings.ws_url,
         ws_headers=settings.ws_headers,
+        # Azure realtime is the GA API (/openai/v1/realtime); it uses the nested
+        # session schema and response.output_audio.* events.
+        ga=settings.provider == "azure",
     )

{voice_vibecoder-2.20.3 → voice_vibecoder-2.22.0}/src/voice_vibecoder/voice_providers/openai.py

@@ -41,12 +41,69 @@ class OpenAIVoiceProvider:
         ws_url: str,
         ws_headers: dict[str, str],
         transcription_model: str = "gpt-4o-mini-transcribe",
+        ga: bool = False,
     ) -> None:
         self._ws_url = ws_url
         self._ws_headers = ws_headers
         self._transcription_model = transcription_model
+        # GA realtime API (Azure /openai/v1/realtime): nested `audio` session
+        # schema + response.output_audio.* events. Preview otherwise.
+        self._ga = ga
         self._ws: Any = None
 
+    def _turn_detection(self, vad: VADSettings | None) -> dict | None:
+        if vad is None:
+            return None
+        return {
+            "type": "server_vad",
+            "threshold": vad.threshold,
+            "prefix_padding_ms": vad.prefix_padding_ms,
+            "silence_duration_ms": vad.silence_duration_ms,
+            "create_response": True,
+        }
+
+    def _session_body(
+        self,
+        instructions: str,
+        voice: str,
+        tools: list[dict],
+        vad: VADSettings | None,
+    ) -> dict[str, Any]:
+        """Build the `session` object for session.update, GA or preview schema."""
+        if self._ga:
+            audio_input: dict[str, Any] = {
+                "format": {"type": "audio/pcm", "rate": 24000},
+                "turn_detection": self._turn_detection(vad),
+            }
+            if self._transcription_model:
+                audio_input["transcription"] = {"model": self._transcription_model}
+            session: dict[str, Any] = {
+                "type": "realtime",
+                "output_modalities": ["audio"],
+                "instructions": instructions,
+                "audio": {
+                    "input": audio_input,
+                    "output": {
+                        "format": {"type": "audio/pcm", "rate": 24000},
+                        "voice": voice,
+                    },
+                },
+            }
+            if tools:
+                session["tools"] = tools
+            return session
+
+        return {
+            "modalities": ["text", "audio"],
+            "instructions": instructions,
+            "voice": voice,
+            "input_audio_format": "pcm16",
+            "output_audio_format": "pcm16",
+            "input_audio_transcription": {"model": self._transcription_model},
+            "tools": tools,
+            "turn_detection": self._turn_detection(vad),
+        }
+
     async def connect(self) -> None:
         self._ws = await websockets.connect(
             self._ws_url, additional_headers=self._ws_headers
@@ -70,30 +127,10 @@ class OpenAIVoiceProvider:
         if not self._ws:
             return
 
-        session_config: dict[str, Any] = {
+        session_config = {
             "type": "session.update",
-            "session": {
-                "modalities": ["text", "audio"],
-                "instructions": instructions,
-                "voice": voice,
-                "input_audio_format": "pcm16",
-                "output_audio_format": "pcm16",
-                "input_audio_transcription": {"model": self._transcription_model},
-                "tools": tools,
-            },
+            "session": self._session_body(instructions, voice, tools, vad),
         }
-
-        if vad is None:
-            session_config["session"]["turn_detection"] = None
-        else:
-            session_config["session"]["turn_detection"] = {
-                "type": "server_vad",
-                "threshold": vad.threshold,
-                "prefix_padding_ms": vad.prefix_padding_ms,
-                "silence_duration_ms": vad.silence_duration_ms,
-                "create_response": True,
-            }
-
         await self._ws.send(json.dumps(session_config))
 
     async def send_audio(self, base64_chunk: str) -> None:
@@ -149,10 +186,10 @@ class OpenAIVoiceProvider:
     async def update_session(self, instructions: str) -> None:
         if not self._ws:
             return
-        msg = {
-            "type": "session.update",
-            "session": {"instructions": instructions},
-        }
+        session: dict[str, Any] = {"instructions": instructions}
+        if self._ga:
+            session["type"] = "realtime"
+        msg = {"type": "session.update", "session": session}
         await self._ws.send(json.dumps(msg))
 
     async def send_tool_result(self, call_id: str, output: str) -> None:
@@ -200,14 +237,12 @@ class OpenAIVoiceProvider:
             # Configure session with the selected voice
             await ws.send(json.dumps({
                 "type": "session.update",
-                "session": {
-                    "modalities": ["text", "audio"],
-                    "voice": voice,
-                    "instructions": "Always speak English. Say a short greeting in 5 words or less.",
-                    "input_audio_format": "pcm16",
-                    "output_audio_format": "pcm16",
-                    "turn_detection": None,
-                },
+                "session": self._session_body(
+                    instructions="Always speak English. Say a short greeting in 5 words or less.",
+                    voice=voice,
+                    tools=[],
+                    vad=None,
+                ),
             }))
 
             # Send a message and request a response
@@ -227,7 +262,7 @@ class OpenAIVoiceProvider:
             while asyncio.get_event_loop().time() < deadline:
                 raw = await asyncio.wait_for(ws.recv(), timeout=15)
                 msg = json.loads(raw)
-                if msg.get("type") == "response.audio.delta":
+                if msg.get("type") in ("response.output_audio.delta", "response.audio.delta"):
                     audio_chunks.append(base64.b64decode(msg["delta"]))
                 elif msg.get("type") == "response.done":
                     break
@@ -245,13 +280,15 @@ class OpenAIVoiceProvider:
     def _map_event(self, msg: dict) -> VoiceEvent | None:
         msg_type = msg.get("type", "")
 
-        if msg_type == "response.audio.delta":
+        # GA realtime renames the audio/transcript response events
+        # (response.output_audio.*); accept both so preview + GA both work.
+        if msg_type in ("response.output_audio.delta", "response.audio.delta"):
             return AudioDelta(delta=msg.get("delta", ""))
 
-        elif msg_type == "response.audio_transcript.delta":
+        elif msg_type in ("response.output_audio_transcript.delta", "response.audio_transcript.delta"):
             return TranscriptDelta(delta=msg.get("delta", ""))
 
-        elif msg_type == "response.audio_transcript.done":
+        elif msg_type in ("response.output_audio_transcript.done", "response.audio_transcript.done"):
             return TranscriptDone(transcript=msg.get("transcript", ""))
 
         elif msg_type == "conversation.item.input_audio_transcription.completed":