voice-vibecoder 2.23.0__tar.gz → 2.24.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/CHANGELOG.md +33 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/PKG-INFO +2 -1
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/app.py +1 -1
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/pyproject.toml +2 -1
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/__init__.py +1 -1
- voice_vibecoder-2.24.1/src/voice_vibecoder/audio.py +232 -0
- voice_vibecoder-2.24.1/src/voice_vibecoder/audio_macos.py +93 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/config.py +8 -4
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/session.py +57 -3
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/tools/file_tree.py +5 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/callbacks.py +13 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/screen.py +69 -27
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/setup.py +3 -2
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/wizard.py +7 -2
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/voice_providers/__init__.py +3 -1
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/worktrees.py +8 -1
- voice_vibecoder-2.24.1/tests/test_audio_gating.py +134 -0
- voice_vibecoder-2.24.1/tests/test_file_tree.py +25 -0
- voice_vibecoder-2.24.1/tests/test_session_interrupts.py +160 -0
- voice_vibecoder-2.23.0/src/voice_vibecoder/audio.py +0 -127
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/.github/workflows/ci.yml +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/.github/workflows/release.yml +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/.gitignore +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/LICENSE +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/README.md +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/index.html +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/app_config.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/cli_serve.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/code_providers/__init__.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/code_providers/claude.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/code_providers/cursor.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/code_providers/openclaw.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/code_providers/registry.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/instances.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/server.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/tools/__init__.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/tools/agent_task.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/tools/definitions.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/tools/dispatch.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/tools/handlers.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/__init__.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/diff_view.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/help_modal.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/panels.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/state.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/styles.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/ui/update_modal.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/voice_providers/gemini.py +0 -0
- {voice_vibecoder-2.23.0 → voice_vibecoder-2.24.1}/src/voice_vibecoder/voice_providers/openai.py +0 -0
|
@@ -2,6 +2,39 @@
|
|
|
2
2
|
|
|
3
3
|
<!-- version list -->
|
|
4
4
|
|
|
5
|
+
## v2.24.1 (2026-07-13)
|
|
6
|
+
|
|
7
|
+
### Bug Fixes
|
|
8
|
+
|
|
9
|
+
- **ui**: Show untracked files in the file tree sidebar
|
|
10
|
+
([`e7a5de7`](https://github.com/snokam/voice-vibecoder/commit/e7a5de72cc8597f1d67df01faf29f42d06edd2bc))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
## v2.24.0 (2026-07-13)
|
|
14
|
+
|
|
15
|
+
### Bug Fixes
|
|
16
|
+
|
|
17
|
+
- **providers**: Use whisper-1 transcription on Azure realtime
|
|
18
|
+
([`4e487a9`](https://github.com/snokam/voice-vibecoder/commit/4e487a9d45ad5d7f870d51a7bd72e7f05d4ab2c0))
|
|
19
|
+
|
|
20
|
+
- **session**: Stop the voice cancelling its own responses
|
|
21
|
+
([`4acaa85`](https://github.com/snokam/voice-vibecoder/commit/4acaa85c6c3c38e353904b7e10203f10c9e02734))
|
|
22
|
+
|
|
23
|
+
- **ui**: Update the running install and surface audio diagnostics
|
|
24
|
+
([`d80ca29`](https://github.com/snokam/voice-vibecoder/commit/d80ca29337182d869cb5f3b2d7557879b64ea8e1))
|
|
25
|
+
|
|
26
|
+
- **worktrees**: Prune stale registrations before reuse
|
|
27
|
+
([`31bb71c`](https://github.com/snokam/voice-vibecoder/commit/31bb71c13d53b79594454ad8a4490c496e1d7336))
|
|
28
|
+
|
|
29
|
+
### Features
|
|
30
|
+
|
|
31
|
+
- **audio**: Full-duplex voice with macOS echo cancellation
|
|
32
|
+
([`303b4b1`](https://github.com/snokam/voice-vibecoder/commit/303b4b161577c49ddc021882b4ada7027b04e4d8))
|
|
33
|
+
|
|
34
|
+
- **voice**: GA voices (marin/cedar) and shorter spoken answers
|
|
35
|
+
([`ffd36ed`](https://github.com/snokam/voice-vibecoder/commit/ffd36ed14ff21ae1ce28957258cfee3ca4e228b2))
|
|
36
|
+
|
|
37
|
+
|
|
5
38
|
## v2.23.0 (2026-06-30)
|
|
6
39
|
|
|
7
40
|
### Features
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: voice-vibecoder
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.24.1
|
|
4
4
|
Summary: Olaf The Vibecoder — a voice-controlled coding assistant using OpenAI Realtime API + Claude Code
|
|
5
5
|
Project-URL: Homepage, https://github.com/snokam/voice-vibecoder
|
|
6
6
|
Project-URL: Repository, https://github.com/snokam/voice-vibecoder
|
|
@@ -24,6 +24,7 @@ Requires-Dist: claude-agent-sdk>=0.1.30
|
|
|
24
24
|
Requires-Dist: cryptography>=42.0
|
|
25
25
|
Requires-Dist: numpy>=1.24
|
|
26
26
|
Requires-Dist: platformdirs>=4.0
|
|
27
|
+
Requires-Dist: pyobjc-framework-avfoundation>=10.0; sys_platform == 'darwin'
|
|
27
28
|
Requires-Dist: rich>=13.0
|
|
28
29
|
Requires-Dist: sounddevice>=0.4
|
|
29
30
|
Requires-Dist: textual>=0.50
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "voice-vibecoder"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.24.1"
|
|
8
8
|
description = "Olaf The Vibecoder — a voice-controlled coding assistant using OpenAI Realtime API + Claude Code"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -32,6 +32,7 @@ dependencies = [
|
|
|
32
32
|
"numpy>=1.24",
|
|
33
33
|
"claude-agent-sdk>=0.1.30",
|
|
34
34
|
"platformdirs>=4.0",
|
|
35
|
+
"pyobjc-framework-avfoundation>=10.0; sys_platform == 'darwin'",
|
|
35
36
|
]
|
|
36
37
|
|
|
37
38
|
[project.urls]
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
"""Terminal audio I/O using sounddevice for mic capture and speaker playback."""
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import logging
|
|
5
|
+
import queue
|
|
6
|
+
import sys
|
|
7
|
+
import threading
|
|
8
|
+
import time
|
|
9
|
+
from collections import deque
|
|
10
|
+
from typing import Callable
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
import sounddevice as sd
|
|
14
|
+
|
|
15
|
+
from voice_vibecoder.config import (
|
|
16
|
+
AUDIO_SAMPLE_RATE,
|
|
17
|
+
AUDIO_CHANNELS,
|
|
18
|
+
AUDIO_DTYPE,
|
|
19
|
+
CHUNK_SIZE,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
POST_PLAYBACK_HANGOVER_SEC = 0.4
|
|
23
|
+
RESPONSE_GAP_BRIDGE_SEC = 2.0
|
|
24
|
+
|
|
25
|
+
BARGE_RMS_FACTOR = 3.0
|
|
26
|
+
BARGE_MIN_RMS = 400.0
|
|
27
|
+
BARGE_ONSET_FRAMES = 2
|
|
28
|
+
BARGE_CALIBRATION_FRAMES = 5
|
|
29
|
+
_PREBUFFER_FRAMES = 3
|
|
30
|
+
|
|
31
|
+
_CHUNK_BYTES = CHUNK_SIZE * 2
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def create_audio_manager(on_audio_chunk: Callable[[str], None]):
|
|
37
|
+
"""Best available audio backend: macOS voice-processing (system echo
|
|
38
|
+
cancellation, true full duplex) when available, sounddevice otherwise."""
|
|
39
|
+
if sys.platform == "darwin":
|
|
40
|
+
try:
|
|
41
|
+
from voice_vibecoder.audio_macos import VoiceProcessingAudioManager
|
|
42
|
+
|
|
43
|
+
return VoiceProcessingAudioManager(on_audio_chunk)
|
|
44
|
+
except Exception as e:
|
|
45
|
+
logger.info("VPIO unavailable, using sounddevice: %s", e)
|
|
46
|
+
return AudioManager(on_audio_chunk)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class AudioManager:
|
|
50
|
+
"""Manages mic input and speaker output for the Realtime API.
|
|
51
|
+
|
|
52
|
+
Audio format: PCM16, 24 kHz, mono — matches OpenAI Realtime requirements.
|
|
53
|
+
While assistant audio plays, mic input passes through an echo-aware
|
|
54
|
+
barge-in gate: speech clearly louder than the measured speaker-echo
|
|
55
|
+
floor is forwarded (so the user can interrupt), everything else is
|
|
56
|
+
dropped (so the assistant's own echo can't trip the server VAD).
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, on_audio_chunk: Callable[[str], None]) -> None:
|
|
60
|
+
self._on_audio_chunk = on_audio_chunk
|
|
61
|
+
self._input_stream: sd.InputStream | None = None
|
|
62
|
+
self._output_stream: sd.OutputStream | None = None
|
|
63
|
+
self._playback_queue: queue.Queue[bytes] = queue.Queue()
|
|
64
|
+
self._muted = False
|
|
65
|
+
self._writing = False
|
|
66
|
+
self._abort_playback = False
|
|
67
|
+
self._response_active = False
|
|
68
|
+
self._last_play_end = 0.0
|
|
69
|
+
self._running = False
|
|
70
|
+
self._playback_thread: threading.Thread | None = None
|
|
71
|
+
self._echo_floor = 0.0
|
|
72
|
+
self._loud_streak = 0
|
|
73
|
+
self._gate_open = False
|
|
74
|
+
self._cal_frames = 0
|
|
75
|
+
self._prebuffer: deque[bytes] = deque(maxlen=_PREBUFFER_FRAMES)
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def muted(self) -> bool:
|
|
79
|
+
return self._muted
|
|
80
|
+
|
|
81
|
+
def start(self) -> None:
|
|
82
|
+
self._running = True
|
|
83
|
+
self._start_input()
|
|
84
|
+
self._start_output()
|
|
85
|
+
|
|
86
|
+
def _start_input(self) -> None:
|
|
87
|
+
self._input_stream = sd.InputStream(
|
|
88
|
+
samplerate=AUDIO_SAMPLE_RATE,
|
|
89
|
+
channels=AUDIO_CHANNELS,
|
|
90
|
+
dtype=AUDIO_DTYPE,
|
|
91
|
+
blocksize=CHUNK_SIZE,
|
|
92
|
+
callback=self._mic_callback,
|
|
93
|
+
)
|
|
94
|
+
self._input_stream.start()
|
|
95
|
+
|
|
96
|
+
def _start_output(self) -> None:
|
|
97
|
+
self._output_stream = sd.OutputStream(
|
|
98
|
+
samplerate=AUDIO_SAMPLE_RATE,
|
|
99
|
+
channels=AUDIO_CHANNELS,
|
|
100
|
+
dtype=AUDIO_DTYPE,
|
|
101
|
+
blocksize=CHUNK_SIZE,
|
|
102
|
+
)
|
|
103
|
+
self._output_stream.start()
|
|
104
|
+
self._playback_thread = threading.Thread(
|
|
105
|
+
target=self._playback_loop, daemon=True
|
|
106
|
+
)
|
|
107
|
+
self._playback_thread.start()
|
|
108
|
+
|
|
109
|
+
def stop(self) -> None:
|
|
110
|
+
self._running = False
|
|
111
|
+
if self._input_stream:
|
|
112
|
+
self._input_stream.stop()
|
|
113
|
+
self._input_stream.close()
|
|
114
|
+
self._input_stream = None
|
|
115
|
+
if self._output_stream:
|
|
116
|
+
self._output_stream.stop()
|
|
117
|
+
self._output_stream.close()
|
|
118
|
+
self._output_stream = None
|
|
119
|
+
self._playback_queue.put(b"")
|
|
120
|
+
|
|
121
|
+
def set_muted(self, muted: bool) -> None:
|
|
122
|
+
self._muted = muted
|
|
123
|
+
|
|
124
|
+
def set_response_active(self, active: bool) -> None:
|
|
125
|
+
"""Called by the session when a model response starts/finishes."""
|
|
126
|
+
self._response_active = active
|
|
127
|
+
|
|
128
|
+
def play_audio(self, base64_pcm: str) -> None:
|
|
129
|
+
pcm_data = base64.b64decode(base64_pcm)
|
|
130
|
+
self._playback_queue.put(pcm_data)
|
|
131
|
+
|
|
132
|
+
def clear_playback(self) -> None:
|
|
133
|
+
"""Stop playback and free the mic immediately — the user took the floor."""
|
|
134
|
+
self._abort_playback = True
|
|
135
|
+
while not self._playback_queue.empty():
|
|
136
|
+
try:
|
|
137
|
+
self._playback_queue.get_nowait()
|
|
138
|
+
except queue.Empty:
|
|
139
|
+
break
|
|
140
|
+
self._last_play_end = 0.0
|
|
141
|
+
self._reset_barge_state()
|
|
142
|
+
|
|
143
|
+
def _mic_suppressed(self) -> bool:
|
|
144
|
+
if self._writing or not self._playback_queue.empty():
|
|
145
|
+
return True
|
|
146
|
+
idle = time.monotonic() - self._last_play_end
|
|
147
|
+
limit = (
|
|
148
|
+
RESPONSE_GAP_BRIDGE_SEC
|
|
149
|
+
if self._response_active
|
|
150
|
+
else POST_PLAYBACK_HANGOVER_SEC
|
|
151
|
+
)
|
|
152
|
+
return idle < limit
|
|
153
|
+
|
|
154
|
+
def _barge_gate(self, rms: float) -> bool:
|
|
155
|
+
"""Decide whether mic audio during playback is the user interrupting.
|
|
156
|
+
|
|
157
|
+
Tracks the speaker-echo level in _echo_floor; opens only for
|
|
158
|
+
sustained input well above it. Closed frames feed the floor
|
|
159
|
+
estimate, so the gate adapts to the current playback volume.
|
|
160
|
+
"""
|
|
161
|
+
if self._cal_frames < BARGE_CALIBRATION_FRAMES:
|
|
162
|
+
self._cal_frames += 1
|
|
163
|
+
self._echo_floor = max(self._echo_floor, rms)
|
|
164
|
+
return False
|
|
165
|
+
threshold = max(BARGE_MIN_RMS, BARGE_RMS_FACTOR * self._echo_floor)
|
|
166
|
+
if self._gate_open:
|
|
167
|
+
if rms < threshold * 0.5:
|
|
168
|
+
self._gate_open = False
|
|
169
|
+
self._loud_streak = 0
|
|
170
|
+
return self._gate_open
|
|
171
|
+
if rms >= threshold:
|
|
172
|
+
self._loud_streak += 1
|
|
173
|
+
if self._loud_streak >= BARGE_ONSET_FRAMES:
|
|
174
|
+
self._gate_open = True
|
|
175
|
+
return True
|
|
176
|
+
else:
|
|
177
|
+
self._loud_streak = 0
|
|
178
|
+
self._echo_floor = (
|
|
179
|
+
0.8 * self._echo_floor + 0.2 * rms if self._echo_floor else rms
|
|
180
|
+
)
|
|
181
|
+
return False
|
|
182
|
+
|
|
183
|
+
def _reset_barge_state(self) -> None:
|
|
184
|
+
self._gate_open = False
|
|
185
|
+
self._loud_streak = 0
|
|
186
|
+
self._echo_floor = 0.0
|
|
187
|
+
self._cal_frames = 0
|
|
188
|
+
self._prebuffer.clear()
|
|
189
|
+
|
|
190
|
+
def _mic_callback(self, indata: np.ndarray, frames: int, time_info, status) -> None:
|
|
191
|
+
if self._muted or not self._running:
|
|
192
|
+
return
|
|
193
|
+
pcm_bytes = indata.tobytes()
|
|
194
|
+
if self._mic_suppressed():
|
|
195
|
+
rms = float(np.sqrt(np.mean(indata.astype(np.float64) ** 2)))
|
|
196
|
+
if not self._barge_gate(rms):
|
|
197
|
+
self._prebuffer.append(pcm_bytes)
|
|
198
|
+
return
|
|
199
|
+
while self._prebuffer:
|
|
200
|
+
buffered = self._prebuffer.popleft()
|
|
201
|
+
self._on_audio_chunk(base64.b64encode(buffered).decode("ascii"))
|
|
202
|
+
else:
|
|
203
|
+
self._reset_barge_state()
|
|
204
|
+
encoded = base64.b64encode(pcm_bytes).decode("ascii")
|
|
205
|
+
self._on_audio_chunk(encoded)
|
|
206
|
+
|
|
207
|
+
def _playback_loop(self) -> None:
|
|
208
|
+
while self._running:
|
|
209
|
+
try:
|
|
210
|
+
pcm_data = self._playback_queue.get(timeout=0.5)
|
|
211
|
+
except queue.Empty:
|
|
212
|
+
continue
|
|
213
|
+
if not pcm_data or not self._running:
|
|
214
|
+
break
|
|
215
|
+
self._abort_playback = False
|
|
216
|
+
self._writing = True
|
|
217
|
+
try:
|
|
218
|
+
for i in range(0, len(pcm_data), _CHUNK_BYTES):
|
|
219
|
+
if self._abort_playback or not self._running:
|
|
220
|
+
break
|
|
221
|
+
chunk = np.frombuffer(
|
|
222
|
+
pcm_data[i : i + _CHUNK_BYTES], dtype=np.int16
|
|
223
|
+
).reshape(-1, 1)
|
|
224
|
+
if self._output_stream:
|
|
225
|
+
self._output_stream.write(chunk)
|
|
226
|
+
except Exception:
|
|
227
|
+
pass
|
|
228
|
+
finally:
|
|
229
|
+
self._writing = False
|
|
230
|
+
if not self._abort_playback:
|
|
231
|
+
self._last_play_end = time.monotonic()
|
|
232
|
+
self._writing = False
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""macOS echo-cancelled capture via AVAudioEngine Voice Processing I/O.
|
|
2
|
+
|
|
3
|
+
macOS's system echo canceller (the one FaceTime uses) removes the app's own
|
|
4
|
+
speaker output from the mic signal at the device level — verified to work
|
|
5
|
+
even when playback goes through sounddevice. With echo gone, the mic stays
|
|
6
|
+
open the whole time: the user can interrupt the assistant mid-sentence and
|
|
7
|
+
the server VAD handles it, with no half-duplex suppression or gating.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import base64
|
|
13
|
+
import logging
|
|
14
|
+
from typing import Callable
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
from voice_vibecoder.audio import AudioManager
|
|
19
|
+
from voice_vibecoder.config import AUDIO_SAMPLE_RATE
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class VoiceProcessingAudioManager(AudioManager):
|
|
25
|
+
"""AudioManager with full-duplex, echo-cancelled mic capture.
|
|
26
|
+
|
|
27
|
+
Playback is inherited from AudioManager (sounddevice). Capture comes
|
|
28
|
+
from an AVAudioEngine input node with voice processing enabled;
|
|
29
|
+
start() raises if the engine can't start, letting the caller fall
|
|
30
|
+
back to the plain half-duplex AudioManager.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
def __init__(self, on_audio_chunk: Callable[[str], None]) -> None:
|
|
34
|
+
super().__init__(on_audio_chunk)
|
|
35
|
+
self._engine = None
|
|
36
|
+
self._in_rate = AUDIO_SAMPLE_RATE
|
|
37
|
+
|
|
38
|
+
def _start_input(self) -> None:
|
|
39
|
+
from AVFoundation import AVAudioEngine
|
|
40
|
+
|
|
41
|
+
engine = AVAudioEngine.alloc().init()
|
|
42
|
+
input_node = engine.inputNode()
|
|
43
|
+
ok, err = input_node.setVoiceProcessingEnabled_error_(True, None)
|
|
44
|
+
if not ok:
|
|
45
|
+
raise RuntimeError(f"voice processing unavailable: {err}")
|
|
46
|
+
|
|
47
|
+
in_fmt = input_node.outputFormatForBus_(0)
|
|
48
|
+
self._in_rate = int(in_fmt.sampleRate()) or AUDIO_SAMPLE_RATE
|
|
49
|
+
|
|
50
|
+
def tap(buf, when) -> None:
|
|
51
|
+
self._handle_tap(buf)
|
|
52
|
+
|
|
53
|
+
input_node.installTapOnBus_bufferSize_format_block_(
|
|
54
|
+
0, max(256, self._in_rate // 10), None, tap
|
|
55
|
+
)
|
|
56
|
+
engine.prepare()
|
|
57
|
+
ok, err = engine.startAndReturnError_(None)
|
|
58
|
+
if not ok:
|
|
59
|
+
raise RuntimeError(f"AVAudioEngine start failed: {err}")
|
|
60
|
+
self._engine = engine
|
|
61
|
+
logger.info("VPIO capture started (%d Hz)", self._in_rate)
|
|
62
|
+
|
|
63
|
+
def _handle_tap(self, buf) -> None:
|
|
64
|
+
if not self._running or self._muted:
|
|
65
|
+
return
|
|
66
|
+
try:
|
|
67
|
+
frames = int(buf.frameLength())
|
|
68
|
+
if not frames:
|
|
69
|
+
return
|
|
70
|
+
channel = buf.floatChannelData()[0]
|
|
71
|
+
data = np.frombuffer(channel.as_buffer(frames), dtype=np.float32)
|
|
72
|
+
if self._in_rate != AUDIO_SAMPLE_RATE:
|
|
73
|
+
out_len = int(frames * AUDIO_SAMPLE_RATE / self._in_rate)
|
|
74
|
+
data = np.interp(
|
|
75
|
+
np.linspace(0.0, frames - 1, out_len),
|
|
76
|
+
np.arange(frames),
|
|
77
|
+
data,
|
|
78
|
+
)
|
|
79
|
+
pcm16 = np.clip(data * 32768.0, -32768, 32767).astype(np.int16)
|
|
80
|
+
self._on_audio_chunk(base64.b64encode(pcm16.tobytes()).decode("ascii"))
|
|
81
|
+
except Exception as e:
|
|
82
|
+
logger.warning("VPIO tap error: %s", e)
|
|
83
|
+
|
|
84
|
+
def stop(self) -> None:
|
|
85
|
+
engine = self._engine
|
|
86
|
+
self._engine = None
|
|
87
|
+
if engine:
|
|
88
|
+
try:
|
|
89
|
+
engine.inputNode().removeTapOnBus_(0)
|
|
90
|
+
engine.stop()
|
|
91
|
+
except Exception:
|
|
92
|
+
pass
|
|
93
|
+
super().stop()
|
|
@@ -31,7 +31,8 @@ def _agent_names(enabled: list[str] | None = None) -> str:
|
|
|
31
31
|
_SYSTEM_INSTRUCTIONS: dict[Language, str] = {
|
|
32
32
|
"no": """\
|
|
33
33
|
Du er Olaf, en hjelpsom stemmeassistent for utviklere. Snakk alltid norsk. \
|
|
34
|
-
Brukeren skal oppleve en helt vanlig samtale — du bare hjelper.
|
|
34
|
+
Brukeren skal oppleve en helt vanlig samtale — du bare hjelper. \
|
|
35
|
+
Dette er en talesamtale: svar kort, én til to setninger, med mindre brukeren ber om mer.
|
|
35
36
|
|
|
36
37
|
Svar direkte på:
|
|
37
38
|
- Hilsener, småprat, korte spørsmål
|
|
@@ -84,7 +85,8 @@ Spørsmål fra agenten:
|
|
|
84
85
|
- Still spørsmålet til brukeren. Videresend svaret ordrett via answer_agent_question.""",
|
|
85
86
|
"en": """\
|
|
86
87
|
You are Olaf, a helpful voice assistant for developers. Always speak English. \
|
|
87
|
-
The user should experience a normal conversation — you just help.
|
|
88
|
+
The user should experience a normal conversation — you just help. \
|
|
89
|
+
This is a spoken conversation: keep answers short, one or two sentences, unless the user asks for more.
|
|
88
90
|
|
|
89
91
|
Answer directly:
|
|
90
92
|
- Greetings, small talk, short questions
|
|
@@ -137,7 +139,8 @@ Questions from the agent:
|
|
|
137
139
|
- Ask the user the question. Forward their answer verbatim via answer_agent_question.""",
|
|
138
140
|
"sv": """\
|
|
139
141
|
Du är Olaf, en hjälpsam röstassistent för utvecklare. Tala alltid svenska. \
|
|
140
|
-
Användaren ska uppleva ett helt vanligt samtal — du bara hjälper.
|
|
142
|
+
Användaren ska uppleva ett helt vanligt samtal — du bara hjälper. \
|
|
143
|
+
Detta är ett röstsamtal: svara kort, en till två meningar, om inte användaren ber om mer.
|
|
141
144
|
|
|
142
145
|
Svara direkt på:
|
|
143
146
|
- Hälsningar, småprat, korta frågor
|
|
@@ -223,6 +226,7 @@ def get_system_instructions(
|
|
|
223
226
|
Provider = Literal["openai", "azure", "gemini"]
|
|
224
227
|
|
|
225
228
|
VOICES = ["ash", "ballad", "coral", "sage", "shimmer", "alloy", "echo", "verse"]
|
|
229
|
+
GA_VOICES = ["marin", "cedar"]
|
|
226
230
|
|
|
227
231
|
OPENAI_WS_URL = "wss://api.openai.com/v1/realtime"
|
|
228
232
|
OPENAI_DEFAULT_MODEL = "gpt-4o-realtime-preview"
|
|
@@ -272,7 +276,7 @@ class AzureConfig:
|
|
|
272
276
|
api_key: str = ""
|
|
273
277
|
endpoint: str = AZURE_DEFAULT_ENDPOINT
|
|
274
278
|
deployment: str = AZURE_DEFAULT_DEPLOYMENT
|
|
275
|
-
voice: str = "
|
|
279
|
+
voice: str = "marin"
|
|
276
280
|
vad_threshold: float = 0.8
|
|
277
281
|
silence_duration_ms: int = 700
|
|
278
282
|
prefix_padding_ms: int = 300
|
|
@@ -134,6 +134,7 @@ class RealtimeSession:
|
|
|
134
134
|
on_instance_status: Callable[[str, InstanceStatus], None] | None = None,
|
|
135
135
|
on_interrupt: Callable[[], None] | None = None,
|
|
136
136
|
on_ui_command: Callable[[str, str, Any], None] | None = None,
|
|
137
|
+
on_response_active: Callable[[bool], None] | None = None,
|
|
137
138
|
) -> None:
|
|
138
139
|
self._provider = provider
|
|
139
140
|
self._settings = settings
|
|
@@ -148,6 +149,7 @@ class RealtimeSession:
|
|
|
148
149
|
self._on_instance_status = on_instance_status
|
|
149
150
|
self._on_interrupt = on_interrupt
|
|
150
151
|
self._on_ui_command = on_ui_command
|
|
152
|
+
self._on_response_active = on_response_active
|
|
151
153
|
self._running = False
|
|
152
154
|
self._assistant_transcript = ""
|
|
153
155
|
self.text_only = False
|
|
@@ -155,6 +157,9 @@ class RealtimeSession:
|
|
|
155
157
|
self._fallback_task: asyncio.Task | None = None
|
|
156
158
|
self._loop: asyncio.AbstractEventLoop | None = None
|
|
157
159
|
self._last_tool_narration: float = 0.0
|
|
160
|
+
self._response_active = False
|
|
161
|
+
self._user_speaking = False
|
|
162
|
+
self._pending_injections: list[str] = []
|
|
158
163
|
|
|
159
164
|
# Create instance registry and wire callbacks
|
|
160
165
|
self._registry = InstanceRegistry(user_email=user_email, user_jwt=user_jwt)
|
|
@@ -281,6 +286,11 @@ class RealtimeSession:
|
|
|
281
286
|
# Store in registry for state persistence
|
|
282
287
|
self._registry.add_transcript("user", text)
|
|
283
288
|
|
|
289
|
+
if self._response_active:
|
|
290
|
+
await self._provider.cancel_response()
|
|
291
|
+
if self._on_interrupt:
|
|
292
|
+
self._on_interrupt()
|
|
293
|
+
|
|
284
294
|
# Send to voice provider to generate response
|
|
285
295
|
await self._provider.inject_message(text)
|
|
286
296
|
|
|
@@ -325,28 +335,35 @@ class RealtimeSession:
|
|
|
325
335
|
self._on_transcript("user", t)
|
|
326
336
|
|
|
327
337
|
case SpeechStarted():
|
|
328
|
-
|
|
338
|
+
self._user_speaking = True
|
|
339
|
+
if self._response_active:
|
|
340
|
+
await self._provider.cancel_response()
|
|
329
341
|
if self._on_interrupt:
|
|
330
342
|
self._on_interrupt()
|
|
331
343
|
if self._on_status:
|
|
332
344
|
self._on_status("listening")
|
|
333
345
|
|
|
334
346
|
case SpeechStopped():
|
|
347
|
+
self._user_speaking = False
|
|
335
348
|
if self._on_status:
|
|
336
349
|
self._on_status("processing")
|
|
337
350
|
|
|
338
351
|
case AudioCommitted():
|
|
352
|
+
self._user_speaking = False
|
|
339
353
|
self._response_pending = True
|
|
340
354
|
self._start_fallback()
|
|
341
355
|
|
|
342
356
|
case ResponseCreated():
|
|
343
357
|
self._response_pending = False
|
|
344
358
|
self._cancel_fallback()
|
|
359
|
+
self._set_response_active(True)
|
|
345
360
|
|
|
346
361
|
case FunctionCall(call_id=cid, name=n, arguments=a):
|
|
347
362
|
await self._handle_function_call(cid, n, a)
|
|
348
363
|
|
|
349
364
|
case ResponseDone():
|
|
365
|
+
self._set_response_active(False)
|
|
366
|
+
await self._flush_pending_injections()
|
|
350
367
|
if self._on_status:
|
|
351
368
|
self._on_status("ready")
|
|
352
369
|
|
|
@@ -356,7 +373,40 @@ class RealtimeSession:
|
|
|
356
373
|
if self._on_status:
|
|
357
374
|
self._on_status(f"error: {m}")
|
|
358
375
|
|
|
376
|
+
def _set_response_active(self, active: bool) -> None:
|
|
377
|
+
self._response_active = active
|
|
378
|
+
if self._on_response_active:
|
|
379
|
+
try:
|
|
380
|
+
self._on_response_active(active)
|
|
381
|
+
except Exception:
|
|
382
|
+
pass
|
|
383
|
+
|
|
384
|
+
@property
|
|
385
|
+
def _conversation_busy(self) -> bool:
|
|
386
|
+
"""True while the assistant is responding or the user is speaking."""
|
|
387
|
+
return self._response_active or self._user_speaking
|
|
388
|
+
|
|
389
|
+
async def _inject_or_queue(self, text: str) -> None:
|
|
390
|
+
"""Deliver a background message once the floor is free (see
|
|
391
|
+
_flush_pending_injections); injecting mid-response or while the
|
|
392
|
+
user speaks made the assistant talk over the conversation."""
|
|
393
|
+
if self._conversation_busy:
|
|
394
|
+
self._pending_injections.append(text)
|
|
395
|
+
return
|
|
396
|
+
await self._provider.inject_message(text)
|
|
397
|
+
|
|
398
|
+
async def _flush_pending_injections(self) -> None:
|
|
399
|
+
if self._user_speaking or not self._pending_injections:
|
|
400
|
+
return
|
|
401
|
+
text = self._pending_injections.pop(0)
|
|
402
|
+
try:
|
|
403
|
+
await self._provider.inject_message(text)
|
|
404
|
+
except Exception as e:
|
|
405
|
+
logger.error("Failed to flush pending injection: %s", e)
|
|
406
|
+
|
|
359
407
|
async def _attempt_reconnect(self) -> None:
|
|
408
|
+
self._set_response_active(False)
|
|
409
|
+
self._user_speaking = False
|
|
360
410
|
for attempt in range(RECONNECT_MAX_ATTEMPTS):
|
|
361
411
|
delay = RECONNECT_DELAYS[min(attempt, len(RECONNECT_DELAYS) - 1)]
|
|
362
412
|
if self._on_status:
|
|
@@ -379,6 +429,8 @@ class RealtimeSession:
|
|
|
379
429
|
|
|
380
430
|
async def disconnect(self) -> None:
|
|
381
431
|
self._running = False
|
|
432
|
+
self._set_response_active(False)
|
|
433
|
+
self._user_speaking = False
|
|
382
434
|
await self._provider.disconnect()
|
|
383
435
|
if self._on_status:
|
|
384
436
|
self._on_status("disconnected")
|
|
@@ -403,6 +455,8 @@ class RealtimeSession:
|
|
|
403
455
|
return
|
|
404
456
|
if not isinstance(line, str):
|
|
405
457
|
return
|
|
458
|
+
if self._conversation_busy:
|
|
459
|
+
return
|
|
406
460
|
hint = _friendly_tool_hint(line)
|
|
407
461
|
if not hint:
|
|
408
462
|
return
|
|
@@ -478,7 +532,7 @@ class RealtimeSession:
|
|
|
478
532
|
agent_label = get_agent(agent_type).label
|
|
479
533
|
system_text = f"{agent_label} finished on '{display_key}':\n\n{result}"
|
|
480
534
|
try:
|
|
481
|
-
await self.
|
|
535
|
+
await self._inject_or_queue(system_text)
|
|
482
536
|
except Exception as e:
|
|
483
537
|
logger.error("Failed to inject agent result: %s", e)
|
|
484
538
|
if self._on_status:
|
|
@@ -526,7 +580,7 @@ class RealtimeSession:
|
|
|
526
580
|
system_text = f"{label} on '{display_key}' is asking:\n\n{formatted}"
|
|
527
581
|
|
|
528
582
|
try:
|
|
529
|
-
await self.
|
|
583
|
+
await self._inject_or_queue(system_text)
|
|
530
584
|
except Exception as e:
|
|
531
585
|
logger.error("Failed to inject agent question: %s", e)
|
|
532
586
|
|
|
@@ -163,6 +163,11 @@ def get_file_tree(worktree_path: str) -> dict:
|
|
|
163
163
|
if f:
|
|
164
164
|
all_paths[f] = changed.get(f, "unchanged")
|
|
165
165
|
|
|
166
|
+
# Untracked/new files aren't in ls-files but must show in the tree
|
|
167
|
+
for path, status in changed.items():
|
|
168
|
+
if path not in all_paths and status != "deleted":
|
|
169
|
+
all_paths[path] = status
|
|
170
|
+
|
|
166
171
|
# Build tree structure
|
|
167
172
|
tree = {
|
|
168
173
|
"name": root,
|
|
@@ -15,8 +15,15 @@ from voice_vibecoder.ui.styles import _CATEGORY_STYLES, _format_output_line
|
|
|
15
15
|
class CallbackMixin:
|
|
16
16
|
"""Mixin providing callback methods for VoiceCodingScreen."""
|
|
17
17
|
|
|
18
|
+
_mic_confirmed = False
|
|
19
|
+
|
|
18
20
|
def _send_audio_chunk(self, base64_data: str) -> None:
|
|
19
21
|
if self._loop and self._session and self._session.connected:
|
|
22
|
+
if not self._mic_confirmed:
|
|
23
|
+
self._mic_confirmed = True
|
|
24
|
+
self.app.call_from_thread(
|
|
25
|
+
self._log, "[dim]mic live — audio streaming[/dim]"
|
|
26
|
+
)
|
|
20
27
|
asyncio.run_coroutine_threadsafe(
|
|
21
28
|
self._session.send_audio(base64_data), self._loop
|
|
22
29
|
)
|
|
@@ -39,6 +46,10 @@ class CallbackMixin:
|
|
|
39
46
|
if self._audio:
|
|
40
47
|
self._audio.clear_playback()
|
|
41
48
|
|
|
49
|
+
def _on_response_active(self, active: bool) -> None:
|
|
50
|
+
if self._audio:
|
|
51
|
+
self._audio.set_response_active(active)
|
|
52
|
+
|
|
42
53
|
_prev_category: dict[str, str | None] = {}
|
|
43
54
|
|
|
44
55
|
def _on_agent_output(self, instance_id: str, line: str) -> None:
|
|
@@ -133,6 +144,8 @@ class CallbackMixin:
|
|
|
133
144
|
self._update_file_tree(instance_id, data)
|
|
134
145
|
|
|
135
146
|
def _on_status(self, status: str) -> None:
|
|
147
|
+
if status.startswith("error:"):
|
|
148
|
+
self.app.call_from_thread(self._log, f"[red]{status}[/red]")
|
|
136
149
|
self.app.call_from_thread(self._update_status_bar, status)
|
|
137
150
|
|
|
138
151
|
def _on_tool_call(self, name: str, arguments: dict) -> None:
|