voice-mcp-server 0.1.0

package/src/adapters_real/kokoro_speaker.py

@@ -0,0 +1,122 @@
+import os
+import time
+import subprocess
+import torch
+import soundfile as sf
+from kokoro import KPipeline
+from simulation.ports import ISpeaker
+from simulation.models import VirtualAudioFrame
+
+class KokoroSpeaker(ISpeaker):
+    def __init__(self, wpm=150, voice="af_heart"): # 'af_heart' is a highly expressive American Female voice
+        self.wpm = wpm
+        self.words_per_ms = (wpm / 60) / 1000
+        self.current_text = ""
+        self.words = []
+        self.process = None
+        self.start_time = 0
+        self.temp_file = "/tmp/kokoro_output.wav"
+        
+        print(f"[DEBUG SPEAKER] Loading local Kokoro TTS model (Voice: {voice})...")
+        # Load the pipeline. Since you are on M4 Max, we will try to use MPS if available
+        if torch.backends.mps.is_available():
+            self.device = "mps"
+        elif torch.cuda.is_available():
+            self.device = "cuda"
+        else:
+            self.device = "cpu"
+            
+        # Initialize the pipeline (downloads weights first time ~300MB)
+        # We use lang_code 'a' for American English
+        self.pipeline = KPipeline(lang_code='a', device=self.device)
+        self.voice = voice
+        print(f"[DEBUG SPEAKER] Kokoro TTS loaded successfully on {self.device}.")
+
+    def speak(self, text: str):
+        if not text.strip():
+            return
+
+        self.current_text = text
+        self.words = text.split()
+        
+        try:
+            print(f"[DEBUG SPEAKER] Generating Kokoro audio for: {text[:50]}...")
+            # Generate the audio locally
+            generator = self.pipeline(
+                text, voice=self.voice, # <= change voice here
+                speed=1.0, split_pattern=r'\n+'
+            )
+            
+            # Since pipeline returns a generator of segments, we can just grab the first one (or concatenate them)
+            # For simplicity in testing, we concatenate them all
+            audio_segments = []
+            for gs, ps, audio in generator:
+                audio_segments.append(audio)
+            
+            if not audio_segments:
+                print("[DEBUG SPEAKER] Kokoro generated empty audio.")
+                return
+                
+            final_audio = torch.cat(audio_segments, dim=0).cpu().numpy()
+            
+            # Save to temporary file at 24kHz (Kokoro's default sample rate)
+            sf.write(self.temp_file, final_audio, 24000)
+            print("[DEBUG SPEAKER] Audio generated, starting playback.")
+            
+            # Play the generated audio using afplay (macOS native)
+            self.start_time = time.time()
+            self.process = subprocess.Popen(
+                ["afplay", self.temp_file],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL
+            )
+            
+        except Exception as e:
+            print(f"[DEBUG SPEAKER] Kokoro Generation Error: {e}")
+            # Fallback to macOS say
+            self.start_time = time.time()
+            self.process = subprocess.Popen(
+                ["say", text],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL
+            )
+
+    def play_frame(self, frame: VirtualAudioFrame):
+        pass
+
+    def tick(self, ms: int):
+        pass
+
+    def is_speaking(self) -> bool:
+        if self.process is None:
+            return False
+        is_running = self.process.poll() is None
+        if not is_running:
+            self.current_text = ""
+            self.words = []
+            self.process = None
+        return is_running
+
+    def has_started_audio(self) -> bool:
+        return self.is_speaking()
+
+    def flush(self) -> str:
+        if not self.is_speaking():
+            return ""
+
+        # Immediately kill the playback process
+        self.process.kill()
+        
+        # Explicitly wait for the process to terminate and reap it
+        self.process.wait()
+        
+        elapsed_ms = (time.time() - self.start_time) * 1000
+        words_spoken = int(elapsed_ms * self.words_per_ms)
+        
+        spoken = " ".join(self.words[:words_spoken])
+        
+        self.current_text = ""
+        self.words = []
+        self.process = None
+        
+        return spoken

package/src/adapters_real/live_mic.py

@@ -0,0 +1,64 @@
+import pyaudio
+import queue
+from simulation.ports import IMicrophone
+from simulation.models import VirtualAudioFrame
+
+class LiveMicrophone(IMicrophone):
+    def __init__(self, rate=16000, chunk=160):
+        self.rate = rate
+        self.chunk = chunk
+        self.q = queue.Queue(maxsize=100)
+        self.p = pyaudio.PyAudio()
+        self.stream = None
+
+    def start_stream(self):
+        if self.stream is not None:
+            return
+            
+        # Clear any stale data from previous sessions
+        while not self.q.empty():
+            try:
+                self.q.get_nowait()
+            except queue.Empty:
+                break
+                
+        self.stream = self.p.open(
+            format=pyaudio.paInt16,
+            channels=1,
+            rate=self.rate,
+            input=True,
+            frames_per_buffer=self.chunk,
+            stream_callback=self._callback
+        )
+        self.stream.start_stream()
+
+    def stop_stream(self):
+        if self.stream is not None:
+            self.stream.stop_stream()
+            self.stream.close()
+            self.stream = None
+
+    def _callback(self, in_data, frame_count, time_info, status):
+        try:
+            self.q.put_nowait(in_data)
+        except queue.Full:
+            pass # drop frames if queue is full rather than blocking audio thread
+        return (None, pyaudio.paContinue)
+
+    def read_frame(self) -> VirtualAudioFrame:
+        if self.stream is None:
+             return VirtualAudioFrame(10, False, False, "", b"")
+             
+        try:
+            raw_bytes = self.q.get(timeout=0.1) # Block briefly to act as clock
+            # If we didn't get 320 bytes, that's weird but we handle it
+            if len(raw_bytes) < self.chunk * 2:
+                return VirtualAudioFrame(10, False, False, "", b"")
+            return VirtualAudioFrame(10, False, False, "", raw_bytes)
+        except queue.Empty:
+            # If queue is empty, yield silence frame
+            return VirtualAudioFrame(10, False, False, "", b"")
+
+    def close(self):
+        self.stop_stream()
+        self.p.terminate()

package/src/adapters_real/live_speaker.py

@@ -0,0 +1,66 @@
+import subprocess
+import time
+from simulation.ports import ISpeaker
+from simulation.models import VirtualAudioFrame
+
+class LiveSpeaker(ISpeaker):
+    def __init__(self, wpm=150):
+        self.wpm = wpm
+        self.words_per_ms = (wpm / 60) / 1000
+        self.current_text = ""
+        self.words = []
+        self.process = None
+        self.start_time = 0
+
+    def speak(self, text: str):
+        self.current_text = text
+        self.words = text.split()
+        self.start_time = time.time()
+        
+        # Start say command non-blocking
+        self.process = subprocess.Popen(
+            ["say", text],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL
+        )
+
+    def play_frame(self, frame: VirtualAudioFrame):
+        pass
+
+    def tick(self, ms: int):
+        pass # Clock is driven by the mic in a real environment
+
+    def is_speaking(self) -> bool:
+        if self.process is None:
+            return False
+        # Poll returns None if process is still running
+        is_running = self.process.poll() is None
+        if not is_running:
+            self.current_text = ""
+            self.words = []
+            self.process = None
+        return is_running
+
+    def has_started_audio(self) -> bool:
+        return self.is_speaking() # Approximation for macOS say
+
+    def flush(self) -> str:
+        if not self.is_speaking():
+            return ""
+
+        # Immediately kill the say process
+        self.process.kill()
+        
+        # Explicitly wait for the process to terminate and reap it
+        self.process.wait()
+        
+        elapsed_ms = (time.time() - self.start_time) * 1000
+        words_spoken = int(elapsed_ms * self.words_per_ms)
+        
+        spoken = " ".join(self.words[:words_spoken])
+        
+        self.current_text = ""
+        self.words = []
+        self.process = None
+        
+        return spoken

package/src/adapters_real/ptt_vad.py

@@ -0,0 +1,36 @@
+from pynput import keyboard
+from simulation.ports import IVAD
+from simulation.models import VirtualAudioFrame
+
+class PushToTalkVAD(IVAD):
+    def __init__(self, key_name="shift", **kwargs):
+        self.is_pressed = False
+        print(f"[DEBUG VAD] Initializing Push-To-Talk VAD. Walkie-Talkie Hotkey: '{key_name}'")
+        
+        # Map string names to pynput Key objects
+        key_map = {
+            "shift": keyboard.Key.shift,
+            "shift_r": keyboard.Key.shift_r,
+            "ctrl": keyboard.Key.ctrl,
+            "alt": keyboard.Key.alt,
+            "cmd": keyboard.Key.cmd,
+            "space": keyboard.Key.space
+        }
+        
+        self.hotkey = key_map.get(key_name.lower(), keyboard.Key.shift)
+
+        def on_press(key):
+            if key == self.hotkey:
+                self.is_pressed = True
+
+        def on_release(key):
+            if key == self.hotkey:
+                self.is_pressed = False
+
+        self.listener = keyboard.Listener(on_press=on_press, on_release=on_release)
+        self.listener.start()
+
+    def analyze(self, frame: VirtualAudioFrame) -> float:
+        # If the key is held down, we return 1.0 (100% certainty of speech).
+        # If the key is released, we return 0.0 (0% certainty of speech).
+        return 1.0 if self.is_pressed else 0.0

package/src/adapters_real/queue_llm.py

@@ -0,0 +1,36 @@
+import queue
+from simulation.ports import ILLMBridge
+
+class QueueLLMBridge(ILLMBridge):
+    def __init__(self, command_queue: queue.Queue, result_queue: queue.Queue):
+        self.cmd_q = command_queue
+        self.res_q = result_queue
+        self.is_requesting = False
+        
+    def call_mcp_tool(self, context: dict) -> dict:
+        self.start_request(context)
+        while True:
+            resp = self.get_response()
+            if resp is not None:
+                return resp
+            import time
+            time.sleep(0.01)
+
+    def start_request(self, context: dict):
+        self.is_requesting = True
+        # Send the transcript to the MCP server
+        self.res_q.put(context)
+
+    def tick(self, ms: int):
+        pass
+
+    def get_response(self) -> dict | None:
+        if self.is_requesting:
+            try:
+                # Non-blocking check for next command from MCP server
+                cmd = self.cmd_q.get_nowait()
+                self.is_requesting = False
+                return cmd
+            except queue.Empty:
+                return None
+        return None

package/src/adapters_real/silero_vad.py

@@ -0,0 +1,43 @@
+import numpy as np
+import torch
+from silero_vad import load_silero_vad
+from simulation.ports import IVAD
+from simulation.models import VirtualAudioFrame
+
+class RealSileroVAD(IVAD):
+    def __init__(self, **kwargs):
+        # Determine if Apple Silicon (MPS) or CUDA is available for hardware acceleration
+        if torch.backends.mps.is_available():
+            self.device = torch.device("mps")
+            print("[DEBUG VAD] Initializing Silero VAD on Apple Metal GPU (MPS)")
+        elif torch.cuda.is_available():
+            self.device = torch.device("cuda")
+        else:
+            self.device = torch.device("cpu")
+            
+        self.model = load_silero_vad().to(self.device)
+        self.buffer = b""
+        self.last_prob = 0.0
+
+    def analyze(self, frame: VirtualAudioFrame) -> float:
+        if not frame.raw_bytes:
+            return 0.0
+            
+        self.buffer += frame.raw_bytes
+        
+        # 512 samples = 1024 bytes
+        if len(self.buffer) >= 1024:
+            chunk = self.buffer[:1024]
+            self.buffer = self.buffer[1024:]
+            
+            audio_int16 = np.frombuffer(chunk, dtype=np.int16)
+            audio_float32 = audio_int16.astype(np.float32) / 32768.0
+            
+            # Move the tensor to the active GPU/CPU device
+            tensor = torch.from_numpy(audio_float32).to(self.device)
+            # silero-vad model natively handles the state!
+            prob = self.model(tensor, 16000).item()
+            self.last_prob = float(prob)
+            # print(f"[DEBUG VAD] Prob: {self.last_prob:.3f}")
+            
+        return self.last_prob

package/src/adapters_real/wav_mic.py

@@ -0,0 +1,17 @@
+import wave
+from simulation.ports import IMicrophone
+from simulation.models import VirtualAudioFrame
+
+class WavFileMicrophone(IMicrophone):
+    def __init__(self, filepath: str):
+        self.wf = wave.open(filepath, 'rb')
+        if self.wf.getnchannels() != 1 or self.wf.getsampwidth() != 2 or self.wf.getframerate() != 16000:
+            raise ValueError("Wav file must be 16kHz, 16-bit, mono")
+            
+    def read_frame(self) -> VirtualAudioFrame:
+        # 16kHz * 2 bytes/sample * 1 channel * 0.010s = 320 bytes
+        # 160 frames at 2 bytes = 320 bytes
+        raw_bytes = self.wf.readframes(160)
+        if len(raw_bytes) < 320:
+            return VirtualAudioFrame(10, False, False, "", b"")
+        return VirtualAudioFrame(10, False, False, "", raw_bytes)

package/src/adapters_real/whisper_stt.py

@@ -0,0 +1,32 @@
+import numpy as np
+import mlx_whisper
+from typing import List
+
+from simulation.ports import ISTT
+from simulation.models import VirtualAudioFrame
+
+class RealWhisperSTT(ISTT):
+    def __init__(self, model_size="mlx-community/whisper-large-v3-mlx"):
+        self.model_size = model_size
+        print(f"[DEBUG STT] Preparing MLX Whisper model ({model_size}) for Apple Silicon...")
+        # MLX will lazily load and compile the model on the first inference, but we print here to indicate we are using the MLX backend.
+
+    def transcribe(self, frames: List[VirtualAudioFrame]) -> str:
+        raw_bytes = b"".join(f.raw_bytes for f in frames if f.raw_bytes)
+        if not raw_bytes:
+            return ""
+        
+        # Convert 16-bit PCM (expected from microphone) to float32 [-1.0, 1.0] expected by Whisper
+        audio_data = np.frombuffer(raw_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+        
+        print(f"[DEBUG STT] Transcribing {len(audio_data)} samples with Apple MLX Whisper ({self.model_size})...")
+        
+        try:
+            # We explicitly set English since you are speaking English, and fp16 for Metal acceleration
+            result = mlx_whisper.transcribe(audio_data, path_or_hf_repo=self.model_size, language="en")
+            text = result.get("text", "").strip()
+            print(f"[DEBUG STT] MLX Transcription result: {text}")
+            return text
+        except Exception as e:
+            print(f"[DEBUG STT] MLX Whisper transcription error: {e}")
+            return ""