dulus 0.2.0__py3-none-any.whl

voice/recorder.py

@@ -0,0 +1,263 @@
+"""Audio capture for voice input.
+
+Backend priority (tried in order):
+  1. sounddevice   — cross-platform, pure-Python wrapper around PortAudio.
+                     Best option: works on macOS, Linux, Windows.
+                     pip install sounddevice
+  2. arecord       — Linux ALSA utility.  No pip install needed.
+  3. sox rec       — SoX command-line recorder.  Supports silence detection.
+                     sudo apt install sox  /  brew install sox
+
+All backends capture raw PCM: 16 kHz, 16-bit signed little-endian, mono.
+"""
+
+from __future__ import annotations
+
+import io
+import shutil
+import subprocess
+import threading
+from pathlib import Path
+
+SAMPLE_RATE = 16000
+CHANNELS = 1
+DTYPE = "int16"
+BYTES_PER_SAMPLE = 2  # int16
+
+# Silence detection parameters
+SILENCE_THRESHOLD_RMS = 0.012   # fraction of int16 max (0..1)
+SILENCE_DURATION_SECS = 1.8     # stop after this many seconds of silence
+CHUNK_SECS = 0.08               # 80 ms chunks for RMS poll
+
+
+def _has_cmd(cmd: str) -> bool:
+    return shutil.which(cmd) is not None
+
+
+# ── Availability ──────────────────────────────────────────────────────────
+
+def check_recording_availability() -> tuple[bool, str | None]:
+    """Return (available, reason_if_not)."""
+    # sounddevice (ImportError = not installed; OSError = PortAudio library missing)
+    try:
+        import sounddevice  # noqa: F401
+        return True, None
+    except (ImportError, OSError):
+        pass
+
+    # arecord
+    if _has_cmd("arecord"):
+        return True, None
+
+    # sox rec
+    if _has_cmd("rec"):
+        return True, None
+
+    return False, (
+        "No audio recording backend found.\n"
+        "Install one of:\n"
+        "  pip install sounddevice   (recommended, cross-platform)\n"
+        "  sudo apt install alsa-utils  (Linux — provides arecord)\n"
+        "  sudo apt install sox  /  brew install sox  (SoX rec)"
+    )
+
+
+# ── sounddevice backend ───────────────────────────────────────────────────
+
+def list_input_devices() -> list[dict]:
+    """Return a list of available input devices with index and name."""
+    import sounddevice as sd
+    devices = sd.query_devices()
+    result = []
+    for i, d in enumerate(devices):
+        if d["max_input_channels"] > 0:
+            result.append({"index": i, "name": d["name"]})
+    return result
+
+
+def _record_sounddevice(
+    max_seconds: int = 30,
+    on_energy: "callable | None" = None,
+    device_index: "int | None" = None,
+) -> bytes:
+    import sounddevice as sd
+    import numpy as np
+
+    chunk_samples = int(SAMPLE_RATE * CHUNK_SECS)
+    silence_chunks_needed = int(SILENCE_DURATION_SECS / CHUNK_SECS)
+    max_chunks = int(max_seconds / CHUNK_SECS)
+
+    chunks: list[bytes] = []
+    silence_count = 0
+    done_evt = threading.Event()
+
+    def callback(indata: "np.ndarray", frames: int, time_info, status) -> None:
+        nonlocal silence_count
+        mono = indata[:, 0].copy()
+        chunks.append(mono.tobytes())
+
+        # RMS energy (normalised 0..1)
+        rms = float(np.sqrt(np.mean(mono.astype(np.float32) ** 2))) / 32768.0
+        if on_energy:
+            on_energy(rms)
+
+        if rms < SILENCE_THRESHOLD_RMS:
+            silence_count += 1
+        else:
+            silence_count = 0
+
+        # Only auto-stop on silence *after* we have some speech (≥3 chunks with signal)
+        has_speech = len(chunks) >= 3
+        if has_speech and silence_count >= silence_chunks_needed:
+            done_evt.set()
+            raise sd.CallbackStop()
+        if len(chunks) >= max_chunks:
+            done_evt.set()
+            raise sd.CallbackStop()
+
+    stream_kwargs = dict(
+        samplerate=SAMPLE_RATE,
+        channels=CHANNELS,
+        dtype=DTYPE,
+        blocksize=chunk_samples,
+        callback=callback,
+    )
+    if device_index is not None:
+        stream_kwargs["device"] = device_index
+    with sd.InputStream(**stream_kwargs):
+        done_evt.wait(timeout=max_seconds + 2)
+
+    return b"".join(chunks)
+
+
+# ── arecord backend (Linux ALSA) ──────────────────────────────────────────
+
+def _record_arecord(
+    max_seconds: int = 30,
+    on_energy: "callable | None" = None,
+) -> bytes:
+    """Record via arecord.  Silence detection done in Python on the piped PCM."""
+    import numpy as np
+
+    cmd = [
+        "arecord",
+        "-f", "S16_LE",
+        "-r", str(SAMPLE_RATE),
+        "-c", str(CHANNELS),
+        "-t", "raw",
+        "-q",
+        "-d", str(max_seconds),
+        "-",
+    ]
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+
+    chunk_bytes = int(SAMPLE_RATE * CHUNK_SECS) * BYTES_PER_SAMPLE
+    silence_chunks_needed = int(SILENCE_DURATION_SECS / CHUNK_SECS)
+
+    chunks: list[bytes] = []
+    silence_count = 0
+
+    try:
+        while True:
+            raw = proc.stdout.read(chunk_bytes)
+            if not raw:
+                break
+            chunks.append(raw)
+
+            arr = np.frombuffer(raw, dtype=np.int16).astype(np.float32)
+            rms = float(np.sqrt(np.mean(arr ** 2))) / 32768.0
+            if on_energy:
+                on_energy(rms)
+
+            if rms < SILENCE_THRESHOLD_RMS:
+                silence_count += 1
+            else:
+                silence_count = 0
+
+            has_speech = len(chunks) >= 3
+            if has_speech and silence_count >= silence_chunks_needed:
+                break
+    finally:
+        proc.terminate()
+        try:
+            proc.wait(timeout=2)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+
+    return b"".join(chunks)
+
+
+# ── SoX rec backend ───────────────────────────────────────────────────────
+
+def _record_sox(
+    max_seconds: int = 30,
+    on_energy: "callable | None" = None,
+) -> bytes:
+    """Record via SoX `rec` with built-in silence detection."""
+    silence_threshold = "3%"
+    silence_pre_duration = "0.1"
+    silence_post_duration = str(SILENCE_DURATION_SECS)
+
+    cmd = [
+        "rec",
+        "-q",
+        "--buffer", "1024",
+        "-t", "raw",
+        "-r", str(SAMPLE_RATE),
+        "-e", "signed",
+        "-b", "16",
+        "-c", str(CHANNELS),
+        "-",
+        "silence",
+        "1", silence_pre_duration, silence_threshold,
+        "1", silence_post_duration, silence_threshold,
+    ]
+
+    # Honour max_seconds via a timeout
+    try:
+        result = subprocess.run(
+            cmd,
+            capture_output=True,
+            timeout=max_seconds,
+        )
+        return result.stdout
+    except subprocess.TimeoutExpired as e:
+        return e.stdout or b""
+
+
+# ── Public entry point ────────────────────────────────────────────────────
+
+def record_until_silence(
+    max_seconds: int = 30,
+    on_energy: "callable | None" = None,
+    device_index: "int | None" = None,
+) -> bytes:
+    """Record from microphone until silence or max_seconds.
+
+    Returns raw PCM bytes: int16, 16 kHz, mono.
+    Tries backends in order: sounddevice → arecord → sox rec.
+    Raises RuntimeError if no backend is available.
+    """
+    try:
+        import sounddevice  # noqa: F401
+        return _record_sounddevice(max_seconds=max_seconds, on_energy=on_energy, device_index=device_index)
+    except (ImportError, OSError):
+        pass
+
+    if _has_cmd("arecord"):
+        try:
+            import numpy  # noqa: F401
+            return _record_arecord(max_seconds=max_seconds, on_energy=on_energy)
+        except ImportError:
+            # numpy missing — fall through to sox (no RMS feedback)
+            return _record_arecord(max_seconds=max_seconds, on_energy=None)
+
+    if _has_cmd("rec"):
+        return _record_sox(max_seconds=max_seconds, on_energy=on_energy)
+
+    raise RuntimeError(
+        "No audio recording backend found.\n"
+        "Install sounddevice:  pip install sounddevice\n"
+        "Or install arecord:   sudo apt install alsa-utils\n"
+        "Or install SoX:       sudo apt install sox"
+    )

voice/stt.py

@@ -0,0 +1,408 @@
+"""Speech-to-text (STT) backends.
+
+Backend priority (tried in order):
+  1. NVIDIA Riva    — cloud, whisper-large-v3 via gRPC, needs NVIDIA_API_KEY.
+                       pip install nvidia-riva-client
+  2. faster-whisper — local, offline, fast, best for coding vocab.
+                       pip install faster-whisper
+  3. openai-whisper — local, offline, original OpenAI Whisper library.
+                       pip install openai-whisper
+  4. OpenAI Whisper API — cloud, needs OPENAI_API_KEY.
+                          pip install openai  (already in requirements)
+
+All backends receive raw PCM (int16, 16 kHz, mono) and return a text string.
+Keyterms are passed as initial_prompt to local Whisper backends so that
+coding-domain vocabulary (grep, MCP, TypeScript, …) is recognised correctly.
+Riva does not accept initial_prompt; keyterms are ignored on that path.
+"""
+
+from __future__ import annotations
+
+import io
+import os
+import struct
+import tempfile
+from pathlib import Path
+from typing import List, Optional
+
+from .recorder import SAMPLE_RATE, CHANNELS, BYTES_PER_SAMPLE
+
+# ── Cached model handles ──────────────────────────────────────────────────
+
+_faster_whisper_model = None
+_openai_whisper_model = None
+
+# Model size: "tiny", "base", "small", "medium", "large-v2", "large-v3"
+# "base" is a good balance of speed and accuracy for coding dictation.
+# Override with env var DULUS_WHISPER_MODEL.
+DEFAULT_MODEL_SIZE = os.environ.get("DULUS_WHISPER_MODEL", "medium")
+
+# ── NVIDIA Riva (whisper-large-v3 via NVCF gRPC) ─────────────────────────
+RIVA_SERVER       = os.environ.get("DULUS_RIVA_SERVER", "grpc.nvcf.nvidia.com:443")
+RIVA_FUNCTION_ID  = os.environ.get("DULUS_RIVA_FUNCTION_ID",
+                                   "b702f636-f60c-4a3d-a6f4-f3568c13bd7d")
+
+
+def _riva_available() -> bool:
+    """Riva backend is usable iff the client lib is installed AND we have a key."""
+    if not os.environ.get("NVIDIA_API_KEY"):
+        return False
+    try:
+        import riva.client  # noqa: F401
+        return True
+    except ImportError:
+        return False
+
+
+def _transcribe_nvidia_riva(
+    pcm_bytes: bytes,
+    language: Optional[str],
+    translate: bool = False,
+) -> str:
+    """Transcribe via NVIDIA NVCF Riva (whisper-large-v3, gRPC).
+
+    Riva expects a real audio container — we wrap raw PCM in WAV.
+    `language=None` or "auto" → "multi" (Riva auto-detect).
+    `translate=True` adds custom_configuration "task:translate" so foreign
+    speech comes back as English.
+    """
+    import riva.client
+    api_key = os.environ["NVIDIA_API_KEY"]
+    auth = riva.client.Auth(
+        None,             # ssl_cert
+        True,             # use_ssl
+        RIVA_SERVER,
+        [("function-id", RIVA_FUNCTION_ID),
+         ("authorization", f"Bearer {api_key}")],
+    )
+    asr = riva.client.ASRService(auth)
+    lang_code = "multi" if (not language or language == "auto") else language
+    config = riva.client.RecognitionConfig(
+        encoding=riva.client.AudioEncoding.LINEAR_PCM,
+        sample_rate_hertz=SAMPLE_RATE,
+        audio_channel_count=CHANNELS,
+        language_code=lang_code,
+        max_alternatives=1,
+        enable_automatic_punctuation=True,
+    )
+    if translate:
+        riva.client.add_custom_configuration_to_config(config, "task:translate")
+    wav = _pcm_to_wav(pcm_bytes)
+    resp = asr.offline_recognize(wav, config)
+    parts = []
+    for r in resp.results:
+        if r.alternatives:
+            parts.append(r.alternatives[0].transcript)
+    return " ".join(parts).strip()
+
+
+# ── OGG/audio file → PCM conversion ──────────────────────────────────────
+
+def _audio_file_to_pcm(audio_bytes: bytes, suffix: str = ".ogg") -> bytes:
+    """Convert an audio file (OGG, MP3, etc.) to raw int16 PCM (16kHz mono) via ffmpeg."""
+    import subprocess
+    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as f:
+        f.write(audio_bytes)
+        f.flush()
+        tmp_in = f.name
+    try:
+        r = subprocess.run(
+            ["ffmpeg", "-y", "-i", tmp_in, "-f", "s16le", "-ar", str(SAMPLE_RATE),
+             "-ac", str(CHANNELS), "-acodec", "pcm_s16le", "-"],
+            capture_output=True, timeout=30,
+        )
+        if r.returncode != 0:
+            raise RuntimeError(f"ffmpeg failed: {r.stderr[:200]}")
+        return r.stdout
+    finally:
+        Path(tmp_in).unlink(missing_ok=True)
+
+
+# ── WAV helper ────────────────────────────────────────────────────────────
+
+def _pcm_to_wav(pcm_bytes: bytes) -> bytes:
+    """Wrap raw int16 PCM in a minimal WAV container."""
+    num_samples = len(pcm_bytes) // BYTES_PER_SAMPLE
+    byte_rate = SAMPLE_RATE * CHANNELS * BYTES_PER_SAMPLE
+    block_align = CHANNELS * BYTES_PER_SAMPLE
+    data_size = len(pcm_bytes)
+    header = struct.pack(
+        "<4sI4s4sIHHIIHH4sI",
+        b"RIFF",
+        36 + data_size,
+        b"WAVE",
+        b"fmt ",
+        16,          # chunk size
+        1,           # PCM format
+        CHANNELS,
+        SAMPLE_RATE,
+        byte_rate,
+        block_align,
+        16,          # bits per sample
+        b"data",
+        data_size,
+    )
+    return header + pcm_bytes
+
+
+# ── Availability ──────────────────────────────────────────────────────────
+
+def check_stt_availability() -> tuple[bool, str | None]:
+    """Return (available, reason_if_not)."""
+    if _riva_available():
+        return True, None
+    try:
+        import faster_whisper  # noqa: F401
+        return True, None
+    except ImportError:
+        pass
+    try:
+        import whisper  # noqa: F401
+        return True, None
+    except ImportError:
+        pass
+    if os.environ.get("OPENAI_API_KEY"):
+        return True, None
+
+    return False, (
+        "No STT backend available.\n"
+        "Install one of:\n"
+        "  pip install nvidia-riva-client  (cloud, whisper-large-v3 — set NVIDIA_API_KEY)\n"
+        "  pip install faster-whisper      (local, recommended)\n"
+        "  pip install openai-whisper      (local, original)\n"
+        "  Set OPENAI_API_KEY to use the OpenAI Whisper cloud API"
+    )
+
+
+def get_stt_backend_name() -> str:
+    """Return a human-readable name of the backend that will be used."""
+    if _riva_available():
+        return "NVIDIA Riva (whisper-large-v3, cloud)"
+    try:
+        import faster_whisper  # noqa: F401
+        return f"faster-whisper ({DEFAULT_MODEL_SIZE})"
+    except ImportError:
+        pass
+    try:
+        import whisper  # noqa: F401
+        return f"openai-whisper ({DEFAULT_MODEL_SIZE})"
+    except ImportError:
+        pass
+    if os.environ.get("OPENAI_API_KEY"):
+        return "OpenAI Whisper API"
+    return "(none)"
+
+
+# ── faster-whisper ────────────────────────────────────────────────────────
+
+def _get_faster_whisper_model():
+    global _faster_whisper_model
+    if _faster_whisper_model is None:
+        from faster_whisper import WhisperModel
+        # Use CPU by default; set device="cuda" if GPU available.
+        device = "cuda" if _has_cuda() else "cpu"
+        compute = "float16" if device == "cuda" else "int8"
+        _faster_whisper_model = WhisperModel(
+            DEFAULT_MODEL_SIZE,
+            device=device,
+            compute_type=compute,
+        )
+    return _faster_whisper_model
+
+
+def _has_cuda() -> bool:
+    try:
+        import torch
+        return torch.cuda.is_available()
+    except ImportError:
+        pass
+    try:
+        import ctranslate2
+        return "cuda" in ctranslate2.get_supported_compute_types("cuda")
+    except Exception:
+        return False
+
+
+def _transcribe_faster_whisper(
+    pcm_bytes: bytes,
+    keyterms: List[str],
+    language: Optional[str],
+) -> str:
+    import numpy as np
+
+    model = _get_faster_whisper_model()
+
+    # Convert int16 PCM to float32 normalised array
+    audio = np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+
+    initial_prompt = _keyterms_to_prompt(keyterms)
+    lang = None if not language or language == "auto" else language
+
+    segments, _info = model.transcribe(
+        audio,
+        language=lang,
+        initial_prompt=initial_prompt,
+        vad_filter=True,          # skip silent regions
+        vad_parameters=dict(
+            min_silence_duration_ms=300,
+        ),
+    )
+    return " ".join(seg.text for seg in segments).strip()
+
+
+# ── openai-whisper ────────────────────────────────────────────────────────
+
+def _get_openai_whisper_model():
+    global _openai_whisper_model
+    if _openai_whisper_model is None:
+        import whisper
+        _openai_whisper_model = whisper.load_model(DEFAULT_MODEL_SIZE)
+    return _openai_whisper_model
+
+
+def _transcribe_openai_whisper(
+    pcm_bytes: bytes,
+    keyterms: List[str],
+    language: Optional[str],
+) -> str:
+    import numpy as np
+
+    model = _get_openai_whisper_model()
+    audio = np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32768.0
+
+    initial_prompt = _keyterms_to_prompt(keyterms)
+    options: dict = {"initial_prompt": initial_prompt} if initial_prompt else {}
+    if language and language != "auto":
+        options["language"] = language
+
+    result = model.transcribe(audio, **options)
+    return result.get("text", "").strip()
+
+
+# ── OpenAI Whisper API ────────────────────────────────────────────────────
+
+def _transcribe_openai_api(
+    pcm_bytes: bytes,
+    language: Optional[str],
+) -> str:
+    from openai import OpenAI
+
+    client = OpenAI()  # uses OPENAI_API_KEY from env
+    wav = _pcm_to_wav(pcm_bytes)
+
+    kwargs: dict = {"model": "whisper-1", "file": ("audio.wav", io.BytesIO(wav), "audio/wav")}
+    if language and language != "auto":
+        kwargs["language"] = language
+
+    transcript = client.audio.transcriptions.create(**kwargs)
+    return transcript.text.strip()
+
+
+# ── Keyterms → prompt ─────────────────────────────────────────────────────
+
+def _keyterms_to_prompt(keyterms: List[str]) -> str:
+    """Convert a list of keywords into a Whisper initial_prompt string.
+
+    Whisper treats the initial_prompt as preceding context; sprinkling the
+    coding vocabulary terms nudges the model to prefer these spellings.
+    """
+    if not keyterms:
+        return ""
+    # Keep it short — Whisper truncates at ~224 tokens.
+    return ", ".join(keyterms[:40])
+
+
+# ── Public entry point ────────────────────────────────────────────────────
+
+def transcribe(
+    pcm_bytes: bytes,
+    keyterms: Optional[List[str]] = None,
+    language: str = "auto",
+) -> str:
+    """Transcribe raw PCM audio to text.
+
+    Args:
+        pcm_bytes: Raw int16 PCM, 16 kHz, mono.
+        keyterms:  Coding-domain vocabulary hints (improves accuracy).
+        language:  BCP-47 language code, or 'auto' for detection.
+
+    Returns:
+        Transcribed text, or empty string if audio contains no speech.
+    """
+    if not pcm_bytes:
+        return ""
+
+    terms = keyterms or []
+    lang = None if language == "auto" else language
+
+    # NVIDIA Riva (whisper-large-v3, cloud) — preferred when configured
+    if _riva_available():
+        try:
+            return _transcribe_nvidia_riva(pcm_bytes, lang)
+        except Exception as e:
+            # Network blip / quota / auth — fall through to local backends
+            print(f"  [STT] Riva failed, falling back: {e}")
+
+    # faster-whisper (local)
+    try:
+        import faster_whisper  # noqa: F401
+        return _transcribe_faster_whisper(pcm_bytes, terms, lang)
+    except ImportError:
+        pass
+
+    # openai-whisper (local, fallback)
+    try:
+        import whisper  # noqa: F401
+        return _transcribe_openai_whisper(pcm_bytes, terms, lang)
+    except ImportError:
+        pass
+
+    # OpenAI Whisper API (cloud, last resort)
+    if os.environ.get("OPENAI_API_KEY"):
+        return _transcribe_openai_api(pcm_bytes, lang)
+
+    raise RuntimeError(
+        "No STT backend available.\n"
+        "Install nvidia-riva-client (set NVIDIA_API_KEY), faster-whisper,\n"
+        "or set OPENAI_API_KEY to use the OpenAI Whisper cloud API."
+    )
+
+
+def transcribe_audio_file(
+    audio_bytes: bytes,
+    suffix: str = ".ogg",
+    language: str = "auto",
+) -> str:
+    """Transcribe an audio file (OGG, MP3, etc.) to text.
+
+    Converts to PCM via ffmpeg, then runs through the STT pipeline.
+    Falls back to OpenAI Whisper API (which accepts OGG natively) if
+    ffmpeg is not available.
+    """
+    # Try ffmpeg conversion → local STT
+    try:
+        pcm = _audio_file_to_pcm(audio_bytes, suffix)
+    except (RuntimeError, FileNotFoundError):
+        pcm = None
+
+    if pcm is not None:
+        try:
+            return transcribe(pcm, language=language)
+        except RuntimeError:
+            pass  # local STT backend failed, fall through to cloud API
+
+    # Fallback: OpenAI Whisper API accepts OGG directly
+    if os.environ.get("OPENAI_API_KEY"):
+        from openai import OpenAI
+        client = OpenAI()
+        kwargs: dict = {"model": "whisper-1", "file": (f"audio{suffix}", io.BytesIO(audio_bytes), "audio/ogg")}
+        lang = None if language == "auto" else language
+        if lang:
+            kwargs["language"] = lang
+        transcript = client.audio.transcriptions.create(**kwargs)
+        return transcript.text.strip()
+
+    raise RuntimeError(
+        "Cannot transcribe audio file.\n"
+        "Install ffmpeg for local conversion, or set OPENAI_API_KEY for cloud STT."
+    )