interaction-harness 0.1.0__py3-none-any.whl

interaction_harness/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

interaction_harness/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

interaction_harness/cli.py

@@ -0,0 +1,140 @@
+"""interaction-harness command line.
+
+    interaction-harness serve      the dashboard on http://localhost:8765
+    interaction-harness doctor     check all four model roles and report each one's latency
+    interaction-harness selftest   talk to the agent with synthesized speech and check every behaviour
+    interaction-harness warmup     download and load the in-process speech models, then exit
+
+Every subcommand takes the same settings flags; anything not given comes from the environment, .env, or
+interaction-harness.toml (see .env.example). API keys come from the environment or files, never from flags,
+so they stay out of shell history.
+"""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+import threading
+import time
+import webbrowser
+
+from . import __version__
+from .config import LOCAL_BACKENDS, THINKING_MODES, Settings, load_settings
+
+ROLES = ("foreground", "background", "asr", "tts")
+
+
+def _add_settings_flags(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--config", metavar="FILE", help="TOML settings (default: ./interaction-harness.toml if present)")
+    g = p.add_argument_group("model roles", "each role is any OpenAI-compatible endpoint; ASR/TTS run "
+                                            "in-process when their base URL is blank or 'local'")
+    for role in ROLES:
+        g.add_argument(f"--{role}-base-url", metavar="URL")
+        g.add_argument(f"--{role}-model", metavar="NAME")
+    g.add_argument("--foreground-thinking", choices=THINKING_MODES, help="how to switch thinking on that server")
+    g.add_argument("--background-thinking", choices=THINKING_MODES)
+    s = p.add_argument_group("speech")
+    s.add_argument("--tts-voice", metavar="NAME")
+    s.add_argument("--backend", choices=LOCAL_BACKENDS, help="in-process speech (default auto: mlx on Apple "
+                                                             "Silicon, onnx elsewhere)")
+    s.add_argument("--device", choices=("cpu", "cuda"), help="onnx backend device (default cpu)")
+    p.add_argument("--tools", action="append", metavar="FILE", help="Python file of @tool functions (repeatable)")
+    p.add_argument("--host", help="bind address (default 127.0.0.1)")
+    p.add_argument("--port", type=int, help="port (default 8765)")
+
+
+def _flags(args: argparse.Namespace) -> dict:
+    keys = [f"{r}_{f}" for r in ROLES for f in ("base_url", "model")] + [
+        "foreground_thinking", "background_thinking", "tts_voice", "backend", "device", "tools", "host", "port",
+        "ssl_certfile", "ssl_keyfile"]
+    return {k.upper(): getattr(args, k, None) for k in keys}
+
+
+def _settings(args: argparse.Namespace, require_foreground: bool = True) -> Settings:
+    try:
+        return load_settings(_flags(args), config_path=args.config, require_foreground=require_foreground)
+    except (ValueError, FileNotFoundError) as e:
+        sys.exit(f"interaction-harness: {e}")
+
+
+def _local_url(settings: Settings) -> str:
+    host = "localhost" if settings.host in ("127.0.0.1", "0.0.0.0", "::", "localhost") else settings.host
+    return f"{'https' if settings.ssl_certfile else 'http'}://{host}:{settings.port}"
+
+
+def cmd_serve(args: argparse.Namespace) -> None:
+    import httpx
+    import uvicorn
+
+    from .server import create_app
+    settings = _settings(args)
+    url = _local_url(settings)
+    print(f"interaction-harness {__version__}  ->  {url}", flush=True)
+    if settings.host not in ("127.0.0.1", "localhost", "::1") and not settings.ssl_certfile:
+        print("note: browsers only allow the microphone on localhost or HTTPS. From another machine, use an SSH "
+              "tunnel (ssh -L 8765:localhost:8765 this-host) or pass --ssl-certfile/--ssl-keyfile.", flush=True)
+
+    if args.open:
+        def open_when_ready():
+            for _ in range(600):
+                try:
+                    if httpx.get(f"{url}/api/status", timeout=1, verify=False).status_code == 200:
+                        webbrowser.open(url)
+                        return
+                except httpx.HTTPError:
+                    pass
+                time.sleep(1)
+        threading.Thread(target=open_when_ready, daemon=True).start()
+
+    uvicorn.run(create_app(settings), host=settings.host, port=settings.port, log_level="warning",
+                ssl_certfile=settings.ssl_certfile, ssl_keyfile=settings.ssl_keyfile)
+
+
+def cmd_doctor(args: argparse.Namespace) -> None:
+    from .doctor import run
+    sys.exit(asyncio.run(run(_settings(args))))
+
+
+def cmd_selftest(args: argparse.Namespace) -> None:
+    from .selftest import run
+    sys.exit(asyncio.run(run(_settings(args), url=args.url)))
+
+
+def cmd_warmup(args: argparse.Namespace) -> None:
+    from .doctor import warmup
+    sys.exit(asyncio.run(warmup(_settings(args, require_foreground=False))))
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(prog="interaction-harness", description=__doc__.split("\n\n")[0],
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    p = sub.add_parser("serve", help="run the dashboard and the voice session server")
+    _add_settings_flags(p)
+    p.add_argument("--ssl-certfile", metavar="FILE", help="serve HTTPS so a remote browser may use the mic")
+    p.add_argument("--ssl-keyfile", metavar="FILE")
+    p.add_argument("--open", action="store_true", help="open the dashboard in a browser once it is ready")
+    p.set_defaults(fn=cmd_serve)
+
+    p = sub.add_parser("doctor", help="check every model role and report latency")
+    _add_settings_flags(p)
+    p.set_defaults(fn=cmd_doctor)
+
+    p = sub.add_parser("selftest", help="end-to-end test with synthesized speech (starts a server if none is up)")
+    _add_settings_flags(p)
+    p.add_argument("--url", help="WebSocket of a running server (default: ws://127.0.0.1:<port>/ws, started "
+                                 "in-process if nothing is listening)")
+    p.set_defaults(fn=cmd_selftest)
+
+    p = sub.add_parser("warmup", help="download and load the in-process speech models, then exit")
+    _add_settings_flags(p)
+    p.set_defaults(fn=cmd_warmup)
+
+    args = parser.parse_args(argv)
+    args.fn(args)
+
+
+if __name__ == "__main__":
+    main()

interaction_harness/config.py

@@ -0,0 +1,210 @@
+"""Settings: four model roles, each an OpenAI-compatible endpoint, plus the knobs around them.
+
+    ASR         speech -> text           POST {base}/audio/transcriptions
+    TTS         text -> speech           POST {base}/audio/speech
+    FOREGROUND  holds the conversation   POST {base}/chat/completions   (also judges end-of-turn)
+    BACKGROUND  works on handed-off tasks POST {base}/chat/completions
+
+Every role reads <ROLE>_BASE_URL, <ROLE>_API_KEY and <ROLE>_MODEL. ASR and TTS run in-process when their
+base URL is blank or "local"; BACKGROUND falls back to FOREGROUND field by field; FOREGROUND falls back to
+OPENAI_BASE_URL / OPENAI_API_KEY. VAD is not a role: it runs on every 32 ms frame, in-process.
+
+Precedence, highest first: command-line flags, environment, .env in the working directory, the TOML file
+(--config, or ./interaction-harness.toml when it exists).
+"""
+from __future__ import annotations
+
+import os
+import tomllib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from dotenv import dotenv_values
+
+from . import prompts
+
+THINKING_MODES = ("chat_template_kwargs", "reasoning_effort", "off")
+LOCAL_BACKENDS = ("auto", "mlx", "onnx")
+DEFAULT_TOML = "interaction-harness.toml"
+NO_KEY = "not-needed"
+
+
+@dataclass
+class Endpoint:
+    """One model role. `base_url` None means in-process (ASR/TTS only)."""
+
+    role: str
+    base_url: str | None
+    api_key: str
+    model: str
+
+    @property
+    def local(self) -> bool:
+        return self.base_url is None
+
+    @property
+    def host(self) -> str:
+        """Where the role runs, without path or credentials, for status lines and the dashboard."""
+        if self.base_url is None:
+            return "in-process"
+        p = urlparse(self.base_url)
+        return p.hostname + (f":{p.port}" if p.port else "") if p.hostname else self.base_url
+
+
+@dataclass
+class Settings:
+    foreground: Endpoint
+    background: Endpoint
+    asr: Endpoint
+    tts: Endpoint
+    foreground_thinking: str = "chat_template_kwargs"
+    background_thinking: str = "chat_template_kwargs"
+    background_temperature: float = 0.7
+    background_rounds: int = 10          # model calls per background task
+    asr_language: str | None = None
+    asr_partial_ms: int = 640            # partial transcript cadence; 0 = only when the judge needs one
+    tts_voice: str | None = None         # None: the backend's first voice
+    tts_voices: list[str] = field(default_factory=list)
+    tts_format: str = "wav"              # endpoint TTS: wav or pcm
+    tts_sample_rate: int = 24_000        # endpoint TTS with pcm: the server's rate
+    tts_speed: float = 1.05
+    backend: str = "auto"                # in-process speech: auto | mlx | onnx
+    device: str = "cpu"                  # onnx backend: cpu | cuda
+    parakeet_onnx_variant: str = "int8"  # fp32 | int8 (same transcripts in our tests; a quarter the download)
+    kokoro_onnx_variant: str = "int8"    # fp32 | fp16 | int8 (about twice as fast on CPU)
+    foreground_prompt: str = prompts.FOREGROUND_SYSTEM
+    background_prompt: str = prompts.BACKGROUND_SYSTEM
+    end_of_turn_prompt: str = prompts.END_OF_TURN_SYSTEM
+    tools_files: list[str] = field(default_factory=list)
+    save_utterances: str | None = None   # directory: keep a WAV of every user turn the ASR heard
+    host: str = "127.0.0.1"
+    port: int = 8765
+    ssl_certfile: str | None = None
+    ssl_keyfile: str | None = None
+
+    def roles(self) -> dict[str, Endpoint]:
+        return {"asr": self.asr, "tts": self.tts, "foreground": self.foreground, "background": self.background}
+
+
+def _same_host(a: str, b: str) -> bool:
+    pa, pb = urlparse(a), urlparse(b)
+    return bool(pa.hostname) and (pa.scheme, pa.hostname, pa.port) == (pb.scheme, pb.hostname, pb.port)
+
+
+def _flatten_toml(data: dict[str, Any]) -> dict[str, str]:
+    """[foreground] base_url = ... -> FOREGROUND_BASE_URL; [prompts] foreground = ... -> FOREGROUND_PROMPT;
+    [server] port = ... (or a top-level port = ...) -> PORT."""
+    out: dict[str, str] = {}
+    for key, value in data.items():
+        if isinstance(value, dict):
+            for k, v in value.items():
+                name = f"{k}_PROMPT" if key == "prompts" else k if key == "server" else f"{key}_{k}"
+                out[name.upper()] = ",".join(map(str, v)) if isinstance(v, list) else str(v)
+        else:
+            out[key.upper()] = ",".join(map(str, value)) if isinstance(value, list) else str(value)
+    return out
+
+
+def _sources(flags: dict[str, Any] | None, config_path: str | None, env: dict[str, str] | None,
+             dotenv_path: str | None) -> dict[str, str]:
+    merged: dict[str, str] = {}
+    toml_path = Path(config_path) if config_path else Path(DEFAULT_TOML)
+    if config_path and not toml_path.exists():
+        raise FileNotFoundError(f"config file not found: {toml_path}")
+    if toml_path.exists():
+        merged.update(_flatten_toml(tomllib.loads(toml_path.read_text())))
+    dotenv = Path(dotenv_path) if dotenv_path else Path(".env")
+    if dotenv.exists():
+        merged.update({k: v for k, v in dotenv_values(dotenv).items() if v is not None})
+    merged.update(os.environ if env is None else env)
+    for k, v in (flags or {}).items():
+        if v is not None:
+            merged[k.upper()] = ",".join(map(str, v)) if isinstance(v, list) else str(v)
+    return merged
+
+
+def load_settings(flags: dict[str, Any] | None = None, *, config_path: str | None = None,
+                  env: dict[str, str] | None = None, dotenv_path: str | None = None,
+                  require_foreground: bool = True) -> Settings:
+    """Build Settings from flags, env, .env and TOML. `flags` uses env-style keys (FOREGROUND_MODEL=...)."""
+    src = _sources(flags, config_path, env, dotenv_path)
+
+    def get(*keys: str, default: str | None = None) -> str | None:
+        for k in keys:
+            v = src.get(k)
+            if v is not None and str(v).strip() != "":
+                return str(v).strip()
+        return default
+
+    def speech_url(role: str) -> str | None:
+        url = get(f"{role}_BASE_URL")
+        return None if url is None or url.lower() == "local" else url.rstrip("/")
+
+    fg_url = (get("FOREGROUND_BASE_URL", "OPENAI_BASE_URL") or "").rstrip("/")
+    fg_key = get("FOREGROUND_API_KEY", "OPENAI_API_KEY", default=NO_KEY)
+
+    def key_for(role: str, url: str | None) -> str:
+        # a key only goes to the host it was set for: reuse the foreground key on the same host, never elsewhere
+        own = get(f"{role}_API_KEY")
+        if own:
+            return own
+        return fg_key if url and _same_host(url, fg_url) else NO_KEY
+
+    foreground = Endpoint("foreground", fg_url, fg_key, get("FOREGROUND_MODEL", default=""))
+    bg_url = (get("BACKGROUND_BASE_URL") or fg_url).rstrip("/")
+    background = Endpoint("background", bg_url, key_for("BACKGROUND", bg_url),
+                          get("BACKGROUND_MODEL", default="") or (foreground.model if bg_url == fg_url else ""))
+    asr_url, tts_url = speech_url("ASR"), speech_url("TTS")
+    asr = Endpoint("asr", asr_url, key_for("ASR", asr_url), get("ASR_MODEL", default=""))
+    tts = Endpoint("tts", tts_url, key_for("TTS", tts_url), get("TTS_MODEL", default=""))
+
+    s = Settings(foreground=foreground, background=background, asr=asr, tts=tts)
+    s.foreground_thinking = get("FOREGROUND_THINKING", default=s.foreground_thinking)
+    # same server as the foreground: same way of switching thinking, unless told otherwise
+    s.background_thinking = get("BACKGROUND_THINKING", default=s.foreground_thinking if bg_url == fg_url
+                                else s.background_thinking)
+    s.background_temperature = float(get("BACKGROUND_TEMPERATURE", default=str(s.background_temperature)))
+    s.background_rounds = int(get("BACKGROUND_ROUNDS", default=str(s.background_rounds)))
+    s.asr_language = get("ASR_LANGUAGE")
+    s.asr_partial_ms = int(get("ASR_PARTIAL_MS", default=str(s.asr_partial_ms)))
+    s.tts_voice = get("TTS_VOICE")
+    s.tts_voices = [v.strip() for v in (get("TTS_VOICES") or "").split(",") if v.strip()]
+    s.tts_format = get("TTS_FORMAT", default=s.tts_format).lower()
+    s.tts_sample_rate = int(get("TTS_SAMPLE_RATE", default=str(s.tts_sample_rate)))
+    s.tts_speed = float(get("TTS_SPEED", default=str(s.tts_speed)))
+    s.backend = get("BACKEND", default=s.backend).lower()
+    s.device = get("DEVICE", default=s.device).lower()
+    s.parakeet_onnx_variant = get("PARAKEET_ONNX_VARIANT", default=s.parakeet_onnx_variant).lower()
+    s.kokoro_onnx_variant = get("KOKORO_ONNX_VARIANT", default=s.kokoro_onnx_variant).lower()
+    s.foreground_prompt = get("FOREGROUND_PROMPT", default=s.foreground_prompt)
+    s.background_prompt = get("BACKGROUND_PROMPT", default=s.background_prompt)
+    s.end_of_turn_prompt = get("END_OF_TURN_PROMPT", default=s.end_of_turn_prompt)
+    s.tools_files = [p.strip() for p in (get("TOOLS") or "").split(",") if p.strip()]
+    s.save_utterances = get("SAVE_UTTERANCES")
+    s.host = get("HOST", default=s.host)
+    s.port = int(get("PORT", default=str(s.port)))
+    s.ssl_certfile = get("SSL_CERTFILE")
+    s.ssl_keyfile = get("SSL_KEYFILE")
+    validate(s, require_foreground)
+    return s
+
+
+def validate(s: Settings, require_foreground: bool = True) -> None:
+    for name in ("foreground_thinking", "background_thinking"):
+        if getattr(s, name) not in THINKING_MODES:
+            raise ValueError(f"{name.upper()} must be one of {', '.join(THINKING_MODES)}; got {getattr(s, name)!r}")
+    if s.backend not in LOCAL_BACKENDS:
+        raise ValueError(f"BACKEND must be one of {', '.join(LOCAL_BACKENDS)}; got {s.backend!r}")
+    if s.tts_format not in ("wav", "pcm"):
+        raise ValueError(f"TTS_FORMAT must be wav or pcm; got {s.tts_format!r}")
+    if s.device not in ("cpu", "cuda"):
+        raise ValueError(f"DEVICE must be cpu or cuda; got {s.device!r}")
+    if s.parakeet_onnx_variant not in ("fp32", "int8"):
+        raise ValueError(f"PARAKEET_ONNX_VARIANT must be fp32 or int8; got {s.parakeet_onnx_variant!r}")
+    if s.kokoro_onnx_variant not in ("fp32", "fp16", "int8"):
+        raise ValueError(f"KOKORO_ONNX_VARIANT must be fp32, fp16 or int8; got {s.kokoro_onnx_variant!r}")
+    if require_foreground and not s.foreground.base_url:
+        raise ValueError("No foreground model server. Set FOREGROUND_BASE_URL (or OPENAI_BASE_URL) to any "
+                         "OpenAI-compatible endpoint, e.g. http://127.0.0.1:8080/v1")

interaction_harness/doctor.py

@@ -0,0 +1,171 @@
+"""Check every model role on its own, so a bad setting shows up before you start talking.
+
+    foreground  streams, calls a tool when asked, and answers the end-of-turn prompt in one word
+    background  streams with thinking on and calls a tool
+    TTS -> ASR  speech made by the TTS comes back from the ASR as the same words
+    VAD         hears that speech and not silence
+"""
+from __future__ import annotations
+
+import re
+import time
+
+import numpy as np
+
+from .config import Settings
+from .llm import LLM, resolve_models
+from .speech import MIC_SR, VAD_FRAME, build_speech
+from .speech.base import resample
+from .speech.silero import SileroVAD
+from .tools import schemas
+
+PHRASE = "The quick brown fox jumps over the lazy dog, and the interaction model decides who speaks next."
+
+OK, WARN, FAIL = "ok", "warn", "FAIL"
+_MARK = {OK: "✓", WARN: "!", FAIL: "✗"}
+
+
+class Report:
+    def __init__(self):
+        self.rows: list[tuple[str, str, str, str]] = []
+
+    def add(self, check: str, status: str, detail: str, ms: float | None = None):
+        self.rows.append((check, status, detail, "" if ms is None else f"{ms:,.0f} ms"))
+        print(f"  {_MARK[status]} {check:<26} {self.rows[-1][3]:>10}  {detail}", flush=True)
+
+    @property
+    def failed(self) -> int:
+        return sum(1 for r in self.rows if r[1] == FAIL)
+
+
+def _words(text: str) -> list[str]:
+    return re.findall(r"[a-z]+", text.lower())
+
+
+def word_overlap(said: str, heard: str) -> float:
+    a, b = _words(said), set(_words(heard))
+    return sum(w in b for w in a) / max(1, len(a))
+
+
+async def _chat_check(llm: LLM, report: Report, role: str, thinking: bool):
+    # the same system prompt and tools a real session sends, so the check measures what you will get
+    background = role == "background"
+    prompt = llm.s.background_prompt if background else llm.s.foreground_prompt
+    msgs = [{"role": "system", "content": prompt}, {"role": "user", "content": "What time is it right now?"}]
+    first = done = None
+    calls, reasoning, text = [], 0, []
+    t0 = time.perf_counter()
+    try:
+        async for ev in llm.stream_chat(msgs, tools=schemas(role), thinking=thinking, max_tokens=2000,
+                                        background=background):
+            if ev["type"] == "first_token":
+                first = ev["ms"]
+            elif ev["type"] == "reasoning":
+                reasoning += 1
+            elif ev["type"] == "content":
+                text.append(ev["delta"])
+            elif ev["type"] == "tool_calls":
+                calls = ev["calls"]
+            elif ev["type"] == "done":
+                done = ev
+    except Exception as e:  # noqa: BLE001
+        report.add(f"{role}: chat", FAIL, f"{type(e).__name__}: {str(e)[:160]}", (time.perf_counter() - t0) * 1000)
+        return
+    report.add(f"{role}: first token", OK if first is not None else WARN,
+               "streamed" if first is not None else "no tokens streamed", first)
+    if any(c["name"] == "get_time" for c in calls):
+        report.add(f"{role}: tool call", OK, "called get_time", done["ms"] if done else None)
+    else:
+        said = "".join(text).strip()[:80]
+        report.add(f"{role}: tool call", WARN, f"did not call get_time for the time; it said {said!r}. If tools never "
+                                               "fire, check the server passes them to the model", done["ms"] if done else None)
+    if background:
+        report.add(f"{role}: reasoning", OK if reasoning else WARN,
+                   f"{reasoning} reasoning deltas" if reasoning else
+                   "no reasoning streamed; check BACKGROUND_THINKING for this server")
+
+
+async def _judge_check(llm: LLM, report: Report):
+    for text, want in (("what time is it right now", "COMPLETE"), ("so I was wondering whether and", "INCOMPLETE")):
+        try:
+            complete, ms, raw = await llm.end_of_turn(text)
+        except Exception as e:  # noqa: BLE001
+            report.add("foreground: end-of-turn", FAIL, f"{type(e).__name__}: {str(e)[:160]}")
+            return
+        got = "COMPLETE" if complete else "INCOMPLETE"
+        report.add(f"judge: {want.lower()}", OK if got == want else WARN,
+                   f"{text!r} -> {raw or '(empty)'}" + ("" if got == want else f" (expected {want})"), ms)
+
+
+async def speech_round_trip(settings: Settings, report: Report) -> None:
+    try:
+        speech = await build_speech(settings, log=lambda m: None)
+    except Exception as e:  # noqa: BLE001
+        report.add("speech: load", FAIL, f"{type(e).__name__}: {str(e)[:200]}")
+        return
+    asr, tts = speech.asr, speech.tts
+    report.add("asr: model", OK, f"{asr.model_id} ({asr.where})")
+    report.add("tts: model", OK, f"{tts.model_id} ({tts.where}, voice {settings.tts_voice})")
+    try:
+        pcm, sr, ms = await tts.synth(PHRASE, settings.tts_voice, 1.0)
+    except Exception as e:  # noqa: BLE001
+        report.add("tts: synthesize", FAIL, f"{type(e).__name__}: {str(e)[:200]}")
+        return
+    dur = len(pcm) / sr if sr else 0
+    report.add("tts: synthesize", OK if dur > 1 else FAIL, f"{dur:.2f}s of audio at {sr} Hz (RTF {ms / 1000 / max(dur, 1e-6):.2f})", ms)
+    audio16 = resample(pcm, sr, MIC_SR).astype(np.float32) / 32768.0
+    try:
+        text, ms = await asr.transcribe(audio16)
+    except Exception as e:  # noqa: BLE001
+        report.add("asr: transcribe", FAIL, f"{type(e).__name__}: {str(e)[:200]}")
+        return
+    overlap = word_overlap(PHRASE, text)
+    report.add("tts -> asr round trip", OK if overlap >= 0.8 else WARN if overlap >= 0.5 else FAIL,
+               f"{overlap:.0%} of words back: {text[:80]!r}", ms)
+    stream = await asr.stream()
+    step = MIC_SR // 2
+    t0 = time.perf_counter()
+    partial = ""
+    for i in range(0, len(audio16), step):
+        partial, _ = await stream.feed(audio16[i:i + step])
+    await stream.close()
+    report.add("asr: partials", OK if word_overlap(PHRASE, partial) >= 0.5 else WARN,
+               f"{len(range(0, len(audio16), step))} partials, last {partial[-50:]!r}", (time.perf_counter() - t0) * 1000)
+    vad = SileroVAD()
+    frames = (audio16 * 32767).astype(np.int16)
+    probs = [vad.prob(frames[i:i + VAD_FRAME]) for i in range(0, len(frames) - VAD_FRAME, VAD_FRAME)]
+    vad.reset()
+    quiet = max(vad.prob(np.zeros(VAD_FRAME, dtype=np.int16)) for _ in range(10))
+    speechy = float(np.mean(np.array(probs) > 0.5)) if probs else 0.0
+    report.add("vad", OK if speechy > 0.4 and quiet < 0.2 else WARN,
+               f"speech frames {speechy:.0%}, max p on silence {quiet:.2f}")
+
+
+async def run(settings: Settings) -> int:
+    report = Report()
+    print("model roles", flush=True)
+    for name, ep in settings.roles().items():
+        print(f"  {name:<11} {ep.model or '(server default)':<44} {ep.host}", flush=True)
+    print("\nforeground + background", flush=True)
+    try:
+        for note in await resolve_models(settings):
+            report.add("model", OK, note)
+    except Exception as e:  # noqa: BLE001
+        report.add("model list", FAIL, f"{type(e).__name__}: {str(e)[:200]}")
+    if settings.foreground.model:
+        llm = LLM(settings)
+        await _chat_check(llm, report, "foreground", thinking=False)
+        await _judge_check(llm, report)
+        await _chat_check(llm, report, "background", thinking=True)  # thinking on: its own check even on one server
+    print("\nspeech", flush=True)
+    await speech_round_trip(settings, report)
+    print(f"\n{'all checks passed' if not report.failed else f'{report.failed} check(s) failed'}"
+          f"{'' if not any(r[1] == WARN for r in report.rows) else ' (with warnings)'}", flush=True)
+    return 1 if report.failed else 0
+
+
+async def warmup(settings: Settings) -> int:
+    """Download and load the in-process models, then check they work together."""
+    report = Report()
+    await speech_round_trip(settings, report)
+    return 1 if report.failed else 0