hotato 0.1.0__py3-none-any.whl

hotato/__init__.py

@@ -0,0 +1,33 @@
+"""hotato: the open turn-taking eval for voice agents.
+
+Does your agent drop the turn, or hog it? Free (MIT), self-hostable,
+zero-install, and offline. Its primary consumer is an AI agent: grab it as a CLI
+(``uvx hotato ...``) or as a one-tool MCP server mid-task. It scores voice-agent
+turn-taking - barge-in, overlap/talk-over, and backchannel handling - from a
+call recording, returns a machine-readable verdict, and is the only such tool
+that points a surfaced failure at the KIND of fix it needs: a learned
+engagement-control / addressee-detection layer (an open research problem, not a
+config knob) when the failure is a discrimination one no threshold can solve.
+The pointer is vendor-neutral and names no product.
+
+Honesty is the point, not a footnote: there is no fabricated accuracy anywhere.
+The numbers are reproducible timing measurements with an exposed method and an
+explicit ceiling.
+"""
+
+from .core import LIMITS, SUITE_ID, run_single, run_suite
+
+from ._engine.vad import register_neural_backend as _register_neural_backend
+from .neural import build_silero_backend as _build_silero_backend
+
+# Register the OPTIONAL, non-reference neural VAD backend (Silero VAD, MIT). This
+# only stores the factory reference: the model and its extra are imported lazily,
+# and only if backend="neural" is actually requested -- so importing hotato stays
+# zero-dependency and the energy reference path is untouched. With the [neural]
+# extra absent, a neural request raises a clean BackendUnavailable (never a silent
+# fallback to energy that could change a published number).
+_register_neural_backend(_build_silero_backend)
+
+__version__ = "0.1.0"
+
+__all__ = ["run_single", "run_suite", "LIMITS", "SUITE_ID", "__version__"]

hotato/_engine/__init__.py

@@ -0,0 +1,47 @@
+"""voice-agent-barge-in-tests: a dependency-light harness for measuring the three
+objective barge-in signals (did the agent yield, time to yield, talk-over
+seconds) from a voice-agent call recording.
+
+See the repository README for the failure mode this tests and how to wire it
+into CI. This package measures timing from audio and makes no accuracy claim
+about any detector.
+"""
+
+from .audio import Signal, read_wav, write_wav
+from .score import (
+    ScoreConfig,
+    ScoreResult,
+    Verdict,
+    evaluate,
+    score_channels,
+    score_stereo,
+)
+from .vad import (
+    BackendUnavailable,
+    VADParams,
+    clear_neural_backend,
+    energy_vad,
+    neural_vad,
+    register_neural_backend,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Signal",
+    "read_wav",
+    "write_wav",
+    "ScoreConfig",
+    "ScoreResult",
+    "Verdict",
+    "evaluate",
+    "score_channels",
+    "score_stereo",
+    "VADParams",
+    "energy_vad",
+    "neural_vad",
+    "register_neural_backend",
+    "clear_neural_backend",
+    "BackendUnavailable",
+    "__version__",
+]

hotato/_engine/__main__.py

@@ -0,0 +1,131 @@
+"""Command line entry point.
+
+    python -m barge_scoring score  --stereo call.wav --caller-channel 0 --agent-channel 1
+    python -m barge_scoring score  --caller caller.wav --agent agent.wav --onset 2.4
+    python -m barge_scoring batch  --scenarios scenarios --audio audio
+
+The `score` command works on any recording, not just the bundled fixtures:
+point it at a two-channel export of one of your own calls (caller on one
+channel, agent on the other) and it prints the three signals.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from .audio import Signal, read_wav
+from .batch import run as batch_run
+from .batch import to_json as batch_to_json
+from .batch import to_markdown as batch_to_markdown
+from .score import ScoreConfig, score_channels, score_stereo
+
+
+def _cmd_score(args) -> int:
+    cfg = ScoreConfig(
+        yield_hangover_sec=args.yield_hangover,
+        max_search_sec=args.max_search,
+    )
+    onset = None
+    if args.label:
+        with open(args.label, "r", encoding="utf-8") as fh:
+            label = json.load(fh)
+        onset = label.get("caller_onset_sec")
+    if args.onset is not None:
+        onset = args.onset
+
+    if args.stereo:
+        signal = read_wav(args.stereo)
+        if signal.num_channels < 2:
+            print(
+                "error: --stereo file has one channel; use --caller and --agent "
+                "with two mono files instead",
+                file=sys.stderr,
+            )
+            return 2
+        result = score_stereo(
+            signal, args.caller_channel, args.agent_channel, caller_onset_sec=onset, cfg=cfg
+        )
+    elif args.caller and args.agent:
+        c = read_wav(args.caller)
+        a = read_wav(args.agent)
+        if c.sample_rate != a.sample_rate:
+            print(
+                f"error: sample-rate mismatch (caller {c.sample_rate} Hz, "
+                f"agent {a.sample_rate} Hz); resample so both match",
+                file=sys.stderr,
+            )
+            return 2
+        n = min(c.num_samples, a.num_samples)
+        result = score_channels(
+            c.get(0)[:n], a.get(0)[:n], c.sample_rate, caller_onset_sec=onset, cfg=cfg
+        )
+    else:
+        print("error: provide --stereo FILE, or both --caller FILE and --agent FILE", file=sys.stderr)
+        return 2
+
+    print(json.dumps(result.as_dict(), indent=2))
+    return 0
+
+
+def _cmd_batch(args) -> int:
+    cfg = ScoreConfig()
+    rows = batch_run(
+        args.scenarios,
+        args.audio,
+        suffix=args.suffix,
+        caller_channel=args.caller_channel,
+        agent_channel=args.agent_channel,
+        cfg=cfg,
+    )
+    md = batch_to_markdown(rows)
+    if args.md:
+        with open(args.md, "w", encoding="utf-8") as fh:
+            fh.write(md)
+    if args.json:
+        with open(args.json, "w", encoding="utf-8") as fh:
+            json.dump(batch_to_json(rows), fh, indent=2)
+    print(md)
+    failed = sum(1 for r in rows if not r.passed)
+    if args.fail_on_regression and failed:
+        return 1
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="barge_scoring", description=__doc__)
+    sub = p.add_subparsers(dest="command", required=True)
+
+    s = sub.add_parser("score", help="score a single recording")
+    s.add_argument("--stereo", help="two-channel WAV: one channel caller, one agent")
+    s.add_argument("--caller", help="mono WAV of the caller channel")
+    s.add_argument("--agent", help="mono WAV of the agent channel")
+    s.add_argument("--caller-channel", type=int, default=0)
+    s.add_argument("--agent-channel", type=int, default=1)
+    s.add_argument("--onset", type=float, default=None, help="caller onset in seconds (overrides auto-detect and label)")
+    s.add_argument("--label", help="scenario JSON to read caller_onset_sec from")
+    s.add_argument("--yield-hangover", type=float, default=0.20)
+    s.add_argument("--max-search", type=float, default=4.0)
+    s.set_defaults(func=_cmd_score)
+
+    b = sub.add_parser("batch", help="score every scenario and print a results table")
+    b.add_argument("--scenarios", default="scenarios")
+    b.add_argument("--audio", default="audio")
+    b.add_argument("--suffix", default=".example.wav")
+    b.add_argument("--caller-channel", type=int, default=0)
+    b.add_argument("--agent-channel", type=int, default=1)
+    b.add_argument("--md", help="write the Markdown table to this path")
+    b.add_argument("--json", help="write the JSON summary to this path")
+    b.add_argument("--fail-on-regression", action="store_true", help="exit 1 if any scenario fails (for CI)")
+    b.set_defaults(func=_cmd_batch)
+    return p
+
+
+def main(argv=None) -> int:
+    args = build_parser().parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

hotato/_engine/audio.py

@@ -0,0 +1,149 @@
+"""Minimal WAV reading and framing. Standard library only.
+
+No third-party dependencies are required. PCM WAV at 8, 16, or 32 bit,
+mono or multi-channel, is supported. If numpy happens to be importable it
+is used to speed up the per-frame RMS, but it is never required and the
+results are identical without it.
+"""
+
+from __future__ import annotations
+
+import array
+import math
+import sys
+import wave
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+try:  # optional acceleration only
+    import numpy as _np
+except Exception:  # pragma: no cover - numpy is genuinely optional
+    _np = None
+
+
+@dataclass
+class Signal:
+    """A decoded multi-channel signal with samples in the range [-1, 1]."""
+
+    sample_rate: int
+    channels: List[List[float]]
+
+    @property
+    def num_channels(self) -> int:
+        return len(self.channels)
+
+    @property
+    def num_samples(self) -> int:
+        return len(self.channels[0]) if self.channels else 0
+
+    @property
+    def duration_sec(self) -> float:
+        return self.num_samples / self.sample_rate if self.sample_rate else 0.0
+
+    def get(self, index: int) -> List[float]:
+        if index < 0 or index >= self.num_channels:
+            raise IndexError(
+                f"channel {index} out of range for {self.num_channels}-channel audio"
+            )
+        return self.channels[index]
+
+
+def read_wav(path: str) -> Signal:
+    """Read a PCM WAV file into a Signal with float samples in [-1, 1]."""
+    with wave.open(path, "rb") as wf:
+        n_channels = wf.getnchannels()
+        sampwidth = wf.getsampwidth()
+        sample_rate = wf.getframerate()
+        n_frames = wf.getnframes()
+        raw = wf.readframes(n_frames)
+
+    if sampwidth == 1:
+        # 8-bit PCM is unsigned with a midpoint of 128.
+        a = array.array("B")
+        a.frombytes(raw)
+        samples = [(x - 128) / 128.0 for x in a]
+    elif sampwidth == 2:
+        a = array.array("h")
+        a.frombytes(raw)
+        if sys.byteorder == "big":  # WAV is little-endian
+            a.byteswap()
+        samples = [x / 32768.0 for x in a]
+    elif sampwidth == 4:
+        a = array.array("i")
+        a.frombytes(raw)
+        if sys.byteorder == "big":
+            a.byteswap()
+        samples = [x / 2147483648.0 for x in a]
+    else:
+        raise ValueError(
+            f"unsupported sample width {sampwidth * 8}-bit; "
+            "please convert to 16-bit PCM (for example with ffmpeg -acodec pcm_s16le)"
+        )
+
+    channels = [samples[ch::n_channels] for ch in range(n_channels)]
+    return Signal(sample_rate=sample_rate, channels=channels)
+
+
+def write_wav(path: str, sample_rate: int, channels: List[List[float]]) -> None:
+    """Write float channels in [-1, 1] to a 16-bit PCM WAV file."""
+    n_channels = len(channels)
+    n_samples = len(channels[0]) if channels else 0
+    interleaved = array.array("h", bytes(2 * n_channels * n_samples))
+    for ch in range(n_channels):
+        data = channels[ch]
+        for i in range(n_samples):
+            v = data[i]
+            if v > 1.0:
+                v = 1.0
+            elif v < -1.0:
+                v = -1.0
+            interleaved[i * n_channels + ch] = int(round(v * 32767.0))
+    if sys.byteorder == "big":
+        interleaved.byteswap()
+    with wave.open(path, "wb") as wf:
+        wf.setnchannels(n_channels)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        wf.writeframes(interleaved.tobytes())
+
+
+def frame_rms(
+    samples: List[float],
+    sample_rate: int,
+    frame_ms: float = 20.0,
+    hop_ms: float = 10.0,
+) -> Tuple[List[float], float]:
+    """Return per-frame linear RMS and the hop length in seconds."""
+    frame_len = max(1, int(round(sample_rate * frame_ms / 1000.0)))
+    hop = max(1, int(round(sample_rate * hop_ms / 1000.0)))
+    hop_sec = hop / sample_rate
+    n = len(samples)
+    rms: List[float] = []
+    i = 0
+    if _np is not None:
+        arr = _np.asarray(samples, dtype=_np.float64)
+        while i < n:
+            seg = arr[i : i + frame_len]
+            rms.append(float(_np.sqrt(_np.mean(seg * seg))) if seg.size else 0.0)
+            i += hop
+    else:
+        while i < n:
+            seg = samples[i : i + frame_len]
+            if seg:
+                acc = 0.0
+                for x in seg:
+                    acc += x * x
+                rms.append((acc / len(seg)) ** 0.5)
+            else:
+                rms.append(0.0)
+            i += hop
+    return rms, hop_sec
+
+
+def to_dbfs(rms: List[float], floor_db: float = -120.0) -> List[float]:
+    """Convert linear RMS to dBFS with a floor to avoid log(0)."""
+    out = []
+    lin_floor = 10 ** (floor_db / 20.0)
+    for r in rms:
+        out.append(20.0 * math.log10(r if r > lin_floor else lin_floor))
+    return out

hotato/_engine/batch.py

@@ -0,0 +1,160 @@
+"""Run the scorer over a folder of scenarios and emit a results table.
+
+Produces the Markdown table you can paste into a pull request to show a
+change did not regress interruption handling, plus a machine-readable JSON
+summary you can gate CI on.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from dataclasses import dataclass
+from statistics import median
+from typing import List, Optional
+
+from .audio import read_wav
+from .score import ScoreConfig, evaluate, score_stereo
+
+
+@dataclass
+class Row:
+    scenario_id: str
+    category: str
+    expected_yield: bool
+    did_yield: bool
+    time_to_yield_sec: Optional[float]
+    talk_over_sec: float
+    passed: bool
+    reasons: List[str]
+
+
+def load_scenarios(scenarios_dir: str) -> List[dict]:
+    scenarios = []
+    for name in sorted(os.listdir(scenarios_dir)):
+        if not name.endswith(".json"):
+            continue
+        if name == "manifest.json":
+            continue
+        with open(os.path.join(scenarios_dir, name), "r", encoding="utf-8") as fh:
+            scenarios.append(json.load(fh))
+    return scenarios
+
+
+def run(
+    scenarios_dir: str,
+    audio_dir: str,
+    suffix: str = ".example.wav",
+    caller_channel: int = 0,
+    agent_channel: int = 1,
+    cfg: ScoreConfig = None,
+) -> List[Row]:
+    rows: List[Row] = []
+    for sc in load_scenarios(scenarios_dir):
+        wav_path = os.path.join(audio_dir, sc["id"] + suffix)
+        if not os.path.exists(wav_path):
+            rows.append(
+                Row(
+                    scenario_id=sc["id"],
+                    category=sc.get("category", ""),
+                    expected_yield=bool(sc.get("expected", {}).get("yield", True)),
+                    did_yield=False,
+                    time_to_yield_sec=None,
+                    talk_over_sec=0.0,
+                    passed=False,
+                    reasons=[f"missing audio: {wav_path} (run scenarios/generate_fixtures.py)"],
+                )
+            )
+            continue
+        signal = read_wav(wav_path)
+        result = score_stereo(
+            signal,
+            caller_channel,
+            agent_channel,
+            caller_onset_sec=sc.get("caller_onset_sec"),
+            cfg=cfg,
+        )
+        verdict = evaluate(result, sc.get("expected", {}))
+        rows.append(
+            Row(
+                scenario_id=sc["id"],
+                category=sc.get("category", ""),
+                expected_yield=bool(sc.get("expected", {}).get("yield", True)),
+                did_yield=result.did_yield,
+                time_to_yield_sec=result.time_to_yield_sec,
+                talk_over_sec=result.talk_over_sec,
+                passed=verdict.passed,
+                reasons=verdict.reasons,
+            )
+        )
+    return rows
+
+
+def summarize(rows: List[Row]) -> dict:
+    yield_cases = [r for r in rows if r.expected_yield]
+    yielded = [r for r in yield_cases if r.did_yield]
+    ttoys = [r.time_to_yield_sec for r in yielded if r.time_to_yield_sec is not None]
+    overs = [r.talk_over_sec for r in yield_cases]
+    return {
+        "scenarios": len(rows),
+        "passed": sum(1 for r in rows if r.passed),
+        "failed": sum(1 for r in rows if not r.passed),
+        "yield_rate": (len(yielded) / len(yield_cases)) if yield_cases else None,
+        "median_time_to_yield_sec": round(median(ttoys), 3) if ttoys else None,
+        "median_talk_over_sec": round(median(overs), 3) if overs else None,
+        "max_talk_over_sec": round(max(overs), 3) if overs else None,
+    }
+
+
+def _fmt(v) -> str:
+    return "-" if v is None else (f"{v:.2f}" if isinstance(v, float) else str(v))
+
+
+def to_markdown(rows: List[Row], title: str = "Barge-in regression results") -> str:
+    s = summarize(rows)
+    lines = [f"### {title}", ""]
+    yr = f"{s['yield_rate'] * 100:.0f}%" if s["yield_rate"] is not None else "-"
+    lines.append(
+        f"**{s['passed']}/{s['scenarios']} scenarios pass** | "
+        f"yield rate on should-yield cases: {yr} | "
+        f"median time-to-yield: {_fmt(s['median_time_to_yield_sec'])}s | "
+        f"median talk-over: {_fmt(s['median_talk_over_sec'])}s"
+    )
+    lines.append("")
+    lines.append("| scenario | expected | did yield | time-to-yield (s) | talk-over (s) | result |")
+    lines.append("|---|---|---|---|---|---|")
+    for r in rows:
+        exp = "yield" if r.expected_yield else "hold floor"
+        did = "yes" if r.did_yield else "no"
+        res = "pass" if r.passed else "FAIL"
+        lines.append(
+            f"| {r.scenario_id} | {exp} | {did} | {_fmt(r.time_to_yield_sec)} | "
+            f"{_fmt(r.talk_over_sec)} | {res} |"
+        )
+    fails = [r for r in rows if not r.passed]
+    if fails:
+        lines.append("")
+        lines.append("**Failures**")
+        for r in fails:
+            for why in r.reasons:
+                lines.append(f"- `{r.scenario_id}`: {why}")
+    return "\n".join(lines) + "\n"
+
+
+def to_json(rows: List[Row]) -> dict:
+    return {
+        "summary": summarize(rows),
+        "rows": [
+            {
+                "scenario_id": r.scenario_id,
+                "category": r.category,
+                "expected_yield": r.expected_yield,
+                "did_yield": r.did_yield,
+                "time_to_yield_sec": r.time_to_yield_sec,
+                "talk_over_sec": r.talk_over_sec,
+                "passed": r.passed,
+                "reasons": r.reasons,
+            }
+            for r in rows
+        ],
+    }