attenlabs-saa 0.3.0__py3-none-any.whl

attenlabs_saa-0.3.0.dist-info/METADATA

@@ -0,0 +1,260 @@
+Metadata-Version: 2.4
+Name: attenlabs-saa
+Version: 0.3.0
+Summary: Python SDK for AttentionLabs real-time attention detection.
+Project-URL: Homepage, https://attentionlabs.ai
+Author: Attention Labs
+License: MIT
+License-File: LICENSE
+Keywords: attention,audio,realtime,saa,sd-attention,vad,video,websocket
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.24
+Requires-Dist: opencv-python>=4.8
+Requires-Dist: sounddevice>=0.4.6
+Requires-Dist: websocket-client>=1.6
+Description-Content-Type: text/markdown
+
+# attenlabs-saa
+
+Python SDK for [Attention Labs](https://attentionlabs.ai) real-time selective auditory attention.
+
+Every voice pipeline has the same problem: the microphone hears everything, but your ASR should only process speech directed at the device. Wake words solve this with a rigid trigger phrase. SAA solves it without one — classifying every audio frame as **silent**, **human-directed**, or **device-directed** and routing only what matters.
+
+`attenlabs-saa` streams mic and webcam data to the SAA inference server over WebSocket and emits typed events: attention predictions, voice activity, conversation state, and ready-to-forward speech audio. LLM routing is left to you.
+
+## Sign up
+
+Get your API token at [attentionlabs.ai/dashboard](https://attentionlabs.ai/dashboard). 
+
+**You need your API Key for this project to work**
+
+## Install
+
+```bash
+pip install attenlabs-saa
+```
+
+Requires Python 3.10+. `sounddevice` and `opencv-python` are pulled in automatically for mic and camera access.
+
+## Quickstart
+
+```python
+import time
+from saa import AttentionClient
+
+client = AttentionClient(token="your-token")
+
+@client.on_prediction
+def _(event):
+    label = {0: "silent", 1: "human", 2: "device"}.get(event.cls, "?")
+    print(f"{label}  {event.confidence:.0%}  faces={event.num_faces}  src={event.source}")
+
+@client.on_speech_ready
+def _(event):
+    # event.audio_base64 — base64 PCM16 @ 16 kHz mono, ready for OpenAI Realtime / any LLM
+    # event.audio_pcm16  — same audio as np.int16 array
+    print(f"speech ready ({event.duration_sec:.2f}s)")
+
+@client.on_error
+def _(event):
+    print(f"ERROR: {event.title}: {event.message}")
+
+client.start()
+try:
+    while True:
+        time.sleep(0.1)
+except KeyboardInterrupt:
+    client.stop()
+```
+
+A full CLI demo wiring SAA + OpenAI Realtime lives at [**saa-py-demo**](https://github.com/attenlabs/saa-py-demo).
+
+---
+
+## API
+
+### `AttentionClient`
+
+```python
+from saa import AttentionClient, CameraConfig, MicConfig
+
+client = AttentionClient(
+    token="...",                    # Auth token — sent as WS subprotocol
+    url=None,                      # Server URL (default: wss://server.attentionlabs.ai/ws)
+    video=CameraConfig(),          # Webcam config
+    audio=MicConfig(),             # Mic config
+    initial_threshold=0.7,         # Device-class confidence threshold (0..1)
+    enable_audio=True,             # Set False to skip mic capture
+    enable_video=True,             # Set False to skip webcam capture
+)
+```
+
+### Configuration
+
+#### `MicConfig`
+
+| field      | type                    | default | notes                                      |
+| ---------- | ----------------------- | ------- | ------------------------------------------ |
+| `device`   | `int \| str \| None`   | `None`  | Device index, name, or `None` for system default |
+| `channels` | `int`                   | `1`     | Number of input channels                   |
+
+#### `CameraConfig`
+
+| field          | type  | default | notes                         |
+| -------------- | ----- | ------- | ----------------------------- |
+| `device_index` | `int` | `0`     | Webcam device index           |
+| `width`        | `int` | `1920`  | Capture width                 |
+| `height`       | `int` | `1080`  | Capture height                |
+| `jpeg_quality` | `int` | `60`    | JPEG compression quality 0–100 |
+
+### Methods
+
+| method                       | description |
+| ---------------------------- | ----------- |
+| `start()`                    | Opens WebSocket, acquires mic + camera, starts capture threads. Non-blocking. Raises on handshake failure. |
+| `stop()`                     | Tears down capture, joins threads, closes WebSocket. |
+| `mute()`                     | Pauses upstream audio and signals server to stop VAD. |
+| `unmute()`                   | Resumes upstream audio. |
+| `mark_responding(bool)`      | Tell the server an LLM response is in flight. Server stops emitting predictions while `True`. |
+| `set_threshold(value: float)` | Update device-class confidence threshold (0..1). Server acks via `config` event. |
+
+### Events
+
+Register handlers with decorators. All callbacks fire on internal threads — keep them fast or hand work off to your own thread.
+
+```python
+@client.on_prediction
+def handle(event):
+    ...
+```
+
+| decorator             | payload                                                                  | fires when                              |
+| --------------------- | ------------------------------------------------------------------------ | --------------------------------------- |
+| `@on_connected`       | —                                                                        | WebSocket opens                         |
+| `@on_started`         | —                                                                        | Server-side warmup complete             |
+| `@on_warmup_complete` | —                                                                        | First non-zero-confidence prediction    |
+| `@on_prediction`      | `PredictionEvent`                                                        | Each attention prediction               |
+| `@on_vad`             | `VadEvent`                                                               | Voice activity update                   |
+| `@on_state`           | `StateEvent`                                                             | Conversation state transition           |
+| `@on_speech_ready`    | `SpeechReadyEvent`                                                       | Complete speech segment ready to forward |
+| `@on_config`          | `ConfigEvent`                                                            | Server acks a threshold change          |
+| `@on_stats`           | `StatsEvent`                                                             | Every ~10s with connection health       |
+| `@on_error`           | `AttentionErrorEvent`                                                    | Connection, auth, or server error       |
+| `@on_disconnected`    | `DisconnectedEvent`                                                      | WebSocket closes                        |
+
+### Event types
+
+#### `PredictionEvent`
+
+```python
+cls: int            # 0 = silent, 1 = human-directed, 2 = device-directed
+confidence: float   # 0..1
+source: str         # "video" or "audio"
+num_faces: int      # faces detected in frame
+```
+
+#### `VadEvent`
+
+```python
+probability: float  # VAD probability 0..1
+is_speech: bool     # whether speech was detected
+```
+
+#### `StateEvent`
+
+```python
+state: ConversationState  # "listening" | "sending" | "cancelled" | "idle"
+```
+
+#### `SpeechReadyEvent`
+
+```python
+audio_pcm16: np.ndarray   # int16 array @ 16 kHz mono
+audio_base64: str          # same audio as base64 — ready for OpenAI Realtime, etc.
+duration_sec: float        # duration in seconds
+```
+
+#### `ConfigEvent`
+
+```python
+model_class2_threshold: float  # server-confirmed threshold
+```
+
+#### `StatsEvent`
+
+```python
+rtt_ms: float | None  # round-trip latency in ms
+sent_video: int        # total video frames sent
+skipped_video: int     # total video frames skipped
+sent_audio: int        # total audio chunks sent
+uptime_s: float        # connection uptime in seconds
+```
+
+#### `AttentionErrorEvent`
+
+```python
+title: str                  # error category ("Auth Failed", "Connection Stalled", etc.)
+message: str                # human-readable message
+detail: str | None = None   # technical detail
+code: int | None = None     # WebSocket close code, if applicable
+```
+
+#### `DisconnectedEvent`
+
+```python
+code: int        # WebSocket close code
+reason: str      # close reason
+was_clean: bool  # True if code == 1000
+```
+
+---
+
+## LLM integration
+
+LLM routing is intentionally **not** part of the SDK. The `speech_ready` event hands you PCM16 audio — both as a NumPy array and as base64 — forward it wherever you like.
+
+When your LLM starts generating, call `mute()` + `mark_responding(True)` to suppress predictions during playback. When it finishes, `unmute()` + `mark_responding(False)`.
+
+```python
+from saa import AttentionClient
+
+client = AttentionClient(token="...")
+
+@client.on_speech_ready
+def _(event):
+    # Forward to your LLM of choice
+    your_llm.send(event.audio_base64)
+
+def on_llm_speaking():
+    client.mute()
+    client.mark_responding(True)
+
+def on_llm_done():
+    client.unmute()
+    client.mark_responding(False)
+```
+
+See [**saa-py-demo**](https://github.com/attenlabs/saa-py-demo) for a full working example with OpenAI Realtime.
+
+## Threading model
+
+The SDK manages four threads internally:
+
+| thread           | purpose                            |
+| ---------------- | ---------------------------------- |
+| `saa-ws`         | WebSocket send/receive             |
+| `saa-heartbeat`  | JSON pings every 5s, stats every 10s |
+| `saa-camera`     | JPEG capture at 4 fps (250 ms)     |
+| *(sounddevice)*  | Audio callback at native sample rate, resampled to 16 kHz |
+
+All event callbacks fire on `saa-ws` or `saa-heartbeat`. Don't block them — offload heavy work to your own thread.
+
+## License
+
+MIT

attenlabs_saa-0.3.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+saa/__init__.py,sha256=6sVNTkxUOIOyCzpKyOOk91w8v34o4QutyHFGYiOqkFc,703
+saa/capture.py,sha256=VerHOBnh3GcmcAvJKOy_ZA1_7N67eOznm0AGuIt9zOk,7852
+saa/client.py,sha256=ta-TNIgF-_if1boMgYSjbLXpv_NlotG_apj1ZNxpZ70,18788
+saa/events.py,sha256=rxcQ2AE_YAU9mJeUjWCVaoQFmlgQoPB9K5JmroTK_vA,1541
+saa/ws_protocol.py,sha256=69VVf8gaax6UGpQEaIX8JRMb7sKzSYFuf-zcPuh_cXM,159
+attenlabs_saa-0.3.0.dist-info/METADATA,sha256=LW_OY3eHW2G-4GVtcFC9_f27K869mK9m7DHZaz-vHwQ,9914
+attenlabs_saa-0.3.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+attenlabs_saa-0.3.0.dist-info/licenses/LICENSE,sha256=7qfx8NuzUIobqroBD32MmK1qmZE9B88hscm9N7O_XcQ,1071
+attenlabs_saa-0.3.0.dist-info/RECORD,,

attenlabs_saa-0.3.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

attenlabs_saa-0.3.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Attention Labs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

saa/__init__.py

@@ -0,0 +1,36 @@
+"""saa-py — Python SDK for the SD Attention Server (SAA)."""
+
+from __future__ import annotations
+
+from .capture import CameraConfig, MicConfig
+from .client import AttentionClient
+from .events import (
+    AttentionErrorEvent,
+    ConfigEvent,
+    ConversationState,
+    DisconnectedEvent,
+    PredictionEvent,
+    StateEvent,
+    StatsEvent,
+    TurnFrame,
+    TurnReadyEvent,
+    VadEvent,
+)
+
+__all__ = [
+    "AttentionClient",
+    "CameraConfig",
+    "MicConfig",
+    "PredictionEvent",
+    "VadEvent",
+    "StateEvent",
+    "TurnFrame",
+    "TurnReadyEvent",
+    "ConfigEvent",
+    "StatsEvent",
+    "AttentionErrorEvent",
+    "DisconnectedEvent",
+    "ConversationState",
+]
+
+__version__ = "0.3.0"

saa/capture.py

@@ -0,0 +1,218 @@
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass
+from typing import Any, Callable, Optional, Union
+
+import cv2
+import numpy as np
+
+TARGET_AUDIO_RATE = 16000
+SEND_INTERVAL_SAMPLES = 1600  # 100 ms @ 16 kHz
+VIDEO_INTERVAL_S = 0.25
+
+MicDevice = Union[int, str, None]
+
+
+@dataclass
+class MicConfig:
+    device: MicDevice = None
+    channels: int = 1
+
+
+@dataclass
+class CameraConfig:
+    device_index: int = 0
+    width: int = 1920
+    height: int = 1080
+    jpeg_quality: int = 50
+
+
+class MicCapture:
+    """Sounddevice input stream → int16 PCM @ 16 kHz, chunked to 100 ms blocks."""
+
+    def __init__(self, config: MicConfig, on_pcm16: Callable[[bytes], None]):
+        self.config = config
+        self.on_pcm16 = on_pcm16
+        self._stream: Optional[Any] = None
+        self._muted = False
+        self._buffer = np.zeros(0, dtype=np.float32)
+        self._src_rate = TARGET_AUDIO_RATE
+        self._lock = threading.Lock()
+
+    def start(self) -> None:
+        # Imported lazily so `import saa` doesn't fail on systems without
+        # PortAudio installed — users who only need video capture shouldn't
+        # have to install the audio stack.
+        import sounddevice as sd
+
+        device_info = sd.query_devices(
+            self.config.device if self.config.device is not None else None,
+            kind="input",
+        )
+        native_rate = int(device_info["default_samplerate"])
+
+        # Prefer asking PortAudio for 16 kHz directly — its C-side resampler
+        # is higher quality than our linear interpolator. Fall back to the
+        # device's native rate + manual downsample if the host API refuses.
+        try:
+            self._stream = sd.InputStream(
+                samplerate=TARGET_AUDIO_RATE,
+                channels=self.config.channels,
+                dtype="float32",
+                device=self.config.device,
+                callback=self._audio_callback,
+            )
+            self._src_rate = TARGET_AUDIO_RATE
+        except Exception:
+            self._stream = sd.InputStream(
+                samplerate=native_rate,
+                channels=self.config.channels,
+                dtype="float32",
+                device=self.config.device,
+                callback=self._audio_callback,
+            )
+            self._src_rate = native_rate
+        self._stream.start()
+
+    def _audio_callback(self, indata, frames, time_info, status):
+        # Keep capturing even when muted. The server-side mic_muted flag
+        # (set via the "mute" control action) is what gates the LLM speech
+        # accumulator; the SD-SAA inference ring buffer must keep receiving
+        # live audio every tick or AudioMetricsCalculator computes
+        # amplitude=0 during AI response, which is the key divergence vs
+        # on-device-debug where the mic is always live.
+        samples = indata[:, 0] if indata.ndim > 1 else indata.ravel()
+        if self._src_rate != TARGET_AUDIO_RATE:
+            samples = _linear_downsample(samples, self._src_rate, TARGET_AUDIO_RATE)
+
+        with self._lock:
+            self._buffer = np.concatenate([self._buffer, samples])
+            chunks: list[np.ndarray] = []
+            while len(self._buffer) >= SEND_INTERVAL_SAMPLES:
+                chunks.append(self._buffer[:SEND_INTERVAL_SAMPLES])
+                self._buffer = self._buffer[SEND_INTERVAL_SAMPLES:]
+
+        for chunk in chunks:
+            pcm16 = np.clip(chunk * 32768.0, -32768, 32767).astype(np.int16)
+            try:
+                self.on_pcm16(pcm16.tobytes())
+            except Exception:
+                pass
+
+    def mute(self) -> None:
+        self._muted = True
+
+    def unmute(self) -> None:
+        with self._lock:
+            self._buffer = np.zeros(0, dtype=np.float32)
+        self._muted = False
+
+    def stop(self) -> None:
+        if self._stream is not None:
+            try:
+                self._stream.stop()
+                self._stream.close()
+            except Exception:
+                pass
+            self._stream = None
+
+
+class CameraCapture:
+    """OpenCV webcam → JPEG bytes at a fixed 250 ms interval.
+
+    A reader thread continuously drains the camera so the encoder always
+    snapshots the freshest available frame; without this, AVFoundation /
+    V4L2 / MSMF can hand back frames buffered up to ~150 ms ago.
+    """
+
+    def __init__(self, config: CameraConfig, on_jpeg: Callable[[bytes], None]):
+        self.config = config
+        self.on_jpeg = on_jpeg
+        self._cap: Optional[cv2.VideoCapture] = None
+        self._reader_thread: Optional[threading.Thread] = None
+        self._sender_thread: Optional[threading.Thread] = None
+        self._stop = threading.Event()
+        self._latest_frame: Optional[np.ndarray] = None
+        self._frame_lock = threading.Lock()
+
+    def start(self) -> None:
+        self._cap = cv2.VideoCapture(self.config.device_index)
+        if self._cap.isOpened():
+            self._cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.config.width)
+            self._cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.config.height)
+            self._cap.set(cv2.CAP_PROP_FPS, 30)
+            # Best-effort on backends that honour it (V4L2, MSMF). AVFoundation
+            # ignores it, but the reader thread keeps the queue drained anyway.
+            self._cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+        self._stop.clear()
+        self._latest_frame = None
+        self._reader_thread = threading.Thread(
+            target=self._reader, daemon=True, name="saa-camera-read")
+        self._sender_thread = threading.Thread(
+            target=self._sender, daemon=True, name="saa-camera-send")
+        self._reader_thread.start()
+        self._sender_thread.start()
+
+    def _reader(self) -> None:
+        while not self._stop.is_set():
+            if self._cap is None or not self._cap.isOpened():
+                time.sleep(0.01)
+                continue
+            ok, frame = self._cap.read()
+            if not ok or frame is None:
+                time.sleep(0.005)
+                continue
+            with self._frame_lock:
+                self._latest_frame = frame
+
+    def _sender(self) -> None:
+        jpeg_params = [int(cv2.IMWRITE_JPEG_QUALITY), int(self.config.jpeg_quality)]
+        next_deadline = time.monotonic()
+        while not self._stop.is_set():
+            now = time.monotonic()
+            if now < next_deadline:
+                time.sleep(min(0.05, next_deadline - now))
+                continue
+            next_deadline = now + VIDEO_INTERVAL_S
+
+            with self._frame_lock:
+                frame = self._latest_frame
+            if frame is None:
+                continue
+            ok, buf = cv2.imencode(".jpg", frame, jpeg_params)
+            if not ok:
+                continue
+            try:
+                self.on_jpeg(buf.tobytes())
+            except Exception:
+                pass
+
+    def stop(self) -> None:
+        self._stop.set()
+        for t in (self._reader_thread, self._sender_thread):
+            if t is not None:
+                t.join(timeout=2.0)
+        self._reader_thread = None
+        self._sender_thread = None
+        if self._cap is not None:
+            try:
+                self._cap.release()
+            except Exception:
+                pass
+            self._cap = None
+
+
+def _linear_downsample(samples: np.ndarray, src_rate: int, dst_rate: int) -> np.ndarray:
+    if src_rate == dst_rate or len(samples) == 0:
+        return samples
+    ratio = src_rate / dst_rate
+    out_len = int(len(samples) / ratio)
+    if out_len == 0:
+        return np.zeros(0, dtype=samples.dtype)
+    indices = np.arange(out_len) * ratio
+    lo = np.floor(indices).astype(np.int64)
+    hi = np.clip(lo + 1, 0, len(samples) - 1)
+    frac = (indices - lo).astype(samples.dtype)
+    return samples[lo] * (1 - frac) + samples[hi] * frac

saa/client.py

@@ -0,0 +1,495 @@
+from __future__ import annotations
+
+import base64
+import json
+import logging
+import threading
+import time
+import urllib.error
+import urllib.request
+from typing import Any, Callable, Optional
+
+import numpy as np
+import websocket
+
+from .capture import CameraCapture, CameraConfig, MicCapture, MicConfig
+from .events import (
+    AttentionErrorEvent,
+    ConfigEvent,
+    DisconnectedEvent,
+    PredictionEvent,
+    StateEvent,
+    StatsEvent,
+    TurnFrame,
+    TurnReadyEvent,
+    VadEvent,
+)
+from .ws_protocol import MSG_AUDIO, MSG_VIDEO, frame_binary
+
+# Default URL is the broker. The SDK auto-detects mode from the URL scheme:
+#   http(s)://… → POST /allocate (Bearer token), then connect to the
+#                 wss URL the broker returns. Recommended default — sticky
+#                 least-loaded routing across multiple backends.
+#   ws(s)://…   → connect WS directly (legacy / debugging). Bypasses the
+#                 broker. Useful for pinning a session to a specific
+#                 backend during tests.
+DEFAULT_SERVER_URL = "https://broker.attentionlabs.ai"
+BROKER_ALLOCATE_TIMEOUT_S = 5.0
+DEFAULT_THRESHOLD = 0.7
+
+WS_PING_INTERVAL_S = 5.0
+WS_PONG_TIMEOUT_S = 15.0
+WS_STATS_INTERVAL_S = 10.0
+WS_HANDSHAKE_TIMEOUT_S = 10.0
+
+logger = logging.getLogger("saa")
+
+Listener = Callable[..., Any]
+
+
+class AttentionClient:
+    """Streams mic + webcam to the SAA inference server and emits typed events.
+
+    Callbacks fire on the WebSocket receive thread (or on the heartbeat thread
+    for `stats`/`error`). Keep listeners fast or hand work off to your own
+    thread.
+    """
+
+    def __init__(
+        self,
+        url: Optional[str] = None,
+        token: Optional[str] = None,
+        *,
+        video: Optional[CameraConfig] = None,
+        audio: Optional[MicConfig] = None,
+        initial_threshold: float = DEFAULT_THRESHOLD,
+        enable_audio: bool = True,
+        enable_video: bool = True,
+    ):
+        self.url = url or DEFAULT_SERVER_URL
+        self.token = token
+        self.video_config = video or CameraConfig()
+        self.audio_config = audio or MicConfig()
+        self.enable_audio = enable_audio
+        self.enable_video = enable_video
+        self.threshold = _clamp01(initial_threshold)
+
+        self._listeners: dict[str, list[Listener]] = {}
+        self._ws: Optional[websocket.WebSocketApp] = None
+        self._ws_thread: Optional[threading.Thread] = None
+        self._ws_open = threading.Event()
+        self._ws_closed = threading.Event()
+        self._close_info: dict = {}
+        self._mic: Optional[MicCapture] = None
+        self._cam: Optional[CameraCapture] = None
+        self._stats_thread: Optional[threading.Thread] = None
+        self._stats_stop = threading.Event()
+        self._started = False
+
+        self._sent_audio = 0
+        self._sent_video = 0
+        self._skipped_video = 0
+        self._last_rtt_ms: Optional[float] = None
+        self._last_pong_at: float = 0.0
+        self._ws_opened_at: float = 0.0
+        self._warmed_up = False
+        self._muted = False
+
+    # ── event registration (decorator style) ───────────────────────
+
+    def on_connected(self, func: Listener) -> Listener: return self._register("connected", func)
+    def on_started(self, func: Listener) -> Listener: return self._register("started", func)
+    def on_warmup_complete(self, func: Listener) -> Listener: return self._register("warmup_complete", func)
+    def on_prediction(self, func: Listener) -> Listener: return self._register("prediction", func)
+    def on_vad(self, func: Listener) -> Listener: return self._register("vad", func)
+    def on_state(self, func: Listener) -> Listener: return self._register("state", func)
+    def on_turn_ready(self, func: Listener) -> Listener: return self._register("turn_ready", func)
+    def on_config(self, func: Listener) -> Listener: return self._register("config", func)
+    def on_stats(self, func: Listener) -> Listener: return self._register("stats", func)
+    def on_error(self, func: Listener) -> Listener: return self._register("error", func)
+    def on_disconnected(self, func: Listener) -> Listener: return self._register("disconnected", func)
+
+    def _register(self, event: str, func: Listener) -> Listener:
+        self._listeners.setdefault(event, []).append(func)
+        return func
+
+    def _emit(self, event: str, *args) -> None:
+        for func in self._listeners.get(event, []):
+            try:
+                func(*args)
+            except Exception:
+                logger.exception("saa listener for '%s' raised", event)
+
+    # ── lifecycle ─────────────────────────────────────────────────
+
+    def start(self) -> None:
+        if self._started:
+            raise RuntimeError("AttentionClient already started")
+        self._started = True
+        try:
+            self._open_ws_blocking()
+            if self.enable_audio:
+                self._mic = MicCapture(self.audio_config, on_pcm16=self._on_mic_pcm16)
+                self._mic.start()
+            if self.enable_video:
+                self._cam = CameraCapture(self.video_config, on_jpeg=self._on_cam_jpeg)
+                self._cam.start()
+            self._stats_stop.clear()
+            self._stats_thread = threading.Thread(
+                target=self._heartbeat_loop, daemon=True, name="saa-heartbeat",
+            )
+            self._stats_thread.start()
+        except Exception:
+            self.stop()
+            raise
+
+    def stop(self) -> None:
+        if not self._started:
+            return
+
+        self._stats_stop.set()
+        if self._stats_thread is not None:
+            self._stats_thread.join(timeout=2.0)
+            self._stats_thread = None
+
+        if self._mic is not None:
+            self._mic.stop()
+            self._mic = None
+        if self._cam is not None:
+            self._cam.stop()
+            self._cam = None
+
+        if self._ws is not None:
+            try:
+                self._ws.close()
+            except Exception:
+                pass
+        if self._ws_thread is not None:
+            self._ws_thread.join(timeout=2.0)
+            self._ws_thread = None
+        self._ws = None
+
+        self._started = False
+        self._warmed_up = False
+        self._muted = False
+
+    # ── control ───────────────────────────────────────────────────
+
+    def mute(self) -> None:
+        self._muted = True
+        if self._mic is not None:
+            self._mic.mute()
+        self._send_control({"action": "mute"})
+
+    def unmute(self) -> None:
+        self._muted = False
+        if self._mic is not None:
+            self._mic.unmute()
+        self._send_control({"action": "unmute"})
+
+    def mark_responding(self, responding: bool) -> None:
+        self._send_control({
+            "action": "responding_start" if responding else "responding_stop",
+        })
+
+    def set_threshold(self, value: float) -> None:
+        value = _clamp01(value)
+        self.threshold = value
+        self._send_control({"action": "set_threshold", "value": value})
+
+    # ── WS ────────────────────────────────────────────────────────
+
+    def _resolve_ws_url(self) -> str:
+        """Resolve self.url to a concrete wss://…/ws URL.
+
+        - ws(s)://… is treated as a direct backend URL — returned as-is.
+        - http(s)://… is treated as a broker base URL — POST /allocate
+          with the bearer token, return the wss URL the broker returns.
+
+        Called once per connect, so reconnects pick a fresh least-loaded
+        backend each time. Uses urllib (stdlib) — no new deps.
+        """
+        url = self.url
+        if url.startswith("ws://") or url.startswith("wss://"):
+            return url
+        allocate_url = url.rstrip("/") + "/allocate"
+        headers = {"Authorization": f"Bearer {self.token}"} if self.token else {}
+        req = urllib.request.Request(
+            allocate_url, method="POST", headers=headers,
+            data=b"",  # POST with empty body — broker doesn't read it
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=BROKER_ALLOCATE_TIMEOUT_S) as resp:
+                payload = json.loads(resp.read().decode("utf-8"))
+        except urllib.error.HTTPError as e:
+            body = ""
+            try:
+                body = e.read().decode("utf-8", errors="replace")
+            except Exception:
+                pass
+            raise ConnectionError(
+                f"broker /allocate failed: HTTP {e.code} {body or e.reason}"
+            ) from e
+        except urllib.error.URLError as e:
+            raise ConnectionError(f"broker /allocate request failed: {e.reason}") from e
+        ws_url = payload.get("url") if isinstance(payload, dict) else None
+        if not ws_url:
+            raise ConnectionError(f"broker /allocate returned no url: {payload!r}")
+        return ws_url
+
+    def _open_ws_blocking(self) -> None:
+        self._ws_open.clear()
+        self._ws_closed.clear()
+        self._close_info = {}
+        self._sent_audio = 0
+        self._sent_video = 0
+        self._skipped_video = 0
+        self._last_rtt_ms = None
+        self._ws_opened_at = 0.0
+        self._warmed_up = False
+
+        ws_url = self._resolve_ws_url()
+        subprotocols = [self.token] if self.token else None
+        self._ws = websocket.WebSocketApp(
+            ws_url,
+            subprotocols=subprotocols,
+            on_open=self._on_ws_open,
+            on_message=self._on_ws_message,
+            on_close=self._on_ws_close,
+            on_error=self._on_ws_error,
+        )
+
+        def run_ws():
+            try:
+                # ping_interval=0 disables websocket-client's automatic WS-level
+                # pings — we send our own JSON pings so the server sees them.
+                self._ws.run_forever(ping_interval=0)
+            except Exception:
+                logger.exception("ws run_forever raised")
+
+        self._ws_thread = threading.Thread(target=run_ws, daemon=True, name="saa-ws")
+        self._ws_thread.start()
+
+        opened = self._ws_open.wait(timeout=WS_HANDSHAKE_TIMEOUT_S)
+        if not opened:
+            # If closed during handshake, report the close reason.
+            if self._ws_closed.is_set():
+                info = self._close_info
+                raise ConnectionError(
+                    f"WebSocket closed during handshake: "
+                    f"code={info.get('code')} reason={info.get('reason') or 'none'}"
+                )
+            try:
+                self._ws.close()
+            except Exception:
+                pass
+            raise TimeoutError(
+                f"WebSocket handshake timed out after {WS_HANDSHAKE_TIMEOUT_S}s (url={self.url})"
+            )
+
+    def _on_ws_open(self, ws) -> None:
+        self._ws_opened_at = time.monotonic()
+        self._last_pong_at = self._ws_opened_at
+        self._ws_open.set()
+        self._emit("connected")
+
+    def _on_ws_message(self, ws, message) -> None:
+        if not isinstance(message, str):
+            return
+        try:
+            msg = json.loads(message)
+        except json.JSONDecodeError:
+            return
+        self._handle_msg(msg)
+
+    def _on_ws_close(self, ws, code, reason) -> None:
+        code = code or 0
+        reason = reason or ""
+        was_clean = code == 1000
+        self._close_info = {"code": code, "reason": reason, "was_clean": was_clean}
+        self._ws_closed.set()
+        # Unblock handshake waiter if we closed before opening.
+        self._ws_open.set()
+        self._emit("disconnected", DisconnectedEvent(
+            code=code, reason=reason, was_clean=was_clean,
+        ))
+        err = _close_to_error(code, reason, was_clean)
+        if err is not None:
+            self._emit("error", err)
+
+    def _on_ws_error(self, ws, error) -> None:
+        logger.debug("ws error: %s", error)
+
+    def _handle_msg(self, msg: dict) -> None:
+        t = msg.get("type")
+
+        if t == "pong":
+            self._last_pong_at = time.monotonic()
+            client_ts = msg.get("client_ts")
+            if isinstance(client_ts, (int, float)):
+                self._last_rtt_ms = (time.monotonic() * 1000.0) - float(client_ts)
+            return
+
+        if t == "prediction":
+            # Prefer the server's display_class (e.g. low-conf class-2 relabelled
+            # to class-1). Falls back to raw `class` for older servers.
+            cls = msg.get("display_class")
+            if cls is None:
+                cls = msg.get("class")
+            if cls is None:
+                cls = 0
+            conf = msg.get("confidence") or 0.0
+            if not self._warmed_up and conf > 0:
+                self._warmed_up = True
+                self._emit("warmup_complete")
+            self._emit("prediction", PredictionEvent(
+                cls=int(cls),
+                confidence=float(conf),
+                source=msg.get("source") or "",
+                num_faces=int(msg.get("num_faces") or 0),
+            ))
+        elif t == "vad":
+            self._emit("vad", VadEvent(
+                probability=float(msg.get("probability") or 0.0),
+                is_speech=bool(msg.get("is_speech")),
+            ))
+        elif t == "state":
+            self._emit("state", StateEvent(state=msg.get("state") or "idle"))
+        elif t == "turn_ready":
+            b64 = msg.get("audio_base64") or ""
+            raw_frames = msg.get("frames") or []
+            frames = [
+                TurnFrame(
+                    ts_offset_s=float(f.get("ts_offset_s") or 0.0),
+                    image_base64=str(f.get("image_base64") or ""),
+                )
+                for f in raw_frames
+                if isinstance(f, dict) and f.get("image_base64")
+            ]
+            self._emit("turn_ready", TurnReadyEvent(
+                audio_pcm16=_b64_to_int16(b64),
+                audio_base64=b64,
+                duration_sec=float(msg.get("duration") or 0.0),
+                frames=frames,
+            ))
+        elif t == "started":
+            self._emit("started")
+            self._send_control({"action": "set_threshold", "value": self.threshold})
+        elif t == "config":
+            thr = msg.get("model_class2_threshold")
+            if isinstance(thr, (int, float)):
+                self.threshold = float(thr)
+                self._emit("config", ConfigEvent(model_class2_threshold=float(thr)))
+        elif t == "error":
+            self._emit("error", AttentionErrorEvent(
+                title="Server Error",
+                message=msg.get("message") or "",
+                detail=msg.get("detail"),
+            ))
+
+    def _send_control(self, data: dict) -> None:
+        if self._ws is None or not self._ws_open.is_set():
+            return
+        try:
+            self._ws.send(json.dumps(data))
+        except Exception:
+            pass
+
+    def _on_mic_pcm16(self, pcm16_bytes: bytes) -> None:
+        # Don't gate on self._muted — keep streaming PCM during mute. The
+        # "mute" control action already informs the server, which only blocks
+        # the LLM chunk accumulator (so TTS isn't fed back to OpenAI as user
+        # speech). The inference ring buffer needs live audio every tick.
+        if self._ws is None or not self._ws_open.is_set():
+            return
+        try:
+            self._ws.send(
+                frame_binary(MSG_AUDIO, pcm16_bytes),
+                opcode=websocket.ABNF.OPCODE_BINARY,
+            )
+            self._sent_audio += 1
+        except Exception:
+            pass
+
+    def _on_cam_jpeg(self, jpeg_bytes: bytes) -> None:
+        if self._ws is None or not self._ws_open.is_set():
+            self._skipped_video += 1
+            return
+        try:
+            self._ws.send(
+                frame_binary(MSG_VIDEO, jpeg_bytes),
+                opcode=websocket.ABNF.OPCODE_BINARY,
+            )
+            self._sent_video += 1
+        except Exception:
+            self._skipped_video += 1
+
+    def _heartbeat_loop(self) -> None:
+        last_stats_at = time.monotonic()
+        while not self._stats_stop.wait(WS_PING_INTERVAL_S):
+            if self._ws is None or not self._ws_open.is_set():
+                continue
+            now = time.monotonic()
+            if now - self._last_pong_at > WS_PONG_TIMEOUT_S:
+                self._emit("error", AttentionErrorEvent(
+                    title="Connection Stalled",
+                    message="No pong received within timeout window.",
+                    detail=f"{now - self._last_pong_at:.1f}s since last pong",
+                ))
+            self._send_control({"action": "ping", "ts": time.monotonic() * 1000.0})
+            if now - last_stats_at >= WS_STATS_INTERVAL_S:
+                self._emit("stats", StatsEvent(
+                    rtt_ms=self._last_rtt_ms,
+                    sent_video=self._sent_video,
+                    skipped_video=self._skipped_video,
+                    sent_audio=self._sent_audio,
+                    uptime_s=now - self._ws_opened_at if self._ws_opened_at else 0.0,
+                ))
+                last_stats_at = now
+
+
+def _clamp01(v: float) -> float:
+    if v != v:  # NaN
+        return 0.0
+    return max(0.0, min(1.0, v))
+
+
+def _b64_to_int16(b64: str) -> np.ndarray:
+    if not b64:
+        return np.zeros(0, dtype=np.int16)
+    raw = base64.b64decode(b64)
+    return np.frombuffer(raw, dtype=np.int16).copy()
+
+
+def _close_to_error(code: int, reason: str, was_clean: bool) -> Optional[AttentionErrorEvent]:
+    if code == 1000:
+        return None
+    if code == 1008:
+        return AttentionErrorEvent(
+            title="Auth Failed",
+            message="Server rejected the auth token.",
+            detail=reason or f"close code {code}",
+            code=code,
+        )
+    if code == 1013:
+        return AttentionErrorEvent(
+            title="Rate Limited",
+            message="Throttled by server — try again shortly.",
+            detail=reason or f"close code {code}",
+            code=code,
+        )
+    if code == 1006:
+        return AttentionErrorEvent(
+            title="Connection Failed",
+            message="Could not reach the server.",
+            detail=f"The server may be down or unreachable. (close code {code})",
+            code=code,
+        )
+    if not was_clean:
+        return AttentionErrorEvent(
+            title="Disconnected",
+            message="Connection lost unexpectedly.",
+            detail=f"code={code} reason={reason or 'none'}",
+            code=code,
+        )
+    return None

saa/events.py

@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Literal, Optional
+
+import numpy as np
+
+ConversationState = Literal["listening", "sending", "cancelled", "idle"]
+
+
+@dataclass
+class PredictionEvent:
+    cls: int
+    confidence: float
+    source: str
+    num_faces: int
+
+
+@dataclass
+class VadEvent:
+    probability: float
+    is_speech: bool
+
+
+@dataclass
+class StateEvent:
+    state: ConversationState
+
+
+@dataclass
+class TurnFrame:
+    """One still captured from the conversation turn.
+
+    LLM-agnostic: SDK delivers raw base64-encoded JPEG. Callers wrap it for
+    whatever LLM they target (OpenAI input_image, Anthropic image, Gemini
+    inlineData, …).
+    """
+    ts_offset_s: float  # seconds from listening-start; negative = pre-context
+    image_base64: str   # JPEG bytes, base64-encoded (no data: prefix)
+
+
+@dataclass
+class TurnReadyEvent:
+    audio_pcm16: np.ndarray  # int16, 16 kHz mono
+    audio_base64: str
+    duration_sec: float
+    frames: list[TurnFrame] = field(default_factory=list)
+    """Empty unless the server has frames_per_turn > 0."""
+
+
+@dataclass
+class ConfigEvent:
+    model_class2_threshold: float
+
+
+@dataclass
+class StatsEvent:
+    rtt_ms: Optional[float]
+    sent_video: int
+    skipped_video: int
+    sent_audio: int
+    uptime_s: float
+
+
+@dataclass
+class AttentionErrorEvent:
+    title: str
+    message: str
+    detail: Optional[str] = None
+    code: Optional[int] = None
+
+
+@dataclass
+class DisconnectedEvent:
+    code: int
+    reason: str
+    was_clean: bool

saa/ws_protocol.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+MSG_AUDIO = 0x01
+MSG_VIDEO = 0x02
+
+
+def frame_binary(tag: int, payload: bytes) -> bytes:
+    return bytes([tag]) + payload