agentwire-dev 1.29.3__tar.gz → 1.29.5__tar.gz

{agentwire_dev-1.29.3 → agentwire_dev-1.29.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: agentwire-dev
-Version: 1.29.3
+Version: 1.29.5
 Summary: Multi-session voice web interface for AI coding agents
 Project-URL: Homepage, https://agentwire.dev
 Project-URL: Repository, https://github.com/dotdevdotdev/agentwire-dev
@@ -702,6 +702,8 @@ Provides-Extra: stt
 Requires-Dist: fastapi>=0.104.0; extra == 'stt'
 Requires-Dist: faster-whisper>=1.0.0; extra == 'stt'
 Requires-Dist: python-multipart>=0.0.6; extra == 'stt'
+Requires-Dist: soundfile>=0.12.0; extra == 'stt'
+Requires-Dist: useful-moonshine-onnx>=20250101; extra == 'stt'
 Requires-Dist: uvicorn>=0.24.0; extra == 'stt'
 Provides-Extra: tts
 Requires-Dist: chatterbox-tts>=0.1.6; extra == 'tts'

{agentwire_dev-1.29.3 → agentwire_dev-1.29.5}/agentwire/__init__.py

@@ -1,3 +1,3 @@
 """AgentWire - Multi-session voice web interface for AI coding agents."""
 
-__version__ = "1.29.3"
+__version__ = "1.29.5"

{agentwire_dev-1.29.3 → agentwire_dev-1.29.5}/agentwire/__main__.py

@@ -6182,6 +6182,23 @@ def cmd_doctor(args) -> int:
             else:
                 print(f"     -> Service is remote, start it on {service_config.machine}")
 
+    # 8b. Check STT server. STT isn't a `services.*` entry — it's configured via
+    # `stt.url`, so the loop above can't see it. Without this check, a crashed STT
+    # server is invisible and the portal silently falls back to a model-less
+    # WhisperKit (every push-to-talk then fails). Probe it explicitly.
+    stt_url = getattr(getattr(ctx.config, "stt", None), "url", None)
+    if stt_url:
+        from agentwire.stt.server_backend import STTServerBackend
+
+        if STTServerBackend.is_available(stt_url):
+            print(f"  [ok] Stt: responding on {stt_url}")
+        else:
+            print(f"  [!!] Stt: not responding on {stt_url}")
+            print("       Portal will fall back to WhisperKit (usually no model -> transcription FAILS).")
+            print("       Fix: agentwire stt start")
+            print("       If it won't boot: uv pip install --python .venv/bin/python -e '.[stt]'")
+            issues_found += 1
+
     # 9. Validate remote machines
     print("\nChecking remote machines...")
     remote_machines = {mid: m for mid, m in ctx.machines.items() if mid != ctx.local_machine_id}

{agentwire_dev-1.29.3 → agentwire_dev-1.29.5}/agentwire/static/js/session-window.js

@@ -166,6 +166,12 @@ export class SessionWindow {
             this.ws = null;
         }
 
+        // Remove the two-finger touch-scroll listeners
+        if (this._touchScrollCleanup) {
+            this._touchScrollCleanup();
+            this._touchScrollCleanup = null;
+        }
+
         // Dispose terminal (terminal mode) or output element (monitor mode)
         if (this.terminal) {
             this.terminal.dispose();
@@ -321,6 +327,13 @@ export class SessionWindow {
 
         this.terminal.open(terminalEl);
 
+        // Touch devices emit no `wheel` events, so xterm's wheel-driven
+        // scrolling (tmux copy-mode / the app's own scroll) is unreachable on a
+        // tablet. Translate a two-finger vertical pan into synthetic wheel
+        // events so touch scrolls history exactly like a desktop mouse wheel.
+        // One finger stays free for tap/selection.
+        this._setupTouchScroll(terminalEl);
+
         // Fit after font loads and layout is complete
         const fontFamily = '"FiraMono Nerd Font Mono", Menlo, Monaco, "Courier New", monospace';
         const fontSize = pickTerminalFontSize();
@@ -367,6 +380,154 @@ export class SessionWindow {
         }
     }
 
+    /**
+     * Translate a two-finger vertical pan into tmux mouse-wheel scroll.
+     *
+     * Touch devices emit no `wheel` events, so the desktop scroll path (mouse
+     * wheel → xterm encodes an SGR mouse event → tmux enters copy-mode and
+     * scrolls) never fires on a tablet. Rather than fake a WheelEvent and hope
+     * xterm's mouse encoder picks it up, we send the exact bytes a real wheel
+     * produces — the SGR-1006 mouse sequence — straight down the input
+     * WebSocket. tmux runs with `mouse on`, so it consumes these and scrolls
+     * history. One finger is left untouched for tap/selection.
+     *
+     * SGR wheel: ESC [ < Btn ; Col ; Row M  — Btn 64 = wheel-up, 65 = wheel-down
+     * (1-based Col/Row; any point inside the pane works for scroll).
+     */
+    _setupTouchScroll(terminalEl) {
+        // Finger travel, in text lines, that advances one tmux wheel tick. tmux
+        // scrolls 5 lines per wheel tick (its WheelUp/DownPane `-N 5` binding),
+        // so a strict 1:1 mapping would need 5 lines of finger travel per tick —
+        // on a short window that's nearly the whole draggable height, so you get
+        // one tick then nothing. Firing a tick every ~1.5 lines keeps it ticking
+        // continuously across the whole stroke; momentum then covers distance.
+        const FINGER_LINES_PER_TICK = 1.5;
+        // Cap ticks emitted per animation frame so a fast flick can't flood tmux
+        // faster than it can redraw (the source of the laggy/stuttery feel).
+        const MAX_TICKS_PER_FRAME = 8;
+        // First tick of a gesture fires after only this fraction of a full tick
+        // of travel, so the start feels immediate instead of dead until ~5 lines.
+        const FIRST_TICK_FRACTION = 0.35;
+        // Momentum: after lift, keep scrolling and decay velocity each frame.
+        const FRICTION = 0.94;          // per-frame velocity multiplier
+        const FLING_MIN_V = 0.04;       // px/ms at release needed to start a fling
+        const MOMENTUM_STOP_V = 0.012;  // px/ms below which momentum ends
+
+        let active = false;
+        let lastMidY = 0;
+        let accum = 0;        // unconsumed finger travel (px), sign = direction
+        let rafId = null;
+        let emitted = false;  // has any tick fired this gesture? (first-tick boost)
+        let velocity = 0;     // px/ms, smoothed — drives momentum
+        let lastMoveT = 0;
+        let lastFrameT = 0;
+        let momentum = false;
+
+        const midY = (touches) => (touches[0].clientY + touches[1].clientY) / 2;
+
+        // Finger pixels per tick = lines-per-tick × measured cell height.
+        const pxPerTick = () => {
+            const rows = this.terminal?.rows || 24;
+            const h = terminalEl.getBoundingClientRect().height;
+            const cell = rows > 0 && h > 0 ? h / rows : 18;
+            return FINGER_LINES_PER_TICK * cell;
+        };
+
+        // dir < 0 → wheel-up (older history); dir > 0 → wheel-down (newer).
+        const wheelSeq = (dir) => {
+            const col = Math.max(1, Math.floor(this.terminal.cols / 2));
+            const row = Math.max(1, Math.floor(this.terminal.rows / 2));
+            return `\x1b[<${dir < 0 ? 64 : 65};${col};${row}M`;
+        };
+
+        const sendTicks = (ticks) => {
+            if (!this.ws || this.ws.readyState !== WebSocket.OPEN || !this.terminal) return;
+            this.ws.send(JSON.stringify({ type: 'input', data: wheelSeq(ticks).repeat(Math.abs(ticks)) }));
+        };
+
+        // One frame: advance momentum, then batch all due ticks into one message.
+        const flush = (now) => {
+            rafId = null;
+
+            if (momentum) {
+                const dt = Math.min(now - lastFrameT, 50);
+                lastFrameT = now;
+                accum += velocity * dt;
+                velocity *= FRICTION;
+                if (Math.abs(velocity) < MOMENTUM_STOP_V) momentum = false;
+            }
+            if (active || momentum) rafId = requestAnimationFrame(flush);
+
+            const step = pxPerTick();
+            // Snappier first tick: lower the threshold until the gesture moves.
+            const threshold = emitted ? step : step * FIRST_TICK_FRACTION;
+            let ticks = Math.trunc(accum / threshold);
+            if (ticks === 0) return;
+            if (ticks > MAX_TICKS_PER_FRAME) ticks = MAX_TICKS_PER_FRAME;
+            else if (ticks < -MAX_TICKS_PER_FRAME) ticks = -MAX_TICKS_PER_FRAME;
+            accum -= ticks * threshold;
+            emitted = true;
+            sendTicks(ticks);
+        };
+
+        const startRaf = () => { if (rafId === null) { lastFrameT = performance.now(); rafId = requestAnimationFrame(flush); } };
+        const stopRaf = () => { if (rafId !== null) { cancelAnimationFrame(rafId); rafId = null; } };
+
+        const onTouchStart = (e) => {
+            if (e.touches.length !== 2) { active = false; momentum = false; stopRaf(); return; }
+            active = true;
+            momentum = false;
+            emitted = false;
+            velocity = 0;
+            accum = 0;
+            lastMidY = midY(e.touches);
+            lastMoveT = performance.now();
+            e.preventDefault();  // stop the page/WinBox from claiming the gesture
+            startRaf();
+        };
+
+        const onTouchMove = (e) => {
+            if (!active || e.touches.length !== 2) return;
+            e.preventDefault();
+            const now = performance.now();
+            const y = midY(e.touches);
+            // Fingers up (y decreases) → newer/down; fingers down → older/up.
+            const dy = lastMidY - y;
+            accum += dy;
+            const dt = now - lastMoveT;
+            if (dt > 0) velocity = 0.6 * velocity + 0.4 * (dy / dt);  // smoothed px/ms
+            lastMidY = y;
+            lastMoveT = now;
+        };
+
+        const onTouchEnd = (e) => {
+            if (e.touches.length >= 2) return;
+            active = false;
+            // Carry a fast lift into a decaying fling; otherwise stop clean.
+            if (Math.abs(velocity) >= FLING_MIN_V) {
+                momentum = true;
+                lastFrameT = performance.now();
+                startRaf();
+            } else {
+                velocity = 0;
+                accum = 0;
+            }
+        };
+
+        terminalEl.addEventListener('touchstart', onTouchStart, { passive: false });
+        terminalEl.addEventListener('touchmove', onTouchMove, { passive: false });
+        terminalEl.addEventListener('touchend', onTouchEnd);
+        terminalEl.addEventListener('touchcancel', onTouchEnd);
+
+        this._touchScrollCleanup = () => {
+            stopRaf();
+            terminalEl.removeEventListener('touchstart', onTouchStart);
+            terminalEl.removeEventListener('touchmove', onTouchMove);
+            terminalEl.removeEventListener('touchend', onTouchEnd);
+            terminalEl.removeEventListener('touchcancel', onTouchEnd);
+        };
+    }
+
     _createWinBox(container) {
         const title = `${this.sessionId} (${this.mode})`;
 

{agentwire_dev-1.29.3 → agentwire_dev-1.29.5}/agentwire/stt/__init__.py

@@ -51,7 +51,18 @@ def get_stt_backend(config: Any) -> STTBackend:
             logger.info(f"Using STT server at {stt_url}")
             return STTServerBackend(url=stt_url, timeout=timeout)
         else:
-            logger.warning(f"STT server at {stt_url} not available, falling back to WhisperKit")
+            # Loud, actionable warning: this is the silent-failure trap. A
+            # configured-but-unreachable STT server means the moonshine server
+            # crashed or isn't started, and we're about to fall back to
+            # WhisperKit — which is usually NOT installed, so every transcription
+            # will then fail at request time with a confusing error. Say so here.
+            logger.warning(
+                "STT server configured at %s but not reachable — falling back to "
+                "WhisperKit (likely has no model installed; transcription will FAIL). "
+                "Start it with `agentwire stt start`; if it won't boot, install deps: "
+                "`uv pip install --python .venv/bin/python -e '.[stt]'`.",
+                stt_url,
+            )
 
     # Fall back to WhisperKit
     logger.info("Using local WhisperKitSTT")

{agentwire_dev-1.29.3 → agentwire_dev-1.29.5}/docs/wiki/tts/stt-self-hosted.md

@@ -100,6 +100,60 @@ done
 
 Tried — load time 25.4 s → 3.2 s (real win), per-call inference ~457 ms → ~1342 ms (≈3× slower). Only 389 / 743 graph nodes are CoreML-compatible, so the autoregressive decoder bounces ANE↔CPU per token. Not worth pursuing until the upstream moonshine_onnx graph becomes more CoreML-friendly.
 
+## Troubleshooting: "STT does nothing" (the silent double-fallback)
+
+This bit us once and is worth recognizing fast, because it fails **silently in two
+places**. Symptom: push-to-talk produces no transcription and no visible error
+(often first noticed on a tablet/phone, which red-herrings you toward the client).
+
+**The failure chain:**
+
+1. The STT server (`agentwire-stt` tmux session on `:8101`) **crashes on startup** —
+   most commonly `ModuleNotFoundError: No module named 'fastapi'` (or `moonshine_onnx`)
+   because the project `.venv` is missing the STT deps. The crash is only visible in
+   the tmux pane; the session stays alive at a shell prompt, so it *looks* started.
+2. The portal builds `self.stt` once at startup via `get_stt_backend()`. With `:8101`
+   unreachable, `STTServerBackend.is_available()` is `False`, so it **falls back to
+   `WhisperKitSTT`** — which usually has no model installed.
+3. Every `/transcribe` then returns `{"error": ...}` with HTTP **200**, so the browser
+   shows nothing. The only trace is `ERROR agentwire.stt.whisperkit: WhisperKit model
+   not found` in the portal log.
+
+**Diagnose (in order):**
+
+```bash
+agentwire doctor                                   # now prints [!!] Stt if :8101 is down
+curl -s http://localhost:8101/health               # {"status":"ok","model":{"backend":"moonshine",...}}
+tmux capture-pane -pt agentwire-stt -S -50         # shows the real crash (missing import)
+tmux capture-pane -pt agentwire-portal -S -200 | grep -i stt   # "Using STT server" vs "falling back to WhisperKit"
+```
+
+The portal log line at startup is the tell:
+- `Using STT server at http://localhost:8101` + `STT backend: STTServerBackend` → good.
+- `STT server ... not reachable — falling back to WhisperKit` → the server is down.
+
+**Fix:** install the STT deps into the venv the server actually launches with
+(`.venv/bin/python`), restart the server, then **restart the portal** (it only picks
+the backend at startup):
+
+```bash
+uv pip install --python .venv/bin/python -e '.[stt]'   # declares moonshine + soundfile + fastapi
+agentwire stt stop && agentwire stt start              # wait for "Moonshine ONNX loaded"
+agentwire portal restart --dev                         # re-runs get_stt_backend()
+```
+
+**Why it stayed hidden:** `stt.url` is configured but the server was down, and the
+WhisperKit fallback only errors at *request* time, not at startup. As of this writing
+the fallback now logs a loud, actionable warning and `agentwire doctor` health-checks
+`:8101` — so next time it surfaces immediately instead of silently degrading.
+
+**Tablet note:** if the mic does nothing on a phone/tablet *before* audio ever reaches
+`/transcribe`, that's a different problem — a non-secure browser context. Reaching the
+portal at `https://<LAN-IP>:8765` with a click-through self-signed cert is **not** a
+secure context, so `navigator.mediaDevices` is `undefined` and recording silently
+no-ops. Give the tablet a trusted origin (tunnel/valid cert) or, to test, allow the
+origin via `chrome://flags/#unsafely-treat-insecure-origin-as-secure`.
+
 ## Reference
 
 - Portal endpoint: `POST /transcribe` — multipart `file=@audio.webm`. Returns `{"text": "..."}`.

{agentwire_dev-1.29.3 → agentwire_dev-1.29.5}/pyproject.toml

@@ -43,11 +43,16 @@ dev = [
     "pytest-asyncio>=0.23.0",
     "ruff>=0.1.0",
 ]
-# STT server dependencies (faster-whisper for speech-to-text)
+# STT server dependencies. Moonshine ONNX is the default backend on Mac
+# (small, fast, CPU-only, no torch) — it MUST be declared here or a fresh venv
+# silently loses the default backend and the portal falls back to a model-less
+# WhisperKit. faster-whisper is the higher-accuracy fallback for `STT_BACKEND=whisper`.
 stt = [
     "fastapi>=0.104.0",
     "uvicorn>=0.24.0",
     "python-multipart>=0.0.6",
+    "useful-moonshine-onnx>=20250101",
+    "soundfile>=0.12.0",
     "faster-whisper>=1.0.0",
 ]
 # TTS server dependencies (for GPU machines running the TTS backend)