yaver-feedback-react-native 0.8.13 → 0.9.0

package/src/P2PClient.ts

@@ -3,8 +3,10 @@ import {
   CapabilitySnapshot,
   FeedbackBundle,
   IncidentEvent,
+  OpenCodeConfigSummary,
   OperationState,
   RunnerBrowserAuthSession,
+  RunnerAuthStatusRow,
   TestSession,
   VoiceCapability,
 } from './types';
@@ -154,6 +156,25 @@ export class P2PClient {
     this.relayPassword = password;
   }
 
+  /** Read-only base URL — used by the voice vibe-coding path to probe
+   *  GET /voice/status before opening the stream. */
+  get agentBaseUrl(): string {
+    return this.baseUrl;
+  }
+
+  /** WebSocket URL for the agent's voice stream (WS /voice/stream). The
+   *  voice vibe-coding loop streams mic audio here and receives the
+   *  transcript + agent task + TTS frames back. */
+  voiceStreamUrl(): string {
+    return this.baseUrl.replace(/^http/, 'ws') + '/voice/stream';
+  }
+
+  /** Auth headers for the voice WS + status probe — same bearer (and
+   *  relay password) as every other agent request. */
+  voiceAuthHeaders(): Record<string, string> {
+    return this.authHeaders();
+  }
+
   /** Merge in Authorization + (optional) X-Relay-Password on top of a header block. */
   private authHeaders(extra: Record<string, string> = {}): Record<string, string> {
     const h: Record<string, string> = { ...extra };
@@ -222,6 +243,60 @@ export class P2PClient {
     return data.session as RunnerBrowserAuthSession;
   }
 
+  async getRunnerAuthStatus(): Promise<RunnerAuthStatusRow[]> {
+    const resp = await fetch(`${this.baseUrl}/runner-auth/status`, {
+      headers: this.authHeaders(),
+    });
+    if (!resp.ok) {
+      const text = await resp.text().catch(() => '');
+      throw new Error(`getRunnerAuthStatus HTTP ${resp.status}: ${text}`);
+    }
+    const data = await resp.json().catch(() => ({} as Record<string, unknown>));
+    return Array.isArray(data.runners) ? (data.runners as RunnerAuthStatusRow[]) : [];
+  }
+
+  async getOpenCodeConfig(): Promise<OpenCodeConfigSummary | null> {
+    const resp = await fetch(`${this.baseUrl}/runner/opencode/config`, {
+      headers: this.authHeaders(),
+    });
+    if (!resp.ok) {
+      const text = await resp.text().catch(() => '');
+      throw new Error(`getOpenCodeConfig HTTP ${resp.status}: ${text}`);
+    }
+    const data = await resp.json().catch(() => ({} as Record<string, unknown>));
+    return (data.config ?? null) as OpenCodeConfigSummary | null;
+  }
+
+  async saveOpenCodeConfig(patch: {
+    defaultAgent?: string;
+    model?: string;
+    smallModel?: string;
+    buildModel?: string;
+    planModel?: string;
+    providers?: Array<{
+      id: string;
+      name?: string;
+      baseUrl?: string;
+      apiKey?: string;
+      delete?: boolean;
+    }>;
+  }): Promise<{ ok: boolean; config?: OpenCodeConfigSummary; error?: string }> {
+    try {
+      const resp = await fetch(`${this.baseUrl}/runner/opencode/config`, {
+        method: 'POST',
+        headers: { ...this.authHeaders(), 'Content-Type': 'application/json' },
+        body: JSON.stringify(patch),
+      });
+      const data = await resp.json().catch(() => ({} as Record<string, unknown>));
+      if (!resp.ok) {
+        return { ok: false, error: (data.error as string | undefined) || `HTTP ${resp.status}` };
+      }
+      return { ok: true, config: data.config as OpenCodeConfigSummary | undefined };
+    } catch (err) {
+      return { ok: false, error: err instanceof Error ? err.message : String(err) };
+    }
+  }
+
   async capabilitySnapshot(): Promise<CapabilitySnapshot | null> {
     try {
       const resp = await fetch(`${this.baseUrl}/capabilities/snapshot`, { headers: this.authHeaders() });
@@ -422,6 +497,10 @@ export class P2PClient {
       s2sReady: data.s2sReady ?? false,
       sttProvider: data.sttProvider ?? undefined,
       sttReady: data.sttReady ?? false,
+      ttsProvider: data.ttsProvider ?? undefined,
+      ttsReady: data.ttsReady ?? false,
+      enabled: data.enabled ?? false,
+      defaultProject: data.defaultProject ?? undefined,
     };
   }
 

package/src/VibeChatScreen.tsx

@@ -27,9 +27,13 @@ import {
   View,
 } from 'react-native';
 import type { P2PClient } from './P2PClient';
+import { SDKVoiceSession, pcmToTempWavURI, isVoiceStreamSupported } from './voice';
+import { startPcmRecording, stopPcmRecording, isVoiceCaptureSupported } from './capture';
 
 export type VibeTurnRole = 'user' | 'assistant' | 'status';
 
+type VoiceState = 'idle' | 'recording' | 'uploading' | 'thinking' | 'speaking';
+
 export interface VibeTurn {
   id: string;
   role: VibeTurnRole;
@@ -45,6 +49,11 @@ interface Props {
   /** Called when the user taps Reload after a task completes — uses
    *  P2PClient.reloadApp() with the active project context. */
   onReload?: () => Promise<void>;
+  /** Optional context forwarded to the voice stream so the agent runs
+   *  the task against the right project / runner / model. */
+  project?: string;
+  model?: string;
+  runner?: string;
 }
 
 export function VibeChatScreen({
@@ -53,6 +62,9 @@ export function VibeChatScreen({
   initialUserPrompt,
   onClose,
   onReload,
+  project,
+  model,
+  runner,
 }: Props) {
   const [taskId, setTaskId] = useState(initialTaskId);
   const [turns, setTurns] = useState<VibeTurn[]>(() => [
@@ -77,6 +89,17 @@ export function VibeChatScreen({
   const scrollRef = useRef<ScrollView | null>(null);
   const abortRef = useRef<(() => void) | null>(null);
 
+  // ── Voice vibe coding ──────────────────────────────────────────────
+  const [voiceState, setVoiceState] = useState<VoiceState>('idle');
+  const [voiceAvailable, setVoiceAvailable] = useState(false);
+  // "local" = whisper.cpp on the host (free, private); "flux" = Deepgram
+  // nova-3 streaming. fluxAvailable gates the toggle on whether the agent
+  // has a Deepgram key. activeEngine is echoed back by the agent.
+  const [voiceMode, setVoiceMode] = useState<'local' | 'flux'>('local');
+  const [fluxAvailable, setFluxAvailable] = useState(false);
+  const [activeEngine, setActiveEngine] = useState<string>('');
+  const voiceSessionRef = useRef<SDKVoiceSession | null>(null);
+
   // Subscribe to the current task's SSE stream. Re-runs whenever the
   // taskId changes (resumeTask reuses the same id, so this only fires
   // once per task — which is fine).
@@ -192,6 +215,157 @@ export function VibeChatScreen({
     }
   }, [isReloading, onReload]);
 
+  // Probe whether voice is usable: deps present (expo-av + expo-file-
+  // system + buffer) AND the agent reports STT/TTS ready. Hide the mic
+  // entirely otherwise so users never tap a dead button.
+  useEffect(() => {
+    let cancelled = false;
+    (async () => {
+      if (!isVoiceCaptureSupported() || !isVoiceStreamSupported()) return;
+      try {
+        const res = await fetch(`${client.agentBaseUrl}/voice/status`, { headers: client.voiceAuthHeaders() });
+        if (!res.ok) return;
+        const body = await res.json();
+        if (cancelled) return;
+        // Local whisper is always usable when voice is enabled; Flux needs
+        // a Deepgram key on the agent. Show the mic if either path works.
+        const localOk = !!body?.enabled;
+        const fluxOk = !!body?.enabled && !!body?.deepgramSet;
+        if (localOk || fluxOk) setVoiceAvailable(true);
+        setFluxAvailable(fluxOk);
+        if (!localOk && fluxOk) setVoiceMode('flux');
+      } catch { /* leave hidden */ }
+    })();
+    return () => { cancelled = true; };
+  }, [client]);
+
+  useEffect(() => () => { voiceSessionRef.current?.close(); }, []);
+
+  // Local TTS: the agent streams no audio for "local"/"device" engines,
+  // so the client speaks the result text with the device synthesizer.
+  // expo-speech is optional — if absent, the text is still shown.
+  const speakLocalText = useCallback((text: string) => {
+    if (!text) return;
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const Speech = require('expo-speech');
+      const headline = text.length > 280 ? `${text.slice(0, 280)} — see screen for the rest.` : text;
+      Speech.stop?.();
+      Speech.speak?.(headline);
+    } catch { /* expo-speech not installed — text remains visible */ }
+  }, []);
+
+  const playTTS = useCallback(async (pcm: Uint8Array, sampleRate: number) => {
+    try {
+      const wavUri = await pcmToTempWavURI(pcm, sampleRate);
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const { Audio } = require('expo-av');
+      const { sound } = await Audio.Sound.createAsync({ uri: wavUri }, { shouldPlay: true });
+      sound.setOnPlaybackStatusUpdate((st: any) => {
+        if (st.didJustFinish) sound.unloadAsync().catch(() => {});
+      });
+    } catch { /* playback best-effort */ }
+  }, []);
+
+  const stopVoiceAndProcess = useCallback(async () => {
+    setVoiceState('uploading');
+    let uri: string | null = null;
+    try {
+      uri = await stopPcmRecording();
+    } catch (e) {
+      setVoiceState('idle');
+      setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+      return;
+    }
+    if (!uri) { setVoiceState('idle'); return; }
+
+    const useFlux = voiceMode === 'flux' && fluxAvailable;
+    const session = new SDKVoiceSession({
+      onProviders: (stt, tts) => setActiveEngine(stt === 'deepgram' ? 'Flux (Deepgram)' : stt === 'local' ? 'Local (whisper)' : stt),
+      onTranscriptPartial: (t) => {
+        setTurns((prev) => {
+          const next = prev.filter((x) => x.id !== 'voice-partial');
+          next.push({ id: 'voice-partial', role: 'status', text: `🎙 ${t}`, timestamp: Date.now() });
+          return next;
+        });
+      },
+      onTranscriptFinal: (t) => {
+        setVoiceState('thinking');
+        setTurns((prev) => [
+          ...prev.filter((x) => x.id !== 'voice-partial'),
+          { id: `user-voice-${Date.now()}`, role: 'user', text: t, timestamp: Date.now() },
+          { id: `status-${Date.now()}`, role: 'status', text: 'thinking…', timestamp: Date.now() },
+        ]);
+      },
+      onTaskCreated: (id) => {
+        // Hand the chat's SSE subscription the new task so its agent
+        // output streams into the transcript exactly like a typed turn.
+        if (id) { setStatus('running'); setStreamBuffer(''); setTaskId(id); }
+      },
+      onTaskResult: (_id, text) => {
+        setVoiceState('speaking');
+        // Local TTS path: agent sends no audio frames, so speak here.
+        if (!useFlux) speakLocalText(text);
+      },
+      onTTSReady: (pcm, sr) => { void playTTS(pcm, sr); },
+      onDone: () => setTimeout(() => setVoiceState('idle'), 1200),
+      onError: (msg) => {
+        setVoiceState('idle');
+        setTurns((prev) => [...prev.filter((x) => x.id !== 'voice-partial'), { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${msg}`, timestamp: Date.now() }]);
+      },
+    });
+    voiceSessionRef.current = session;
+    try {
+      await session.start({
+        wsUrl: client.voiceStreamUrl(),
+        headers: client.voiceAuthHeaders(),
+        project,
+        model,
+        runner,
+        surface: 'feedback-sdk',
+        ttsBudget: 280,
+        // Local: whisper.cpp on the host + device synth. Flux: Deepgram
+        // nova-3 STT + Aura TTS streamed back as PCM.
+        sttProvider: useFlux ? 'deepgram' : 'local',
+        ttsProvider: useFlux ? 'deepgram' : 'local',
+      });
+      await session.streamAudioFile(uri, { skipWavHeader: true });
+      session.finalize();
+    } catch (e) {
+      setVoiceState('idle');
+      session.close();
+      setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+    }
+  }, [client, project, model, runner, playTTS, voiceMode, fluxAvailable, speakLocalText]);
+
+  const handleVoicePress = useCallback(async () => {
+    if (voiceState === 'recording') {
+      void stopVoiceAndProcess();
+      return;
+    }
+    if (voiceState !== 'idle') {
+      // Mid-flow tap cancels.
+      voiceSessionRef.current?.close();
+      voiceSessionRef.current = null;
+      setVoiceState('idle');
+      return;
+    }
+    try {
+      await startPcmRecording();
+      setVoiceState('recording');
+    } catch (e) {
+      setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+    }
+  }, [voiceState, stopVoiceAndProcess]);
+
+  const voiceLabel: Record<VoiceState, string> = {
+    idle: '🎙 speak',
+    recording: '■ stop',
+    uploading: 'sending…',
+    thinking: 'thinking…',
+    speaking: 'speaking…',
+  };
+
   return (
     <View style={styles.container}>
       <View style={styles.header}>
@@ -240,6 +414,12 @@ export function VibeChatScreen({
       </ScrollView>
 
       <View style={styles.footer}>
+        {voiceAvailable && voiceState !== 'idle' && (
+          <Text style={styles.engineCaption}>
+            {voiceState === 'recording' ? 'listening' : voiceState === 'uploading' ? 'sending' : voiceState === 'thinking' ? 'agent working' : 'speaking'}
+            {activeEngine ? ` · ${activeEngine}` : ` · ${voiceMode === 'flux' ? 'Flux (Deepgram)' : 'Local (whisper)'}`}
+          </Text>
+        )}
         <TextInput
           style={styles.input}
           value={followUp}
@@ -250,6 +430,40 @@ export function VibeChatScreen({
           multiline
         />
         <View style={styles.actions}>
+          {voiceAvailable && (
+            <>
+              {/* Local ↔ Flux engine toggle. Only shows Flux when the
+                  agent has a Deepgram key; otherwise the label just
+                  states "Local" so the active engine is always clear. */}
+              {fluxAvailable ? (
+                <TouchableOpacity
+                  style={[styles.actionBtn, styles.engineToggle]}
+                  onPress={() => setVoiceMode((m) => (m === 'local' ? 'flux' : 'local'))}
+                  disabled={voiceState !== 'idle'}
+                  accessibilityLabel="Toggle voice engine"
+                >
+                  <Text style={styles.engineToggleText}>{voiceMode === 'flux' ? '⚡ Flux' : '🔒 Local'}</Text>
+                </TouchableOpacity>
+              ) : (
+                <View style={[styles.actionBtn, styles.engineToggle]}>
+                  <Text style={styles.engineToggleText}>🔒 Local</Text>
+                </View>
+              )}
+              <TouchableOpacity
+                style={[
+                  styles.actionBtn,
+                  styles.voiceBtn,
+                  voiceState === 'recording' && styles.voiceBtnActive,
+                  (voiceState === 'uploading' || voiceState === 'thinking' || voiceState === 'speaking') && styles.actionBtnDisabled,
+                ]}
+                onPress={handleVoicePress}
+                disabled={voiceState === 'uploading' || voiceState === 'thinking' || voiceState === 'speaking'}
+                accessibilityLabel="Vibe code by voice"
+              >
+                <Text style={styles.actionText}>{voiceLabel[voiceState]}</Text>
+              </TouchableOpacity>
+            </>
+          )}
           {onReload && (
             <TouchableOpacity
               style={[
@@ -357,6 +571,11 @@ const styles = StyleSheet.create({
   },
   actionBtnDisabled: { opacity: 0.5 },
   reloadBtn: { backgroundColor: 'rgba(255,255,255,0.08)' },
+  voiceBtn: { backgroundColor: 'rgba(16,185,129,0.18)' },
+  voiceBtnActive: { backgroundColor: '#ef4444' },
+  engineToggle: { backgroundColor: 'rgba(255,255,255,0.06)', marginRight: 'auto', marginLeft: 0 },
+  engineToggleText: { color: '#cbd5e1', fontSize: 12, fontWeight: '600' },
+  engineCaption: { color: '#9ca3af', fontSize: 11, marginBottom: 6, marginLeft: 4 },
   sendBtn: { backgroundColor: '#7582f5' },
   actionText: { color: '#fff', fontSize: 13, fontWeight: '600' },
 });

package/src/capture.ts

@@ -336,3 +336,54 @@ export async function stopAudioRecording(): Promise<{ path: string; duration: nu
 export function isAudioRecording(): boolean {
   return audioRecorderActive;
 }
+
+// ── Voice-stream recording (raw LPCM WAV) ──────────────────────────────
+// The voice vibe-coding path streams audio to the agent's STT WS, which
+// expects raw 16-bit / 16 kHz mono PCM (we strip the WAV header on the
+// way out). That's a different format from the HIGH_QUALITY m4a recorder
+// above — a compressed .m4a can't be streamed to Deepgram/whisper — so
+// this uses its own recording options, mirroring the Yaver app's
+// AgentVoiceButton.
+
+let pcmRecorderRef: any = null;
+let pcmRecorderActive = false;
+
+// Raw LPCM 16-bit LE, 16 kHz mono. iOS uses lpcm; Android records WAV.
+const PCM_RECORDING_OPTIONS: any = {
+  android: { extension: '.wav', outputFormat: 2, audioEncoder: 3, sampleRate: 16000, numberOfChannels: 1, bitRate: 256000 },
+  ios: {
+    extension: '.wav', outputFormat: 'lpcm', audioQuality: 0x40, sampleRate: 16000,
+    numberOfChannels: 1, bitRate: 256000, linearPCMBitDepth: 16, linearPCMIsBigEndian: false, linearPCMIsFloat: false,
+  },
+  web: { mimeType: 'audio/wav', bitsPerSecond: 256000 },
+};
+
+/** Begin a raw-PCM recording for the voice stream. */
+export async function startPcmRecording(): Promise<void> {
+  if (pcmRecorderActive) throw new Error('[YaverFeedback] A voice recording is already in progress.');
+  const ExpoAv = loadExpoAvOrThrow();
+  const { Audio } = ExpoAv;
+  const perm = await Audio.requestPermissionsAsync();
+  if (!perm.granted) {
+    throw new Error('[YaverFeedback] Microphone permission denied. Enable it in Settings ▸ Your App ▸ Microphone.');
+  }
+  await Audio.setAudioModeAsync({ allowsRecordingIOS: true, playsInSilentModeIOS: true, staysActiveInBackground: false });
+  const { recording } = await Audio.Recording.createAsync(PCM_RECORDING_OPTIONS);
+  pcmRecorderRef = recording;
+  pcmRecorderActive = true;
+}
+
+/** Stop the voice recording; returns the WAV file:// URI (or null). */
+export async function stopPcmRecording(): Promise<string | null> {
+  if (!pcmRecorderActive || !pcmRecorderRef) return null;
+  const recording = pcmRecorderRef;
+  pcmRecorderRef = null;
+  pcmRecorderActive = false;
+  try { await recording.stopAndUnloadAsync(); } catch { /* already stopped */ }
+  return typeof recording.getURI === 'function' ? recording.getURI() : null;
+}
+
+/** Whether a voice-stream recording is currently active. */
+export function isPcmRecording(): boolean {
+  return pcmRecorderActive;
+}

package/src/types.ts

@@ -20,6 +20,49 @@ export interface RunnerBrowserAuthSession {
   completedAt?: number;
 }
 
+export interface RunnerAuthStatusRow {
+  id: string;
+  name: string;
+  installed: boolean;
+  ready: boolean;
+  authConfigured: boolean;
+  authSource?: string;
+  warning?: string;
+  error?: string;
+  path?: string;
+  detail?: string;
+  version?: string;
+}
+
+export interface OpenCodeProviderSummary {
+  id: string;
+  name?: string;
+  hasApiKey?: boolean;
+  baseUrl?: string;
+  models?: Array<{ id: string; name?: string; provider?: string }>;
+}
+
+export interface OpenCodeAgentSummary {
+  name: string;
+  model?: string;
+  description?: string;
+  isBuiltin?: boolean;
+}
+
+export interface OpenCodeConfigSummary {
+  path: string;
+  exists: boolean;
+  defaultAgent?: string;
+  model?: string;
+  smallModel?: string;
+  buildModel?: string;
+  planModel?: string;
+  providers?: OpenCodeProviderSummary[];
+  models?: Array<{ id: string; name?: string; provider?: string }>;
+  agents?: OpenCodeAgentSummary[];
+  diagnostics?: string[];
+}
+
 export interface IncidentEvent {
   id: string;
   timestamp: number;
@@ -451,12 +494,20 @@ export interface TestSession {
 export interface VoiceCapability {
   /** Always true — mobile can always record and send audio. */
   voiceInputEnabled: boolean;
-  /** Speech-to-speech provider (e.g. "personaplex", "openai"), or null. */
+  /** Speech-to-speech provider (legacy), or null. */
   s2sProvider?: string;
   /** Whether the S2S provider is ready for real-time sessions. */
   s2sReady?: boolean;
-  /** Speech-to-text provider for transcription (e.g. "whisper", "openai"). */
+  /** Speech-to-text provider for transcription, e.g. "deepgram" for Deepgram Flux. */
   sttProvider?: string;
   /** Whether STT is ready (auto-transcription of voice input). */
   sttReady?: boolean;
+  /** Text-to-speech provider for readback, e.g. "cartesia". */
+  ttsProvider?: string;
+  /** Whether TTS readback is ready. */
+  ttsReady?: boolean;
+  /** Whether the agent-side hands-free task loop is enabled. */
+  enabled?: boolean;
+  /** Default project slug used by the agent voice loop. */
+  defaultProject?: string;
 }