yaver-feedback-react-native 0.8.13 → 0.9.2

package/src/VibeChatScreen.tsx

@@ -27,9 +27,13 @@ import {
   View,
 } from 'react-native';
 import type { P2PClient } from './P2PClient';
+import { SDKVoiceSession, pcmToTempWavURI, isVoiceStreamSupported } from './voice';
+import { startPcmRecording, stopPcmRecording, isVoiceCaptureSupported } from './capture';
 
 export type VibeTurnRole = 'user' | 'assistant' | 'status';
 
+type VoiceState = 'idle' | 'recording' | 'uploading' | 'thinking' | 'speaking';
+
 export interface VibeTurn {
   id: string;
   role: VibeTurnRole;
@@ -45,6 +49,11 @@ interface Props {
   /** Called when the user taps Reload after a task completes — uses
    *  P2PClient.reloadApp() with the active project context. */
   onReload?: () => Promise<void>;
+  /** Optional context forwarded to the voice stream so the agent runs
+   *  the task against the right project / runner / model. */
+  project?: string;
+  model?: string;
+  runner?: string;
 }
 
 export function VibeChatScreen({
@@ -53,6 +62,9 @@ export function VibeChatScreen({
   initialUserPrompt,
   onClose,
   onReload,
+  project,
+  model,
+  runner,
 }: Props) {
   const [taskId, setTaskId] = useState(initialTaskId);
   const [turns, setTurns] = useState<VibeTurn[]>(() => [
@@ -77,6 +89,17 @@ export function VibeChatScreen({
   const scrollRef = useRef<ScrollView | null>(null);
   const abortRef = useRef<(() => void) | null>(null);
 
+  // ── Voice vibe coding ──────────────────────────────────────────────
+  const [voiceState, setVoiceState] = useState<VoiceState>('idle');
+  const [voiceAvailable, setVoiceAvailable] = useState(false);
+  // "local" = whisper.cpp on the host (free, private); "flux" = Deepgram
+  // nova-3 streaming. fluxAvailable gates the toggle on whether the agent
+  // has a Deepgram key. activeEngine is echoed back by the agent.
+  const [voiceMode, setVoiceMode] = useState<'local' | 'flux'>('local');
+  const [fluxAvailable, setFluxAvailable] = useState(false);
+  const [activeEngine, setActiveEngine] = useState<string>('');
+  const voiceSessionRef = useRef<SDKVoiceSession | null>(null);
+
   // Subscribe to the current task's SSE stream. Re-runs whenever the
   // taskId changes (resumeTask reuses the same id, so this only fires
   // once per task — which is fine).
@@ -192,6 +215,157 @@ export function VibeChatScreen({
     }
   }, [isReloading, onReload]);
 
+  // Probe whether voice is usable: deps present (expo-av + expo-file-
+  // system + buffer) AND the agent reports STT/TTS ready. Hide the mic
+  // entirely otherwise so users never tap a dead button.
+  useEffect(() => {
+    let cancelled = false;
+    (async () => {
+      if (!isVoiceCaptureSupported() || !isVoiceStreamSupported()) return;
+      try {
+        const res = await fetch(`${client.agentBaseUrl}/voice/status`, { headers: client.voiceAuthHeaders() });
+        if (!res.ok) return;
+        const body = await res.json();
+        if (cancelled) return;
+        // Local whisper is always usable when voice is enabled; Flux needs
+        // a Deepgram key on the agent. Show the mic if either path works.
+        const localOk = !!body?.enabled;
+        const fluxOk = !!body?.enabled && !!body?.deepgramSet;
+        if (localOk || fluxOk) setVoiceAvailable(true);
+        setFluxAvailable(fluxOk);
+        if (!localOk && fluxOk) setVoiceMode('flux');
+      } catch { /* leave hidden */ }
+    })();
+    return () => { cancelled = true; };
+  }, [client]);
+
+  useEffect(() => () => { voiceSessionRef.current?.close(); }, []);
+
+  // Local TTS: the agent streams no audio for "local"/"device" engines,
+  // so the client speaks the result text with the device synthesizer.
+  // expo-speech is optional — if absent, the text is still shown.
+  const speakLocalText = useCallback((text: string) => {
+    if (!text) return;
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const Speech = require('expo-speech');
+      const headline = text.length > 280 ? `${text.slice(0, 280)} — see screen for the rest.` : text;
+      Speech.stop?.();
+      Speech.speak?.(headline);
+    } catch { /* expo-speech not installed — text remains visible */ }
+  }, []);
+
+  const playTTS = useCallback(async (pcm: Uint8Array, sampleRate: number) => {
+    try {
+      const wavUri = await pcmToTempWavURI(pcm, sampleRate);
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const { Audio } = require('expo-av');
+      const { sound } = await Audio.Sound.createAsync({ uri: wavUri }, { shouldPlay: true });
+      sound.setOnPlaybackStatusUpdate((st: any) => {
+        if (st.didJustFinish) sound.unloadAsync().catch(() => {});
+      });
+    } catch { /* playback best-effort */ }
+  }, []);
+
+  const stopVoiceAndProcess = useCallback(async () => {
+    setVoiceState('uploading');
+    let uri: string | null = null;
+    try {
+      uri = await stopPcmRecording();
+    } catch (e) {
+      setVoiceState('idle');
+      setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+      return;
+    }
+    if (!uri) { setVoiceState('idle'); return; }
+
+    const useFlux = voiceMode === 'flux' && fluxAvailable;
+    const session = new SDKVoiceSession({
+      onProviders: (stt, tts) => setActiveEngine(stt === 'deepgram' ? 'Flux (Deepgram)' : stt === 'local' ? 'Local (whisper)' : stt),
+      onTranscriptPartial: (t) => {
+        setTurns((prev) => {
+          const next = prev.filter((x) => x.id !== 'voice-partial');
+          next.push({ id: 'voice-partial', role: 'status', text: `🎙 ${t}`, timestamp: Date.now() });
+          return next;
+        });
+      },
+      onTranscriptFinal: (t) => {
+        setVoiceState('thinking');
+        setTurns((prev) => [
+          ...prev.filter((x) => x.id !== 'voice-partial'),
+          { id: `user-voice-${Date.now()}`, role: 'user', text: t, timestamp: Date.now() },
+          { id: `status-${Date.now()}`, role: 'status', text: 'thinking…', timestamp: Date.now() },
+        ]);
+      },
+      onTaskCreated: (id) => {
+        // Hand the chat's SSE subscription the new task so its agent
+        // output streams into the transcript exactly like a typed turn.
+        if (id) { setStatus('running'); setStreamBuffer(''); setTaskId(id); }
+      },
+      onTaskResult: (_id, text) => {
+        setVoiceState('speaking');
+        // Local TTS path: agent sends no audio frames, so speak here.
+        if (!useFlux) speakLocalText(text);
+      },
+      onTTSReady: (pcm, sr) => { void playTTS(pcm, sr); },
+      onDone: () => setTimeout(() => setVoiceState('idle'), 1200),
+      onError: (msg) => {
+        setVoiceState('idle');
+        setTurns((prev) => [...prev.filter((x) => x.id !== 'voice-partial'), { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${msg}`, timestamp: Date.now() }]);
+      },
+    });
+    voiceSessionRef.current = session;
+    try {
+      await session.start({
+        wsUrl: client.voiceStreamUrl(),
+        headers: client.voiceAuthHeaders(),
+        project,
+        model,
+        runner,
+        surface: 'feedback-sdk',
+        ttsBudget: 280,
+        // Local: whisper.cpp on the host + device synth. Flux: Deepgram
+        // nova-3 STT + Aura TTS streamed back as PCM.
+        sttProvider: useFlux ? 'deepgram' : 'local',
+        ttsProvider: useFlux ? 'deepgram' : 'local',
+      });
+      await session.streamAudioFile(uri, { skipWavHeader: true });
+      session.finalize();
+    } catch (e) {
+      setVoiceState('idle');
+      session.close();
+      setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+    }
+  }, [client, project, model, runner, playTTS, voiceMode, fluxAvailable, speakLocalText]);
+
+  const handleVoicePress = useCallback(async () => {
+    if (voiceState === 'recording') {
+      void stopVoiceAndProcess();
+      return;
+    }
+    if (voiceState !== 'idle') {
+      // Mid-flow tap cancels.
+      voiceSessionRef.current?.close();
+      voiceSessionRef.current = null;
+      setVoiceState('idle');
+      return;
+    }
+    try {
+      await startPcmRecording();
+      setVoiceState('recording');
+    } catch (e) {
+      setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+    }
+  }, [voiceState, stopVoiceAndProcess]);
+
+  const voiceLabel: Record<VoiceState, string> = {
+    idle: '🎙 speak',
+    recording: '■ stop',
+    uploading: 'sending…',
+    thinking: 'thinking…',
+    speaking: 'speaking…',
+  };
+
   return (
     <View style={styles.container}>
       <View style={styles.header}>
@@ -240,6 +414,12 @@ export function VibeChatScreen({
       </ScrollView>
 
       <View style={styles.footer}>
+        {voiceAvailable && voiceState !== 'idle' && (
+          <Text style={styles.engineCaption}>
+            {voiceState === 'recording' ? 'listening' : voiceState === 'uploading' ? 'sending' : voiceState === 'thinking' ? 'agent working' : 'speaking'}
+            {activeEngine ? ` · ${activeEngine}` : ` · ${voiceMode === 'flux' ? 'Flux (Deepgram)' : 'Local (whisper)'}`}
+          </Text>
+        )}
         <TextInput
           style={styles.input}
           value={followUp}
@@ -250,6 +430,40 @@ export function VibeChatScreen({
           multiline
         />
         <View style={styles.actions}>
+          {voiceAvailable && (
+            <>
+              {/* Local ↔ Flux engine toggle. Only shows Flux when the
+                  agent has a Deepgram key; otherwise the label just
+                  states "Local" so the active engine is always clear. */}
+              {fluxAvailable ? (
+                <TouchableOpacity
+                  style={[styles.actionBtn, styles.engineToggle]}
+                  onPress={() => setVoiceMode((m) => (m === 'local' ? 'flux' : 'local'))}
+                  disabled={voiceState !== 'idle'}
+                  accessibilityLabel="Toggle voice engine"
+                >
+                  <Text style={styles.engineToggleText}>{voiceMode === 'flux' ? '⚡ Flux' : '🔒 Local'}</Text>
+                </TouchableOpacity>
+              ) : (
+                <View style={[styles.actionBtn, styles.engineToggle]}>
+                  <Text style={styles.engineToggleText}>🔒 Local</Text>
+                </View>
+              )}
+              <TouchableOpacity
+                style={[
+                  styles.actionBtn,
+                  styles.voiceBtn,
+                  voiceState === 'recording' && styles.voiceBtnActive,
+                  (voiceState === 'uploading' || voiceState === 'thinking' || voiceState === 'speaking') && styles.actionBtnDisabled,
+                ]}
+                onPress={handleVoicePress}
+                disabled={voiceState === 'uploading' || voiceState === 'thinking' || voiceState === 'speaking'}
+                accessibilityLabel="Vibe code by voice"
+              >
+                <Text style={styles.actionText}>{voiceLabel[voiceState]}</Text>
+              </TouchableOpacity>
+            </>
+          )}
           {onReload && (
             <TouchableOpacity
               style={[
@@ -357,6 +571,11 @@ const styles = StyleSheet.create({
   },
   actionBtnDisabled: { opacity: 0.5 },
   reloadBtn: { backgroundColor: 'rgba(255,255,255,0.08)' },
+  voiceBtn: { backgroundColor: 'rgba(16,185,129,0.18)' },
+  voiceBtnActive: { backgroundColor: '#ef4444' },
+  engineToggle: { backgroundColor: 'rgba(255,255,255,0.06)', marginRight: 'auto', marginLeft: 0 },
+  engineToggleText: { color: '#cbd5e1', fontSize: 12, fontWeight: '600' },
+  engineCaption: { color: '#9ca3af', fontSize: 11, marginBottom: 6, marginLeft: 4 },
   sendBtn: { backgroundColor: '#7582f5' },
   actionText: { color: '#fff', fontSize: 13, fontWeight: '600' },
 });