yaver-feedback-react-native 0.8.13 → 0.9.0

package/dist/VibeChatScreen.js

@@ -53,7 +53,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.VibeChatScreen = VibeChatScreen;
 const react_1 = __importStar(require("react"));
 const react_native_1 = require("react-native");
-function VibeChatScreen({ client, initialTaskId, initialUserPrompt, onClose, onReload, }) {
+const voice_1 = require("./voice");
+const capture_1 = require("./capture");
+function VibeChatScreen({ client, initialTaskId, initialUserPrompt, onClose, onReload, project, model, runner, }) {
     const [taskId, setTaskId] = (0, react_1.useState)(initialTaskId);
     const [turns, setTurns] = (0, react_1.useState)(() => [
         {
@@ -76,6 +78,16 @@ function VibeChatScreen({ client, initialTaskId, initialUserPrompt, onClose, onR
     const [isReloading, setIsReloading] = (0, react_1.useState)(false);
     const scrollRef = (0, react_1.useRef)(null);
     const abortRef = (0, react_1.useRef)(null);
+    // ── Voice vibe coding ──────────────────────────────────────────────
+    const [voiceState, setVoiceState] = (0, react_1.useState)('idle');
+    const [voiceAvailable, setVoiceAvailable] = (0, react_1.useState)(false);
+    // "local" = whisper.cpp on the host (free, private); "flux" = Deepgram
+    // nova-3 streaming. fluxAvailable gates the toggle on whether the agent
+    // has a Deepgram key. activeEngine is echoed back by the agent.
+    const [voiceMode, setVoiceMode] = (0, react_1.useState)('local');
+    const [fluxAvailable, setFluxAvailable] = (0, react_1.useState)(false);
+    const [activeEngine, setActiveEngine] = (0, react_1.useState)('');
+    const voiceSessionRef = (0, react_1.useRef)(null);
     // Subscribe to the current task's SSE stream. Re-runs whenever the
     // taskId changes (resumeTask reuses the same id, so this only fires
     // once per task — which is fine).
@@ -195,6 +207,170 @@ function VibeChatScreen({ client, initialTaskId, initialUserPrompt, onClose, onR
             setIsReloading(false);
         }
     }, [isReloading, onReload]);
+    // Probe whether voice is usable: deps present (expo-av + expo-file-
+    // system + buffer) AND the agent reports STT/TTS ready. Hide the mic
+    // entirely otherwise so users never tap a dead button.
+    (0, react_1.useEffect)(() => {
+        let cancelled = false;
+        (async () => {
+            if (!(0, capture_1.isVoiceCaptureSupported)() || !(0, voice_1.isVoiceStreamSupported)())
+                return;
+            try {
+                const res = await fetch(`${client.agentBaseUrl}/voice/status`, { headers: client.voiceAuthHeaders() });
+                if (!res.ok)
+                    return;
+                const body = await res.json();
+                if (cancelled)
+                    return;
+                // Local whisper is always usable when voice is enabled; Flux needs
+                // a Deepgram key on the agent. Show the mic if either path works.
+                const localOk = !!body?.enabled;
+                const fluxOk = !!body?.enabled && !!body?.deepgramSet;
+                if (localOk || fluxOk)
+                    setVoiceAvailable(true);
+                setFluxAvailable(fluxOk);
+                if (!localOk && fluxOk)
+                    setVoiceMode('flux');
+            }
+            catch { /* leave hidden */ }
+        })();
+        return () => { cancelled = true; };
+    }, [client]);
+    (0, react_1.useEffect)(() => () => { voiceSessionRef.current?.close(); }, []);
+    // Local TTS: the agent streams no audio for "local"/"device" engines,
+    // so the client speaks the result text with the device synthesizer.
+    // expo-speech is optional — if absent, the text is still shown.
+    const speakLocalText = (0, react_1.useCallback)((text) => {
+        if (!text)
+            return;
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            const Speech = require('expo-speech');
+            const headline = text.length > 280 ? `${text.slice(0, 280)} — see screen for the rest.` : text;
+            Speech.stop?.();
+            Speech.speak?.(headline);
+        }
+        catch { /* expo-speech not installed — text remains visible */ }
+    }, []);
+    const playTTS = (0, react_1.useCallback)(async (pcm, sampleRate) => {
+        try {
+            const wavUri = await (0, voice_1.pcmToTempWavURI)(pcm, sampleRate);
+            // eslint-disable-next-line @typescript-eslint/no-var-requires
+            const { Audio } = require('expo-av');
+            const { sound } = await Audio.Sound.createAsync({ uri: wavUri }, { shouldPlay: true });
+            sound.setOnPlaybackStatusUpdate((st) => {
+                if (st.didJustFinish)
+                    sound.unloadAsync().catch(() => { });
+            });
+        }
+        catch { /* playback best-effort */ }
+    }, []);
+    const stopVoiceAndProcess = (0, react_1.useCallback)(async () => {
+        setVoiceState('uploading');
+        let uri = null;
+        try {
+            uri = await (0, capture_1.stopPcmRecording)();
+        }
+        catch (e) {
+            setVoiceState('idle');
+            setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+            return;
+        }
+        if (!uri) {
+            setVoiceState('idle');
+            return;
+        }
+        const useFlux = voiceMode === 'flux' && fluxAvailable;
+        const session = new voice_1.SDKVoiceSession({
+            onProviders: (stt, tts) => setActiveEngine(stt === 'deepgram' ? 'Flux (Deepgram)' : stt === 'local' ? 'Local (whisper)' : stt),
+            onTranscriptPartial: (t) => {
+                setTurns((prev) => {
+                    const next = prev.filter((x) => x.id !== 'voice-partial');
+                    next.push({ id: 'voice-partial', role: 'status', text: `🎙 ${t}`, timestamp: Date.now() });
+                    return next;
+                });
+            },
+            onTranscriptFinal: (t) => {
+                setVoiceState('thinking');
+                setTurns((prev) => [
+                    ...prev.filter((x) => x.id !== 'voice-partial'),
+                    { id: `user-voice-${Date.now()}`, role: 'user', text: t, timestamp: Date.now() },
+                    { id: `status-${Date.now()}`, role: 'status', text: 'thinking…', timestamp: Date.now() },
+                ]);
+            },
+            onTaskCreated: (id) => {
+                // Hand the chat's SSE subscription the new task so its agent
+                // output streams into the transcript exactly like a typed turn.
+                if (id) {
+                    setStatus('running');
+                    setStreamBuffer('');
+                    setTaskId(id);
+                }
+            },
+            onTaskResult: (_id, text) => {
+                setVoiceState('speaking');
+                // Local TTS path: agent sends no audio frames, so speak here.
+                if (!useFlux)
+                    speakLocalText(text);
+            },
+            onTTSReady: (pcm, sr) => { void playTTS(pcm, sr); },
+            onDone: () => setTimeout(() => setVoiceState('idle'), 1200),
+            onError: (msg) => {
+                setVoiceState('idle');
+                setTurns((prev) => [...prev.filter((x) => x.id !== 'voice-partial'), { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${msg}`, timestamp: Date.now() }]);
+            },
+        });
+        voiceSessionRef.current = session;
+        try {
+            await session.start({
+                wsUrl: client.voiceStreamUrl(),
+                headers: client.voiceAuthHeaders(),
+                project,
+                model,
+                runner,
+                surface: 'feedback-sdk',
+                ttsBudget: 280,
+                // Local: whisper.cpp on the host + device synth. Flux: Deepgram
+                // nova-3 STT + Aura TTS streamed back as PCM.
+                sttProvider: useFlux ? 'deepgram' : 'local',
+                ttsProvider: useFlux ? 'deepgram' : 'local',
+            });
+            await session.streamAudioFile(uri, { skipWavHeader: true });
+            session.finalize();
+        }
+        catch (e) {
+            setVoiceState('idle');
+            session.close();
+            setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+        }
+    }, [client, project, model, runner, playTTS, voiceMode, fluxAvailable, speakLocalText]);
+    const handleVoicePress = (0, react_1.useCallback)(async () => {
+        if (voiceState === 'recording') {
+            void stopVoiceAndProcess();
+            return;
+        }
+        if (voiceState !== 'idle') {
+            // Mid-flow tap cancels.
+            voiceSessionRef.current?.close();
+            voiceSessionRef.current = null;
+            setVoiceState('idle');
+            return;
+        }
+        try {
+            await (0, capture_1.startPcmRecording)();
+            setVoiceState('recording');
+        }
+        catch (e) {
+            setTurns((prev) => [...prev, { id: `status-verr-${Date.now()}`, role: 'status', text: `voice: ${e instanceof Error ? e.message : String(e)}`, timestamp: Date.now() }]);
+        }
+    }, [voiceState, stopVoiceAndProcess]);
+    const voiceLabel = {
+        idle: '🎙 speak',
+        recording: '■ stop',
+        uploading: 'sending…',
+        thinking: 'thinking…',
+        speaking: 'speaking…',
+    };
     return (<react_native_1.View style={styles.container}>
       <react_native_1.View style={styles.header}>
         <react_native_1.Text style={styles.title}>Vibe</react_native_1.Text>
@@ -226,8 +402,30 @@ function VibeChatScreen({ client, initialTaskId, initialUserPrompt, onClose, onR
       </react_native_1.ScrollView>
 
       <react_native_1.View style={styles.footer}>
+        {voiceAvailable && voiceState !== 'idle' && (<react_native_1.Text style={styles.engineCaption}>
+            {voiceState === 'recording' ? 'listening' : voiceState === 'uploading' ? 'sending' : voiceState === 'thinking' ? 'agent working' : 'speaking'}
+            {activeEngine ? ` · ${activeEngine}` : ` · ${voiceMode === 'flux' ? 'Flux (Deepgram)' : 'Local (whisper)'}`}
+          </react_native_1.Text>)}
         <react_native_1.TextInput style={styles.input} value={followUp} onChangeText={setFollowUp} placeholder={status === 'running' ? 'wait for the agent…' : 'follow up…'} placeholderTextColor="#666" editable={status !== 'running' && !isResuming} multiline/>
         <react_native_1.View style={styles.actions}>
+          {voiceAvailable && (<>
+              {/* Local ↔ Flux engine toggle. Only shows Flux when the
+                agent has a Deepgram key; otherwise the label just
+                states "Local" so the active engine is always clear. */}
+              {fluxAvailable ? (<react_native_1.TouchableOpacity style={[styles.actionBtn, styles.engineToggle]} onPress={() => setVoiceMode((m) => (m === 'local' ? 'flux' : 'local'))} disabled={voiceState !== 'idle'} accessibilityLabel="Toggle voice engine">
+                  <react_native_1.Text style={styles.engineToggleText}>{voiceMode === 'flux' ? '⚡ Flux' : '🔒 Local'}</react_native_1.Text>
+                </react_native_1.TouchableOpacity>) : (<react_native_1.View style={[styles.actionBtn, styles.engineToggle]}>
+                  <react_native_1.Text style={styles.engineToggleText}>🔒 Local</react_native_1.Text>
+                </react_native_1.View>)}
+              <react_native_1.TouchableOpacity style={[
+                styles.actionBtn,
+                styles.voiceBtn,
+                voiceState === 'recording' && styles.voiceBtnActive,
+                (voiceState === 'uploading' || voiceState === 'thinking' || voiceState === 'speaking') && styles.actionBtnDisabled,
+            ]} onPress={handleVoicePress} disabled={voiceState === 'uploading' || voiceState === 'thinking' || voiceState === 'speaking'} accessibilityLabel="Vibe code by voice">
+                <react_native_1.Text style={styles.actionText}>{voiceLabel[voiceState]}</react_native_1.Text>
+              </react_native_1.TouchableOpacity>
+            </>)}
           {onReload && (<react_native_1.TouchableOpacity style={[
                 styles.actionBtn,
                 styles.reloadBtn,
@@ -323,6 +521,11 @@ const styles = react_native_1.StyleSheet.create({
     },
     actionBtnDisabled: { opacity: 0.5 },
     reloadBtn: { backgroundColor: 'rgba(255,255,255,0.08)' },
+    voiceBtn: { backgroundColor: 'rgba(16,185,129,0.18)' },
+    voiceBtnActive: { backgroundColor: '#ef4444' },
+    engineToggle: { backgroundColor: 'rgba(255,255,255,0.06)', marginRight: 'auto', marginLeft: 0 },
+    engineToggleText: { color: '#cbd5e1', fontSize: 12, fontWeight: '600' },
+    engineCaption: { color: '#9ca3af', fontSize: 11, marginBottom: 6, marginLeft: 4 },
     sendBtn: { backgroundColor: '#7582f5' },
     actionText: { color: '#fff', fontSize: 13, fontWeight: '600' },
 });

package/dist/capture.d.ts

@@ -88,3 +88,9 @@ export declare function stopAudioRecording(): Promise<{
 } | null>;
 /** Whether a voice-note recording is currently active. */
 export declare function isAudioRecording(): boolean;
+/** Begin a raw-PCM recording for the voice stream. */
+export declare function startPcmRecording(): Promise<void>;
+/** Stop the voice recording; returns the WAV file:// URI (or null). */
+export declare function stopPcmRecording(): Promise<string | null>;
+/** Whether a voice-stream recording is currently active. */
+export declare function isPcmRecording(): boolean;

package/dist/capture.js

@@ -25,6 +25,9 @@ exports.isVoiceCaptureSupported = isVoiceCaptureSupported;
 exports.startAudioRecording = startAudioRecording;
 exports.stopAudioRecording = stopAudioRecording;
 exports.isAudioRecording = isAudioRecording;
+exports.startPcmRecording = startPcmRecording;
+exports.stopPcmRecording = stopPcmRecording;
+exports.isPcmRecording = isPcmRecording;
 /**
  * Capture the current screen as a PNG image.
  * Requires `react-native-view-shot` to be installed.
@@ -307,3 +310,53 @@ async function stopAudioRecording() {
 function isAudioRecording() {
     return audioRecorderActive;
 }
+// ── Voice-stream recording (raw LPCM WAV) ──────────────────────────────
+// The voice vibe-coding path streams audio to the agent's STT WS, which
+// expects raw 16-bit / 16 kHz mono PCM (we strip the WAV header on the
+// way out). That's a different format from the HIGH_QUALITY m4a recorder
+// above — a compressed .m4a can't be streamed to Deepgram/whisper — so
+// this uses its own recording options, mirroring the Yaver app's
+// AgentVoiceButton.
+let pcmRecorderRef = null;
+let pcmRecorderActive = false;
+// Raw LPCM 16-bit LE, 16 kHz mono. iOS uses lpcm; Android records WAV.
+const PCM_RECORDING_OPTIONS = {
+    android: { extension: '.wav', outputFormat: 2, audioEncoder: 3, sampleRate: 16000, numberOfChannels: 1, bitRate: 256000 },
+    ios: {
+        extension: '.wav', outputFormat: 'lpcm', audioQuality: 0x40, sampleRate: 16000,
+        numberOfChannels: 1, bitRate: 256000, linearPCMBitDepth: 16, linearPCMIsBigEndian: false, linearPCMIsFloat: false,
+    },
+    web: { mimeType: 'audio/wav', bitsPerSecond: 256000 },
+};
+/** Begin a raw-PCM recording for the voice stream. */
+async function startPcmRecording() {
+    if (pcmRecorderActive)
+        throw new Error('[YaverFeedback] A voice recording is already in progress.');
+    const ExpoAv = loadExpoAvOrThrow();
+    const { Audio } = ExpoAv;
+    const perm = await Audio.requestPermissionsAsync();
+    if (!perm.granted) {
+        throw new Error('[YaverFeedback] Microphone permission denied. Enable it in Settings ▸ Your App ▸ Microphone.');
+    }
+    await Audio.setAudioModeAsync({ allowsRecordingIOS: true, playsInSilentModeIOS: true, staysActiveInBackground: false });
+    const { recording } = await Audio.Recording.createAsync(PCM_RECORDING_OPTIONS);
+    pcmRecorderRef = recording;
+    pcmRecorderActive = true;
+}
+/** Stop the voice recording; returns the WAV file:// URI (or null). */
+async function stopPcmRecording() {
+    if (!pcmRecorderActive || !pcmRecorderRef)
+        return null;
+    const recording = pcmRecorderRef;
+    pcmRecorderRef = null;
+    pcmRecorderActive = false;
+    try {
+        await recording.stopAndUnloadAsync();
+    }
+    catch { /* already stopped */ }
+    return typeof recording.getURI === 'function' ? recording.getURI() : null;
+}
+/** Whether a voice-stream recording is currently active. */
+function isPcmRecording() {
+    return pcmRecorderActive;
+}

package/dist/types.d.ts

@@ -19,6 +19,53 @@ export interface RunnerBrowserAuthSession {
     updatedAt: number;
     completedAt?: number;
 }
+export interface RunnerAuthStatusRow {
+    id: string;
+    name: string;
+    installed: boolean;
+    ready: boolean;
+    authConfigured: boolean;
+    authSource?: string;
+    warning?: string;
+    error?: string;
+    path?: string;
+    detail?: string;
+    version?: string;
+}
+export interface OpenCodeProviderSummary {
+    id: string;
+    name?: string;
+    hasApiKey?: boolean;
+    baseUrl?: string;
+    models?: Array<{
+        id: string;
+        name?: string;
+        provider?: string;
+    }>;
+}
+export interface OpenCodeAgentSummary {
+    name: string;
+    model?: string;
+    description?: string;
+    isBuiltin?: boolean;
+}
+export interface OpenCodeConfigSummary {
+    path: string;
+    exists: boolean;
+    defaultAgent?: string;
+    model?: string;
+    smallModel?: string;
+    buildModel?: string;
+    planModel?: string;
+    providers?: OpenCodeProviderSummary[];
+    models?: Array<{
+        id: string;
+        name?: string;
+        provider?: string;
+    }>;
+    agents?: OpenCodeAgentSummary[];
+    diagnostics?: string[];
+}
 export interface IncidentEvent {
     id: string;
     timestamp: number;
@@ -438,12 +485,20 @@ export interface TestSession {
 export interface VoiceCapability {
     /** Always true — mobile can always record and send audio. */
     voiceInputEnabled: boolean;
-    /** Speech-to-speech provider (e.g. "personaplex", "openai"), or null. */
+    /** Speech-to-speech provider (legacy), or null. */
     s2sProvider?: string;
     /** Whether the S2S provider is ready for real-time sessions. */
     s2sReady?: boolean;
-    /** Speech-to-text provider for transcription (e.g. "whisper", "openai"). */
+    /** Speech-to-text provider for transcription, e.g. "deepgram" for Deepgram Flux. */
     sttProvider?: string;
     /** Whether STT is ready (auto-transcription of voice input). */
     sttReady?: boolean;
+    /** Text-to-speech provider for readback, e.g. "cartesia". */
+    ttsProvider?: string;
+    /** Whether TTS readback is ready. */
+    ttsReady?: boolean;
+    /** Whether the agent-side hands-free task loop is enabled. */
+    enabled?: boolean;
+    /** Default project slug used by the agent voice loop. */
+    defaultProject?: string;
 }

package/dist/voice.d.ts

@@ -0,0 +1,61 @@
+export interface SDKVoiceStartOpts {
+    /** WS URL for the agent voice stream — P2PClient.voiceStreamUrl(). */
+    wsUrl: string;
+    /** Auth headers — P2PClient.voiceAuthHeaders(). */
+    headers: Record<string, string>;
+    project?: string;
+    model?: string;
+    runner?: string;
+    /** Surface hint for the agent's prompt wrapper. */
+    surface?: string;
+    /** Max chars for the spoken readback (Cartesia default ~280). */
+    ttsBudget?: number;
+    /** Per-session STT engine. "" = agent default. "local" = free
+     *  whisper.cpp on the host; "deepgram" = Flux nova-3 streaming. */
+    sttProvider?: string;
+    /** Per-session TTS engine. "" = agent default. "local"/"device" =
+     *  client synthesizes from the result text; cloud engines stream PCM. */
+    ttsProvider?: string;
+}
+export interface SDKVoiceCallbacks {
+    /** Active engines, echoed by the agent right after start — lets the UI
+     *  show "Local" vs "Flux". */
+    onProviders?: (stt: string, tts: string) => void;
+    onTranscriptPartial?: (text: string) => void;
+    onTranscriptFinal?: (text: string) => void;
+    onTaskCreated?: (taskId: string) => void;
+    onTaskResult?: (taskId: string, text: string, status: string) => void;
+    onTTSReady?: (pcm: Uint8Array, sampleRate: number) => void;
+    onError?: (msg: string) => void;
+    onDone?: () => void;
+}
+/** True when the deps the voice stream needs (expo-file-system + buffer)
+ *  are installed. The feedback UI hides the mic button otherwise. */
+export declare function isVoiceStreamSupported(): boolean;
+export declare class SDKVoiceSession {
+    private ws;
+    private callbacks;
+    private ttsChunks;
+    private ttsTotalBytes;
+    private ttsSampleRate;
+    private closed;
+    constructor(callbacks: SDKVoiceCallbacks);
+    /** Open the WS and send the start frame. Resolves once open. */
+    start(opts: SDKVoiceStartOpts): Promise<void>;
+    /** Stream a recorded WAV/PCM file as binary frames. Records produced by
+     *  recordPcmWav() are LPCM 16-bit/16kHz mono with a 44-byte RIFF header
+     *  we strip here — the exact shape the backend → STT expects. */
+    streamAudioFile(uri: string, opts?: {
+        skipWavHeader?: boolean;
+        chunkBytes?: number;
+    }): Promise<void>;
+    /** Done speaking — flush STT and create the agent task. */
+    finalize(): void;
+    close(): void;
+    private handleMessage;
+}
+/** Wrap raw PCM (signed 16-bit LE) in a minimal WAV container for
+ *  expo-av playback. 44-byte header + samples. */
+export declare function wrapPCMAsWAV(pcm: Uint8Array, sampleRate: number, channels?: number): Uint8Array;
+/** Write a PCM buffer to a temp WAV file; returns its file:// URI. */
+export declare function pcmToTempWavURI(pcm: Uint8Array, sampleRate: number): Promise<string>;