yaver-feedback-react-native 0.8.12 → 0.9.0

package/src/capture.ts

@@ -38,6 +38,37 @@ export async function captureScreenshot(): Promise<string> {
   }
 }
 
+/**
+ * Capture a screenshot AND return it as base64 + mime type, ready to
+ * embed in a `/tasks` payload's `images` array. Used by the converged
+ * vibe-feedback flow (FeedbackModal → P2PClient.createFeedbackTask).
+ *
+ * Returns null when capture isn't possible (peer dep missing / user
+ * permission denied / running in a context where view-shot can't
+ * grab the screen). Caller should treat null as "send without
+ * screenshot" rather than aborting the whole feedback.
+ */
+export async function captureScreenshotBase64(): Promise<{
+  base64: string;
+  mimeType: string;
+} | null> {
+  try {
+    const ViewShot = require('react-native-view-shot');
+    const result = await ViewShot.captureScreen({
+      format: 'jpg',
+      quality: 0.7,
+      result: 'base64',
+    });
+    if (typeof result === 'string' && result.length > 0) {
+      // ViewShot returns a bare base64 string (no `data:` prefix).
+      return { base64: result, mimeType: 'image/jpeg' };
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
 export interface PickedFeedbackFile {
   path: string;
   name: string;
@@ -305,3 +336,54 @@ export async function stopAudioRecording(): Promise<{ path: string; duration: nu
 export function isAudioRecording(): boolean {
   return audioRecorderActive;
 }
+
+// ── Voice-stream recording (raw LPCM WAV) ──────────────────────────────
+// The voice vibe-coding path streams audio to the agent's STT WS, which
+// expects raw 16-bit / 16 kHz mono PCM (we strip the WAV header on the
+// way out). That's a different format from the HIGH_QUALITY m4a recorder
+// above — a compressed .m4a can't be streamed to Deepgram/whisper — so
+// this uses its own recording options, mirroring the Yaver app's
+// AgentVoiceButton.
+
+let pcmRecorderRef: any = null;
+let pcmRecorderActive = false;
+
+// Raw LPCM 16-bit LE, 16 kHz mono. iOS uses lpcm; Android records WAV.
+const PCM_RECORDING_OPTIONS: any = {
+  android: { extension: '.wav', outputFormat: 2, audioEncoder: 3, sampleRate: 16000, numberOfChannels: 1, bitRate: 256000 },
+  ios: {
+    extension: '.wav', outputFormat: 'lpcm', audioQuality: 0x40, sampleRate: 16000,
+    numberOfChannels: 1, bitRate: 256000, linearPCMBitDepth: 16, linearPCMIsBigEndian: false, linearPCMIsFloat: false,
+  },
+  web: { mimeType: 'audio/wav', bitsPerSecond: 256000 },
+};
+
+/** Begin a raw-PCM recording for the voice stream. */
+export async function startPcmRecording(): Promise<void> {
+  if (pcmRecorderActive) throw new Error('[YaverFeedback] A voice recording is already in progress.');
+  const ExpoAv = loadExpoAvOrThrow();
+  const { Audio } = ExpoAv;
+  const perm = await Audio.requestPermissionsAsync();
+  if (!perm.granted) {
+    throw new Error('[YaverFeedback] Microphone permission denied. Enable it in Settings ▸ Your App ▸ Microphone.');
+  }
+  await Audio.setAudioModeAsync({ allowsRecordingIOS: true, playsInSilentModeIOS: true, staysActiveInBackground: false });
+  const { recording } = await Audio.Recording.createAsync(PCM_RECORDING_OPTIONS);
+  pcmRecorderRef = recording;
+  pcmRecorderActive = true;
+}
+
+/** Stop the voice recording; returns the WAV file:// URI (or null). */
+export async function stopPcmRecording(): Promise<string | null> {
+  if (!pcmRecorderActive || !pcmRecorderRef) return null;
+  const recording = pcmRecorderRef;
+  pcmRecorderRef = null;
+  pcmRecorderActive = false;
+  try { await recording.stopAndUnloadAsync(); } catch { /* already stopped */ }
+  return typeof recording.getURI === 'function' ? recording.getURI() : null;
+}
+
+/** Whether a voice-stream recording is currently active. */
+export function isPcmRecording(): boolean {
+  return pcmRecorderActive;
+}

package/src/preferences.ts

@@ -149,3 +149,59 @@ export async function setQuickIconColorPreset(
 export async function clearQuickIconColorPreset(): Promise<void> {
   await setQuickIconColorPreset(null);
 }
+
+// ── Preferred coding agent + model (used by the standalone feedback
+// SDK's vibe chat to mirror what Yaver mobile's Tasks tab would send.
+// The agent on the remote DOES read userSettings.primaryRunnerByDevice
+// from Convex, but the standalone SDK has no DeviceContext to push the
+// per-device pick. We persist the user's last choice locally; first
+// run picks whatever's signed-in via getRunnerStatus().)
+
+const PREFERRED_RUNNER_KEY = 'yaver_feedback_preferred_runner';
+const PREFERRED_MODEL_KEY = 'yaver_feedback_preferred_model';
+
+export async function getPreferredRunner(): Promise<string | null> {
+  if (!AsyncStorage) return null;
+  try {
+    const v = await AsyncStorage.getItem(PREFERRED_RUNNER_KEY);
+    return v && v.trim() ? v.trim() : null;
+  } catch {
+    return null;
+  }
+}
+
+export async function setPreferredRunner(runner: string | null): Promise<void> {
+  if (!AsyncStorage) return;
+  try {
+    if (!runner || !runner.trim()) {
+      await AsyncStorage.removeItem(PREFERRED_RUNNER_KEY);
+      return;
+    }
+    await AsyncStorage.setItem(PREFERRED_RUNNER_KEY, runner.trim());
+  } catch {
+    /* best-effort */
+  }
+}
+
+export async function getPreferredModel(): Promise<string | null> {
+  if (!AsyncStorage) return null;
+  try {
+    const v = await AsyncStorage.getItem(PREFERRED_MODEL_KEY);
+    return v && v.trim() ? v.trim() : null;
+  } catch {
+    return null;
+  }
+}
+
+export async function setPreferredModel(model: string | null): Promise<void> {
+  if (!AsyncStorage) return;
+  try {
+    if (!model || !model.trim()) {
+      await AsyncStorage.removeItem(PREFERRED_MODEL_KEY);
+      return;
+    }
+    await AsyncStorage.setItem(PREFERRED_MODEL_KEY, model.trim());
+  } catch {
+    /* best-effort */
+  }
+}

package/src/types.ts

@@ -20,6 +20,49 @@ export interface RunnerBrowserAuthSession {
   completedAt?: number;
 }
 
+export interface RunnerAuthStatusRow {
+  id: string;
+  name: string;
+  installed: boolean;
+  ready: boolean;
+  authConfigured: boolean;
+  authSource?: string;
+  warning?: string;
+  error?: string;
+  path?: string;
+  detail?: string;
+  version?: string;
+}
+
+export interface OpenCodeProviderSummary {
+  id: string;
+  name?: string;
+  hasApiKey?: boolean;
+  baseUrl?: string;
+  models?: Array<{ id: string; name?: string; provider?: string }>;
+}
+
+export interface OpenCodeAgentSummary {
+  name: string;
+  model?: string;
+  description?: string;
+  isBuiltin?: boolean;
+}
+
+export interface OpenCodeConfigSummary {
+  path: string;
+  exists: boolean;
+  defaultAgent?: string;
+  model?: string;
+  smallModel?: string;
+  buildModel?: string;
+  planModel?: string;
+  providers?: OpenCodeProviderSummary[];
+  models?: Array<{ id: string; name?: string; provider?: string }>;
+  agents?: OpenCodeAgentSummary[];
+  diagnostics?: string[];
+}
+
 export interface IncidentEvent {
   id: string;
   timestamp: number;
@@ -123,6 +166,15 @@ export interface FeedbackConfig {
   preferredDeviceId?: string;
   /** How feedback collection is triggered */
   trigger?: 'shake' | 'floating-button' | 'manual';
+  /**
+   * App slug used by the in-modal Deploy panel when calling the agent's
+   * `/fleet/deploy-options` and `/deploy/ship` endpoints. Should match an
+   * `apps[].name` entry in the agent's `yaver.workspace.yaml`. When omitted
+   * the panel falls back to the last dot-segment of `bundleId` (e.g.
+   * `io.yaver.sfmg` → `sfmg`). Set explicitly when the workspace name
+   * differs from the bundleId tail.
+   */
+  deployAppSlug?: string;
   /**
    * Non-default escape hatch for host apps that want the SDK without
    * shake gesture handling. When enabled:
@@ -442,12 +494,20 @@ export interface TestSession {
 export interface VoiceCapability {
   /** Always true — mobile can always record and send audio. */
   voiceInputEnabled: boolean;
-  /** Speech-to-speech provider (e.g. "personaplex", "openai"), or null. */
+  /** Speech-to-speech provider (legacy), or null. */
   s2sProvider?: string;
   /** Whether the S2S provider is ready for real-time sessions. */
   s2sReady?: boolean;
-  /** Speech-to-text provider for transcription (e.g. "whisper", "openai"). */
+  /** Speech-to-text provider for transcription, e.g. "deepgram" for Deepgram Flux. */
   sttProvider?: string;
   /** Whether STT is ready (auto-transcription of voice input). */
   sttReady?: boolean;
+  /** Text-to-speech provider for readback, e.g. "cartesia". */
+  ttsProvider?: string;
+  /** Whether TTS readback is ready. */
+  ttsReady?: boolean;
+  /** Whether the agent-side hands-free task loop is enabled. */
+  enabled?: boolean;
+  /** Default project slug used by the agent voice loop. */
+  defaultProject?: string;
 }

package/src/voice.ts

@@ -0,0 +1,270 @@
+// Voice vibe-coding for the feedback SDK — speak a change, the agent
+// acts (full MCP), and reads back a spoken headline. Ported from the
+// Yaver mobile app's AgentVoiceSession (mobile/src/lib/agentVoice.ts) but
+// decoupled from the app: connection details (WS URL + auth headers) come
+// from the SDK's P2PClient, so it works both inside the Yaver container
+// (inherited auth) and standalone in a third-party app.
+//
+// Local vs remote STT/TTS is decided entirely agent-side (free local
+// whisper.cpp by default, Deepgram/OpenAI/etc when keys are configured),
+// so nothing model-related is bundled here — the SDK just streams mic
+// audio and renders the transcript + TTS the agent sends back.
+//
+// expo-file-system + buffer are loaded lazily so a bare-RN app that
+// hasn't installed them can still use the rest of the SDK; the voice
+// button hides itself via isVoiceStreamSupported() when they're absent.
+
+export interface SDKVoiceStartOpts {
+  /** WS URL for the agent voice stream — P2PClient.voiceStreamUrl(). */
+  wsUrl: string;
+  /** Auth headers — P2PClient.voiceAuthHeaders(). */
+  headers: Record<string, string>;
+  project?: string;
+  model?: string;
+  runner?: string;
+  /** Surface hint for the agent's prompt wrapper. */
+  surface?: string;
+  /** Max chars for the spoken readback (Cartesia default ~280). */
+  ttsBudget?: number;
+  /** Per-session STT engine. "" = agent default. "local" = free
+   *  whisper.cpp on the host; "deepgram" = Flux nova-3 streaming. */
+  sttProvider?: string;
+  /** Per-session TTS engine. "" = agent default. "local"/"device" =
+   *  client synthesizes from the result text; cloud engines stream PCM. */
+  ttsProvider?: string;
+}
+
+export interface SDKVoiceCallbacks {
+  /** Active engines, echoed by the agent right after start — lets the UI
+   *  show "Local" vs "Flux". */
+  onProviders?: (stt: string, tts: string) => void;
+  onTranscriptPartial?: (text: string) => void;
+  onTranscriptFinal?: (text: string) => void;
+  onTaskCreated?: (taskId: string) => void;
+  onTaskResult?: (taskId: string, text: string, status: string) => void;
+  onTTSReady?: (pcm: Uint8Array, sampleRate: number) => void;
+  onError?: (msg: string) => void;
+  onDone?: () => void;
+}
+
+interface WireMsg {
+  type: string;
+  text?: string;
+  taskId?: string;
+  status?: string;
+  pcm?: string;
+  sampleRate?: number;
+  error?: string;
+  stt?: string;
+  tts?: string;
+}
+
+function loadBuffer(): any {
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  return require('buffer').Buffer;
+}
+
+function loadFS(): any {
+  // expo-file-system v16+ moved the classic API under /legacy; fall back
+  // to the root export for older installs.
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    return require('expo-file-system/legacy');
+  } catch {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    return require('expo-file-system');
+  }
+}
+
+/** True when the deps the voice stream needs (expo-file-system + buffer)
+ *  are installed. The feedback UI hides the mic button otherwise. */
+export function isVoiceStreamSupported(): boolean {
+  try {
+    loadBuffer();
+    const fs = loadFS();
+    return typeof fs?.readAsStringAsync === 'function';
+  } catch {
+    return false;
+  }
+}
+
+export class SDKVoiceSession {
+  private ws: WebSocket | null = null;
+  private callbacks: SDKVoiceCallbacks;
+  private ttsChunks: Uint8Array[] = [];
+  private ttsTotalBytes = 0;
+  private ttsSampleRate = 22050;
+  private closed = false;
+
+  constructor(callbacks: SDKVoiceCallbacks) {
+    this.callbacks = callbacks;
+  }
+
+  /** Open the WS and send the start frame. Resolves once open. */
+  async start(opts: SDKVoiceStartOpts): Promise<void> {
+    return new Promise((resolve, reject) => {
+      let opened = false;
+      let ws: WebSocket;
+      try {
+        // RN-specific 3rd arg carries request headers.
+        ws = new (WebSocket as any)(opts.wsUrl, undefined, { headers: opts.headers });
+      } catch (e) {
+        reject(e);
+        return;
+      }
+      this.ws = ws;
+      ws.onopen = () => {
+        opened = true;
+        try {
+          ws.send(
+            JSON.stringify({
+              type: 'start',
+              project: opts.project ?? '',
+              model: opts.model ?? '',
+              runner: opts.runner ?? '',
+              surface: opts.surface ?? '',
+              ttsBudget: opts.ttsBudget ?? 0,
+              sttProvider: opts.sttProvider ?? '',
+              ttsProvider: opts.ttsProvider ?? '',
+            }),
+          );
+          resolve();
+        } catch (e) {
+          reject(e);
+        }
+      };
+      ws.onmessage = (e: any) => this.handleMessage(e);
+      ws.onerror = () => {
+        if (!opened) reject(new Error('voice WS connect failed'));
+        else this.callbacks.onError?.('voice WS error');
+      };
+      ws.onclose = () => {
+        this.closed = true;
+        if (!opened) reject(new Error('voice WS closed before open'));
+      };
+    });
+  }
+
+  /** Stream a recorded WAV/PCM file as binary frames. Records produced by
+   *  recordPcmWav() are LPCM 16-bit/16kHz mono with a 44-byte RIFF header
+   *  we strip here — the exact shape the backend → STT expects. */
+  async streamAudioFile(uri: string, opts?: { skipWavHeader?: boolean; chunkBytes?: number }): Promise<void> {
+    if (!this.ws || this.closed) throw new Error('voice WS not open');
+    const Buffer = loadBuffer();
+    const FileSystem = loadFS();
+    const skipWavHeader = opts?.skipWavHeader ?? true;
+    const chunkBytes = opts?.chunkBytes ?? 16384;
+
+    const b64 = await FileSystem.readAsStringAsync(uri, { encoding: 'base64' });
+    let buf = Buffer.from(b64, 'base64');
+    if (skipWavHeader && buf.length > 44 && buf.slice(0, 4).toString() === 'RIFF') {
+      buf = buf.slice(44);
+    }
+    for (let i = 0; i < buf.length; i += chunkBytes) {
+      if (this.closed) return;
+      const slice = buf.slice(i, Math.min(i + chunkBytes, buf.length));
+      this.ws.send(slice.buffer.slice(slice.byteOffset, slice.byteOffset + slice.byteLength));
+      await new Promise((r) => setTimeout(r, 0));
+    }
+  }
+
+  /** Done speaking — flush STT and create the agent task. */
+  finalize(): void {
+    if (this.ws && !this.closed) {
+      this.ws.send(JSON.stringify({ type: 'stop' }));
+    }
+  }
+
+  close(): void {
+    this.closed = true;
+    try { this.ws?.close(); } catch { /* ignore */ }
+    this.ws = null;
+  }
+
+  private handleMessage(e: { data: any }): void {
+    let msg: WireMsg;
+    try {
+      msg = JSON.parse(typeof e.data === 'string' ? e.data : '');
+    } catch {
+      return;
+    }
+    switch (msg.type) {
+      case 'providers':
+        this.callbacks.onProviders?.(msg.stt ?? '', msg.tts ?? '');
+        break;
+      case 'transcript-partial':
+        this.callbacks.onTranscriptPartial?.(msg.text ?? '');
+        break;
+      case 'transcript-final':
+        this.callbacks.onTranscriptFinal?.(msg.text ?? '');
+        break;
+      case 'task-created':
+        this.callbacks.onTaskCreated?.(msg.taskId ?? '');
+        break;
+      case 'task-result':
+        this.callbacks.onTaskResult?.(msg.taskId ?? '', msg.text ?? '', msg.status ?? '');
+        break;
+      case 'tts-frame': {
+        if (!msg.pcm) break;
+        const Buffer = loadBuffer();
+        const arr = new Uint8Array(Buffer.from(msg.pcm, 'base64'));
+        this.ttsChunks.push(arr);
+        this.ttsTotalBytes += arr.length;
+        if (msg.sampleRate) this.ttsSampleRate = msg.sampleRate;
+        break;
+      }
+      case 'done': {
+        if (this.ttsTotalBytes > 0) {
+          const pcm = new Uint8Array(this.ttsTotalBytes);
+          let off = 0;
+          for (const c of this.ttsChunks) { pcm.set(c, off); off += c.length; }
+          this.callbacks.onTTSReady?.(pcm, this.ttsSampleRate);
+        }
+        this.callbacks.onDone?.();
+        this.close();
+        break;
+      }
+      case 'error':
+        this.callbacks.onError?.(msg.error ?? 'voice error');
+        break;
+    }
+  }
+}
+
+/** Wrap raw PCM (signed 16-bit LE) in a minimal WAV container for
+ *  expo-av playback. 44-byte header + samples. */
+export function wrapPCMAsWAV(pcm: Uint8Array, sampleRate: number, channels = 1): Uint8Array {
+  const bitsPerSample = 16;
+  const byteRate = (sampleRate * channels * bitsPerSample) / 8;
+  const blockAlign = (channels * bitsPerSample) / 8;
+  const dataSize = pcm.length;
+  const buf = new Uint8Array(44 + dataSize);
+  const dv = new DataView(buf.buffer);
+  buf[0] = 0x52; buf[1] = 0x49; buf[2] = 0x46; buf[3] = 0x46; // RIFF
+  dv.setUint32(4, 36 + dataSize, true);
+  buf[8] = 0x57; buf[9] = 0x41; buf[10] = 0x56; buf[11] = 0x45; // WAVE
+  buf[12] = 0x66; buf[13] = 0x6d; buf[14] = 0x74; buf[15] = 0x20; // "fmt "
+  dv.setUint32(16, 16, true);
+  dv.setUint16(20, 1, true);
+  dv.setUint16(22, channels, true);
+  dv.setUint32(24, sampleRate, true);
+  dv.setUint32(28, byteRate, true);
+  dv.setUint16(32, blockAlign, true);
+  dv.setUint16(34, bitsPerSample, true);
+  buf[36] = 0x64; buf[37] = 0x61; buf[38] = 0x74; buf[39] = 0x61; // data
+  dv.setUint32(40, dataSize, true);
+  buf.set(pcm, 44);
+  return buf;
+}
+
+/** Write a PCM buffer to a temp WAV file; returns its file:// URI. */
+export async function pcmToTempWavURI(pcm: Uint8Array, sampleRate: number): Promise<string> {
+  const Buffer = loadBuffer();
+  const FileSystem = loadFS();
+  const wav = wrapPCMAsWAV(pcm, sampleRate);
+  const b64 = Buffer.from(wav).toString('base64');
+  const dir = FileSystem.cacheDirectory ?? FileSystem.documentDirectory ?? '';
+  const path = `${dir}yaver-voice-tts-${Date.now()}.wav`;
+  await FileSystem.writeAsStringAsync(path, b64, { encoding: 'base64' });
+  return path;
+}