zerg-ztc 0.1.10 → 0.1.11

package/src/utils/dictation.ts

@@ -0,0 +1,467 @@
+/**
+ * Audio dictation with push-to-talk support
+ * Automatic provider fallback:
+ * 1. OpenAI Whisper API (if OPENAI_API_KEY available)
+ * 2. Local whisper (if installed)
+ * 3. macOS native (SFSpeechRecognizer)
+ */
+
+import { spawn, execSync, spawnSync, ChildProcess } from 'child_process';
+import { writeFile, readFile, unlink, mkdir } from 'fs/promises';
+import { existsSync } from 'fs';
+import { join } from 'path';
+import { tmpdir, homedir, platform } from 'os';
+
+export type DictationProvider = 'openai' | 'local' | 'macos';
+
+export interface DictationResult {
+  text: string;
+  duration?: number;
+  provider: DictationProvider;
+}
+
+const RECORDING_DIR = join(tmpdir(), 'ztc-dictation');
+const RECORDING_FILE = join(RECORDING_DIR, 'recording.wav');
+
+// Global state for push-to-talk
+let activeRecording: {
+  process: ChildProcess;
+  startTime: number;
+} | null = null;
+
+let cachedRecordingTool: 'sox' | 'ffmpeg' | 'arecord' | null = null;
+let cachedProvider: DictationProvider | null = null;
+
+/**
+ * Check for available recording tool (cached)
+ */
+function getRecordingTool(): 'sox' | 'ffmpeg' | 'arecord' | null {
+  if (cachedRecordingTool !== null) return cachedRecordingTool;
+
+  const os = platform();
+
+  if (os === 'darwin' || os === 'linux') {
+    try {
+      execSync('which rec', { stdio: 'ignore' });
+      cachedRecordingTool = 'sox';
+      return 'sox';
+    } catch {}
+
+    try {
+      execSync('which ffmpeg', { stdio: 'ignore' });
+      cachedRecordingTool = 'ffmpeg';
+      return 'ffmpeg';
+    } catch {}
+  }
+
+  if (os === 'linux') {
+    try {
+      execSync('which arecord', { stdio: 'ignore' });
+      cachedRecordingTool = 'arecord';
+      return 'arecord';
+    } catch {}
+  }
+
+  return null;
+}
+
+/**
+ * Get the best available transcription provider (cached)
+ */
+export function getBestProvider(): DictationProvider | null {
+  if (cachedProvider !== null) return cachedProvider;
+
+  // Check OpenAI first (best quality)
+  if (process.env.OPENAI_API_KEY) {
+    cachedProvider = 'openai';
+    return 'openai';
+  }
+
+  // Check local whisper
+  try {
+    execSync('which whisper', { stdio: 'ignore' });
+    cachedProvider = 'local';
+    return 'local';
+  } catch {}
+
+  try {
+    execSync('which whisper-cpp', { stdio: 'ignore' });
+    cachedProvider = 'local';
+    return 'local';
+  } catch {}
+
+  // macOS native as fallback
+  if (platform() === 'darwin') {
+    cachedProvider = 'macos';
+    return 'macos';
+  }
+
+  return null;
+}
+
+/**
+ * Check if dictation is available
+ */
+export function isDictationAvailable(): boolean {
+  return getRecordingTool() !== null && getBestProvider() !== null;
+}
+
+/**
+ * Check if currently recording
+ */
+export function isRecording(): boolean {
+  return activeRecording !== null;
+}
+
+/**
+ * Get the name of the recording device/tool being used
+ */
+export function getRecordingDeviceName(): string | null {
+  const tool = getRecordingTool();
+  if (!tool) return null;
+
+  const os = platform();
+
+  if (tool === 'sox') {
+    // On macOS, sox uses coreaudio
+    if (os === 'darwin') {
+      return 'Default Input (sox)';
+    }
+    return 'Default Input (sox)';
+  }
+
+  if (tool === 'ffmpeg') {
+    if (os === 'darwin') {
+      return 'Default Input (avfoundation)';
+    }
+    return 'Default Input (alsa)';
+  }
+
+  if (tool === 'arecord') {
+    return 'Default Input (alsa)';
+  }
+
+  return tool;
+}
+
+/**
+ * Start recording (push-to-talk: press to start)
+ */
+export function startRecording(): boolean {
+  if (activeRecording) {
+    return false; // Already recording
+  }
+
+  const tool = getRecordingTool();
+  if (!tool) {
+    throw new Error('No recording tool available. Install sox: brew install sox');
+  }
+
+  // Ensure directory exists
+  if (!existsSync(RECORDING_DIR)) {
+    execSync(`mkdir -p "${RECORDING_DIR}"`);
+  }
+
+  const os = platform();
+  let proc: ChildProcess;
+
+  if (tool === 'sox') {
+    proc = spawn('rec', [
+      '-q',           // Quiet
+      '-r', '16000',  // 16kHz (Whisper optimal)
+      '-c', '1',      // Mono
+      '-b', '16',     // 16-bit
+      RECORDING_FILE
+    ], { stdio: 'ignore' });
+  } else if (tool === 'ffmpeg') {
+    if (os === 'darwin') {
+      proc = spawn('ffmpeg', [
+        '-y', '-f', 'avfoundation', '-i', ':0',
+        '-ar', '16000', '-ac', '1', '-sample_fmt', 's16',
+        RECORDING_FILE
+      ], { stdio: 'ignore' });
+    } else {
+      proc = spawn('ffmpeg', [
+        '-y', '-f', 'alsa', '-i', 'default',
+        '-ar', '16000', '-ac', '1', '-sample_fmt', 's16',
+        RECORDING_FILE
+      ], { stdio: 'ignore' });
+    }
+  } else {
+    proc = spawn('arecord', [
+      '-f', 'S16_LE', '-r', '16000', '-c', '1',
+      RECORDING_FILE
+    ], { stdio: 'ignore' });
+  }
+
+  activeRecording = {
+    process: proc,
+    startTime: Date.now()
+  };
+
+  return true;
+}
+
+/**
+ * Stop recording and transcribe (push-to-talk: release to stop)
+ */
+export async function stopRecordingAndTranscribe(): Promise<DictationResult> {
+  if (!activeRecording) {
+    throw new Error('Not currently recording');
+  }
+
+  const { process: proc, startTime } = activeRecording;
+  activeRecording = null;
+
+  // Stop the recording process
+  await new Promise<void>((resolve) => {
+    proc.on('close', () => resolve());
+    proc.on('error', () => resolve());
+    proc.kill('SIGINT');
+    // Fallback timeout
+    setTimeout(() => resolve(), 1000);
+  });
+
+  // Small delay to ensure file is written
+  await new Promise(resolve => setTimeout(resolve, 100));
+
+  if (!existsSync(RECORDING_FILE)) {
+    throw new Error('Recording file not created');
+  }
+
+  const recordingDuration = Date.now() - startTime;
+  const provider = getBestProvider();
+
+  if (!provider) {
+    throw new Error('No transcription provider available');
+  }
+
+  // Transcribe
+  const transcribeStart = Date.now();
+  let text: string;
+
+  try {
+    text = await transcribe(RECORDING_FILE, provider);
+  } finally {
+    // Clean up recording file
+    try {
+      await unlink(RECORDING_FILE);
+    } catch {}
+  }
+
+  return {
+    text: text.trim(),
+    duration: Date.now() - transcribeStart,
+    provider
+  };
+}
+
+/**
+ * Cancel active recording without transcribing
+ */
+export function cancelRecording(): void {
+  if (activeRecording) {
+    activeRecording.process.kill('SIGKILL');
+    activeRecording = null;
+    // Clean up file
+    try {
+      if (existsSync(RECORDING_FILE)) {
+        execSync(`rm -f "${RECORDING_FILE}"`);
+      }
+    } catch {}
+  }
+}
+
+/**
+ * Transcribe audio file
+ */
+async function transcribe(audioPath: string, provider: DictationProvider): Promise<string> {
+  switch (provider) {
+    case 'openai':
+      return transcribeOpenAI(audioPath);
+    case 'local':
+      return transcribeLocal(audioPath);
+    case 'macos':
+      return transcribeMacOS(audioPath);
+    default:
+      throw new Error(`Unknown provider: ${provider}`);
+  }
+}
+
+/**
+ * Transcribe using OpenAI Whisper API
+ */
+async function transcribeOpenAI(audioPath: string): Promise<string> {
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) {
+    throw new Error('OPENAI_API_KEY not set');
+  }
+
+  const audioData = await readFile(audioPath);
+
+  const formData = new FormData();
+  formData.append('file', new Blob([audioData], { type: 'audio/wav' }), 'audio.wav');
+  formData.append('model', 'whisper-1');
+
+  const response = await fetch('https://api.openai.com/v1/audio/transcriptions', {
+    method: 'POST',
+    headers: { 'Authorization': `Bearer ${apiKey}` },
+    body: formData
+  });
+
+  if (!response.ok) {
+    const error = await response.text();
+    throw new Error(`Whisper API error: ${error}`);
+  }
+
+  const result = await response.json() as { text: string };
+  return result.text;
+}
+
+/**
+ * Transcribe using local whisper
+ */
+async function transcribeLocal(audioPath: string): Promise<string> {
+  // Try Python whisper first
+  try {
+    execSync('which whisper', { stdio: 'ignore' });
+
+    const result = spawnSync('whisper', [
+      audioPath,
+      '--model', 'base',
+      '--output_format', 'txt',
+      '--output_dir', RECORDING_DIR
+    ], { encoding: 'utf-8', timeout: 60000 });
+
+    if (result.status === 0) {
+      const outputFile = audioPath.replace(/\.wav$/, '.txt');
+      if (existsSync(outputFile)) {
+        const text = await readFile(outputFile, 'utf-8');
+        await unlink(outputFile).catch(() => {});
+        return text.trim();
+      }
+    }
+  } catch {}
+
+  // Try whisper.cpp
+  try {
+    execSync('which whisper-cpp', { stdio: 'ignore' });
+
+    const modelPath = join(homedir(), '.local/share/whisper/ggml-base.bin');
+    const result = spawnSync('whisper-cpp', [
+      '-m', modelPath,
+      '-f', audioPath
+    ], { encoding: 'utf-8', timeout: 60000 });
+
+    if (result.status === 0) {
+      return result.stdout.trim();
+    }
+  } catch {}
+
+  throw new Error('Local whisper transcription failed');
+}
+
+/**
+ * Transcribe using macOS native Speech Recognition
+ */
+async function transcribeMacOS(audioPath: string): Promise<string> {
+  const swiftCode = `
+import Foundation
+import Speech
+
+let semaphore = DispatchSemaphore(value: 0)
+var transcription = ""
+var errorMsg = ""
+
+SFSpeechRecognizer.requestAuthorization { status in
+    guard status == .authorized else {
+        errorMsg = "Speech recognition not authorized"
+        semaphore.signal()
+        return
+    }
+
+    guard let recognizer = SFSpeechRecognizer(locale: Locale(identifier: "en-US")),
+          recognizer.isAvailable else {
+        errorMsg = "Speech recognizer not available"
+        semaphore.signal()
+        return
+    }
+
+    let url = URL(fileURLWithPath: CommandLine.arguments[1])
+    let request = SFSpeechURLRecognitionRequest(url: url)
+    request.shouldReportPartialResults = false
+
+    recognizer.recognitionTask(with: request) { result, error in
+        if let error = error {
+            errorMsg = error.localizedDescription
+            semaphore.signal()
+            return
+        }
+
+        if let result = result, result.isFinal {
+            transcription = result.bestTranscription.formattedString
+            semaphore.signal()
+        }
+    }
+}
+
+_ = semaphore.wait(timeout: .now() + 30)
+
+if !errorMsg.isEmpty {
+    fputs(errorMsg + "\\n", stderr)
+    exit(1)
+}
+print(transcription)
+`;
+
+  const swiftFile = join(RECORDING_DIR, 'transcribe.swift');
+  await writeFile(swiftFile, swiftCode);
+
+  try {
+    const result = spawnSync('swift', [swiftFile, audioPath], {
+      encoding: 'utf-8',
+      timeout: 35000
+    });
+
+    if (result.status !== 0) {
+      throw new Error(result.stderr || 'macOS transcription failed');
+    }
+
+    return result.stdout.trim();
+  } finally {
+    await unlink(swiftFile).catch(() => {});
+  }
+}
+
+/**
+ * Get status message for dictation availability
+ */
+export function getDictationStatus(): string {
+  const tool = getRecordingTool();
+  const provider = getBestProvider();
+
+  const parts: string[] = [];
+
+  if (tool) {
+    parts.push(`Recording: ${tool}`);
+  } else {
+    parts.push('Recording: not available (install sox: brew install sox)');
+  }
+
+  if (provider) {
+    const providerNames: Record<DictationProvider, string> = {
+      openai: 'OpenAI Whisper',
+      local: 'Local Whisper',
+      macos: 'macOS Native'
+    };
+    parts.push(`Transcription: ${providerNames[provider]}`);
+  } else {
+    parts.push('Transcription: not available');
+  }
+
+  if (tool && provider) {
+    parts.push('\nPress Ctrl+T to start/stop recording');
+  }
+
+  return parts.join('\n');
+}

package/src/utils/dictation_native.ts

@@ -0,0 +1,258 @@
+/**
+ * Native real-time dictation using ztc-audio Rust binary
+ * Provides:
+ * - Real-time audio level metering (VU meter)
+ * - Real-time transcription via whisper
+ * - Cross-platform support
+ */
+
+import { spawn, ChildProcess } from 'child_process';
+import { existsSync } from 'fs';
+import { createInterface } from 'readline';
+import { join, dirname } from 'path';
+import { homedir, platform, arch } from 'os';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+// Try to import the @zergai/ztc-dictation package
+let dictationPackage: { getBinaryPath: () => string | null; isAvailable: () => boolean } | null = null;
+try {
+  // @ts-ignore - dynamic import of optional package
+  dictationPackage = await import('@zergai/ztc-dictation');
+} catch {
+  // Package not installed, will fall back to local binary
+}
+
+export interface DictationEvent {
+  type: 'ready' | 'level' | 'text' | 'error' | 'stopped';
+  device?: string;
+  model?: string;
+  db?: number;
+  rms?: number;
+  content?: string;
+  partial?: boolean;
+  message?: string;
+}
+
+export type DictationEventHandler = (event: DictationEvent) => void;
+
+let activeProcess: ChildProcess | null = null;
+let eventHandlers: DictationEventHandler[] = [];
+
+/**
+ * Get the path to the ztc-audio binary
+ */
+function getBinaryPath(): string | null {
+  // 1. Try the @zergai/ztc-dictation package first
+  if (dictationPackage) {
+    const pkgPath = dictationPackage.getBinaryPath();
+    if (pkgPath && existsSync(pkgPath)) {
+      return pkgPath;
+    }
+  }
+
+  // 2. Check local development/install paths
+  const possiblePaths = [
+    // Development (new location)
+    join(process.cwd(), 'packages', 'ztc-dictation', 'target', 'release', 'ztc-audio'),
+    // Development (old location)
+    join(process.cwd(), 'native', 'ztc-audio', 'target', 'release', 'ztc-audio'),
+    // Global install
+    join(homedir(), '.ztc', 'bin', 'ztc-audio'),
+  ];
+
+  for (const p of possiblePaths) {
+    if (existsSync(p)) {
+      return p;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Check if native dictation is available
+ */
+export function isNativeDictationAvailable(): boolean {
+  return getBinaryPath() !== null;
+}
+
+/**
+ * Check if currently recording
+ */
+export function isNativeRecording(): boolean {
+  return activeProcess !== null;
+}
+
+/**
+ * Subscribe to dictation events
+ */
+export function onDictationEvent(handler: DictationEventHandler): () => void {
+  eventHandlers.push(handler);
+  return () => {
+    eventHandlers = eventHandlers.filter(h => h !== handler);
+  };
+}
+
+function emit(event: DictationEvent) {
+  for (const handler of eventHandlers) {
+    try {
+      handler(event);
+    } catch (e) {
+      // Ignore handler errors
+    }
+  }
+}
+
+/**
+ * Start native recording with real-time transcription
+ */
+export function startNativeRecording(options: {
+  model?: string;
+  device?: string;
+} = {}): boolean {
+  if (activeProcess) {
+    return false; // Already recording
+  }
+
+  const binaryPath = getBinaryPath();
+  if (!binaryPath) {
+    emit({
+      type: 'error',
+      message: 'ztc-audio binary not found. Run: cargo build --release in native/ztc-audio'
+    });
+    return false;
+  }
+
+  const args: string[] = [];
+  if (options.model) {
+    args.push('--model', options.model);
+  } else {
+    args.push('--model', 'tiny'); // Default to tiny for speed
+  }
+  if (options.device) {
+    args.push('--device', options.device);
+  }
+
+  try {
+    activeProcess = spawn(binaryPath, args, {
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    // Parse JSON lines from stdout
+    const rl = createInterface({
+      input: activeProcess.stdout!,
+      crlfDelay: Infinity,
+    });
+
+    rl.on('line', (line) => {
+      try {
+        const msg = JSON.parse(line);
+        emit(msg as DictationEvent);
+      } catch (e) {
+        // Ignore parse errors
+      }
+    });
+
+    // Capture stderr for debugging
+    activeProcess.stderr?.on('data', (data) => {
+      const text = data.toString().trim();
+      if (text && !text.includes('Downloading')) {
+        // Don't emit download progress as errors
+        // Could log this for debugging
+      }
+    });
+
+    activeProcess.on('close', (code) => {
+      activeProcess = null;
+      emit({ type: 'stopped' });
+    });
+
+    activeProcess.on('error', (err) => {
+      activeProcess = null;
+      emit({ type: 'error', message: err.message });
+    });
+
+    return true;
+  } catch (err) {
+    emit({
+      type: 'error',
+      message: `Failed to start ztc-audio: ${err instanceof Error ? err.message : 'Unknown error'}`
+    });
+    return false;
+  }
+}
+
+/**
+ * Stop native recording
+ * Returns the final transcription
+ */
+export async function stopNativeRecording(): Promise<string> {
+  if (!activeProcess) {
+    return '';
+  }
+
+  return new Promise((resolve) => {
+    let finalText = '';
+
+    // Listen for the final transcription
+    const cleanup = onDictationEvent((event) => {
+      if (event.type === 'text' && event.partial === false) {
+        finalText = event.content || '';
+      }
+      if (event.type === 'stopped') {
+        cleanup();
+        resolve(finalText);
+      }
+    });
+
+    // Send SIGINT to trigger graceful shutdown
+    activeProcess?.kill('SIGINT');
+
+    // Fallback timeout
+    setTimeout(() => {
+      if (activeProcess) {
+        activeProcess.kill('SIGKILL');
+        activeProcess = null;
+      }
+      cleanup();
+      resolve(finalText);
+    }, 3000);
+  });
+}
+
+/**
+ * Cancel native recording without getting transcription
+ */
+export function cancelNativeRecording(): void {
+  if (activeProcess) {
+    activeProcess.kill('SIGKILL');
+    activeProcess = null;
+  }
+}
+
+/**
+ * Get recording status info
+ */
+export function getNativeDictationStatus(): string {
+  const binaryPath = getBinaryPath();
+
+  const parts: string[] = [];
+
+  if (binaryPath) {
+    parts.push(`Native dictation: available`);
+    parts.push(`Binary: ${binaryPath}`);
+    parts.push(`Model: ~/.ztc/models/ggml-*.bin`);
+  } else {
+    parts.push('Native dictation: not available');
+    parts.push('Build with: cd native/ztc-audio && cargo build --release');
+  }
+
+  if (isNativeRecording()) {
+    parts.push('\nCurrently recording...');
+  }
+
+  return parts.join('\n');
+}