pi-harness-runtime 1.1.337 → 1.1.339

package/packages/voice-api/src/line.ts

@@ -0,0 +1,229 @@
+/**
+ * LINE Voice Handler
+ * 
+ * Handles voice messages from LINE Messaging API.
+ */
+
+import type { VoiceMessage, TranscriptionResult } from './types.js';
+import { VoiceTranscriber } from './transcriber.js';
+
+/**
+ * LINE webhook event
+ */
+export interface LineWebhookEvent {
+  type: string;
+  timestamp: number;
+  source: {
+    type: string;
+    userId?: string;
+    groupId?: string;
+    roomId?: string;
+  };
+  replyToken?: string;
+  message?: {
+    id: string;
+    type: string;
+    duration?: number;
+  };
+}
+
+/**
+ * LINE webhook body
+ */
+export interface LineWebhookBody {
+  destination: string;
+  events: LineWebhookEvent[];
+}
+
+/**
+ * LINE Messaging API response
+ */
+interface LineMessageContent {
+  displayName?: string;
+}
+
+/**
+ * LINE Voice Handler options
+ */
+export interface LineVoiceHandlerOptions {
+  /** LINE channel access token */
+  channelAccessToken: string;
+  /** LINE user ID secret (for validation) */
+  channelSecret?: string;
+  /** Voice transcriber instance */
+  transcriber: VoiceTranscriber;
+  /** Base URL for LINE API */
+  apiBase?: string;
+  /** Callback when voice is transcribed */
+  onTranscribed?: (userId: string, text: string, result: TranscriptionResult) => Promise<void>;
+  /** Callback on error */
+  onError?: (userId: string, error: Error) => void;
+}
+
+/**
+ * LINE Voice Handler
+ * 
+ * Processes voice messages from LINE Messaging API and transcribes them.
+ */
+export class LineVoiceHandler {
+  private channelAccessToken: string;
+  private channelSecret?: string;
+  private transcriber: VoiceTranscriber;
+  private apiBase: string;
+  private onTranscribed?: (userId: string, text: string, result: TranscriptionResult) => Promise<void>;
+  private onError?: (userId: string, error: Error) => void;
+
+  constructor(options: LineVoiceHandlerOptions) {
+    this.channelAccessToken = options.channelAccessToken;
+    this.channelSecret = options.channelSecret;
+    this.transcriber = options.transcriber;
+    this.apiBase = options.apiBase || 'https://api.line.me';
+    this.onTranscribed = options.onTranscribed;
+    this.onError = options.onError;
+  }
+
+  /**
+   * Verify LINE webhook signature
+   */
+  verifySignature(body: string, signature: string): boolean {
+    if (!this.channelSecret) return true; // Skip verification if no secret
+
+    // Note: In production, implement proper HMAC verification
+    // This is a simplified version
+    return true;
+  }
+
+  /**
+   * Handle incoming webhook from LINE
+   */
+  async handleWebhook(body: LineWebhookBody): Promise<{ success: boolean; responses: string[] }> {
+    const responses: string[] = [];
+
+    for (const event of body.events) {
+      if (event.type !== 'message' || event.message?.type !== 'audio') {
+        continue;
+      }
+
+      const userId = event.source.userId || 'unknown';
+      const messageId = event.message.id;
+
+      try {
+        // Get audio content from LINE
+        const audioContent = await this.getAudioContent(messageId);
+        if (!audioContent) {
+          responses.push(`Failed to get audio for message ${messageId}`);
+          continue;
+        }
+
+        // Transcribe
+        const result = await this.transcriber.transcribe({
+          audioBuffer: audioContent,
+        });
+
+        if (result.error) {
+          this.onError?.(userId, new Error(result.error));
+          responses.push(`Transcription error: ${result.error}`);
+          continue;
+        }
+
+        // Notify callback
+        await this.onTranscribed?.(userId, result.text, result);
+        responses.push(`Transcribed: ${result.text}`);
+
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+        this.onError?.(userId, err);
+        responses.push(`Error: ${err.message}`);
+      }
+    }
+
+    return { success: true, responses };
+  }
+
+  /**
+   * Get audio content from LINE
+   */
+  private async getAudioContent(messageId: string): Promise<Uint8Array | null> {
+    try {
+      const response = await fetch(
+        `${this.apiBase}/v2/bot/message/${messageId}/content`,
+        {
+          headers: {
+            'Authorization': `Bearer ${this.channelAccessToken}`,
+          },
+        }
+      );
+
+      if (!response.ok) {
+        return null;
+      }
+
+      const arrayBuffer = await response.arrayBuffer();
+      return new Uint8Array(arrayBuffer);
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Reply to user with text
+   */
+  async reply(replyToken: string, messages: Array<{ type: string; text?: string }>): Promise<boolean> {
+    try {
+      const response = await fetch(
+        `${this.apiBase}/v2/bot/message/reply`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${this.channelAccessToken}`,
+          },
+          body: JSON.stringify({ replyToken, messages }),
+        }
+      );
+
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Push message to user
+   */
+  async push(userId: string, messages: Array<{ type: string; text?: string }>): Promise<boolean> {
+    try {
+      const response = await fetch(
+        `${this.apiBase}/v2/bot/message/push`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${this.channelAccessToken}`,
+          },
+          body: JSON.stringify({ to: userId, messages }),
+        }
+      );
+
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Create webhook handler for Express/Fastify/etc.
+   */
+  createWebhookHandler() {
+    return async (body: LineWebhookBody, headers: Record<string, string>): Promise<{ status: number; body: object }> => {
+      // Verify signature if provided
+      const signature = headers['x-line-signature'];
+      if (signature && !this.verifySignature(JSON.stringify(body), signature)) {
+        return { status: 401, body: { error: 'Invalid signature' } };
+      }
+
+      const result = await this.handleWebhook(body);
+      return { status: 200, body: { success: result.success, responses: result.responses } };
+    };
+  }
+}

package/packages/voice-api/src/provider.ts

@@ -0,0 +1,186 @@
+/**
+ * Whisper Provider - Abstract interface for speech-to-text
+ */
+
+import type { TranscriptionResult, TranscribeOptions } from './types.js';
+
+/**
+ * Whisper provider interface
+ */
+export interface WhisperProvider {
+  /** Provider name */
+  name: 'openai' | 'replicate';
+  /** Transcribe audio to text */
+  transcribe(options: TranscribeOptions): Promise<TranscriptionResult>;
+  /** Check if provider is configured */
+  isConfigured(): boolean;
+}
+
+/**
+ * Create OpenAI Whisper provider
+ */
+export function createOpenAIProvider(apiKey: string): WhisperProvider {
+  return new OpenAIProvider(apiKey);
+}
+
+/**
+ * Create Replicate Whisper provider
+ */
+export function createReplicateProvider(apiToken: string): WhisperProvider {
+  return new ReplicateProvider(apiToken);
+}
+
+/**
+ * OpenAI Whisper Provider
+ */
+class OpenAIProvider implements WhisperProvider {
+  name = 'openai' as const;
+  private apiKey: string;
+
+  constructor(apiKey: string) {
+    this.apiKey = apiKey;
+  }
+
+  isConfigured(): boolean {
+    return Boolean(this.apiKey);
+  }
+
+  async transcribe(options: TranscribeOptions): Promise<TranscriptionResult> {
+    const startTime = Date.now();
+
+    try {
+      // Dynamic import to avoid hard dependency
+      const OpenAI = (await import('openai')).default;
+      const openai = new OpenAI({ apiKey: this.apiKey });
+
+      let audioData: Buffer | ReadableStream | string;
+
+      if (options.audioBuffer) {
+        audioData = Buffer.from(options.audioBuffer);
+      } else if (options.audioUrl) {
+        // Fetch audio from URL
+        const response = await fetch(options.audioUrl);
+        if (!response.ok) {
+          throw new Error(`Failed to fetch audio: ${response.status}`);
+        }
+        const arrayBuffer = await response.arrayBuffer();
+        audioData = Buffer.from(arrayBuffer);
+      } else {
+        throw new Error('No audio data provided');
+      }
+
+      const result = await openai.audio.transcriptions.create({
+        file: audioData as unknown as File,
+        model: options.model || 'whisper-1',
+        language: options.language,
+        response_format: 'verbose_json',
+      });
+
+      return {
+        text: result.text,
+        language: result.language || options.language,
+        duration: result.duration,
+        timestamp: new Date().toISOString(),
+        provider: 'openai',
+      };
+    } catch (error) {
+      return {
+        text: '',
+        timestamp: new Date().toISOString(),
+        provider: 'openai',
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}
+
+/**
+ * Replicate Whisper Provider
+ */
+class ReplicateProvider implements WhisperProvider {
+  name = 'replicate' as const;
+  private apiToken: string;
+
+  constructor(apiToken: string) {
+    this.apiToken = apiToken;
+  }
+
+  isConfigured(): boolean {
+    return Boolean(this.apiToken);
+  }
+
+  async transcribe(options: TranscribeOptions): Promise<TranscriptionResult> {
+    const startTime = Date.now();
+
+    try {
+      // Get audio data
+      let audioData: Uint8Array;
+      
+      if (options.audioBuffer) {
+        audioData = options.audioBuffer;
+      } else if (options.audioUrl) {
+        const response = await fetch(options.audioUrl);
+        if (!response.ok) {
+          throw new Error(`Failed to fetch audio: ${response.status}`);
+        }
+        const arrayBuffer = await response.arrayBuffer();
+        audioData = new Uint8Array(arrayBuffer);
+      } else {
+        throw new Error('No audio data provided');
+      }
+
+      // Convert to base64 for Replicate
+      const base64 = btoa(String.fromCharCode(...audioData));
+      const dataUrl = `data:audio/wav;base64,${base64}`;
+
+      // Call Replicate API
+      const response = await fetch('https://api.replicate.com/v1/predictions', {
+        method: 'POST',
+        headers: {
+          'Authorization': `Token ${this.apiToken}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          version: 'e39e7d2f3c2b7a7d8c9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b',
+          input: {
+            audio: dataUrl,
+            model: options.model || 'large-v3',
+          },
+        }),
+      });
+
+      if (!response.ok) {
+        throw new Error(`Replicate API error: ${response.status}`);
+      }
+
+      const prediction = await response.json();
+
+      // Poll for completion
+      let result = prediction;
+      while (result.status !== 'succeeded' && result.status !== 'failed') {
+        await new Promise(r => setTimeout(r, 1000));
+        const pollResponse = await fetch(`https://api.replicate.com/v1/predictions/${result.id}`, {
+          headers: { 'Authorization': `Token ${this.apiToken}` },
+        });
+        result = await pollResponse.json();
+      }
+
+      if (result.status === 'failed') {
+        throw new Error(result.error || 'Transcription failed');
+      }
+
+      return {
+        text: result.output,
+        timestamp: new Date().toISOString(),
+        provider: 'replicate',
+      };
+    } catch (error) {
+      return {
+        text: '',
+        timestamp: new Date().toISOString(),
+        provider: 'replicate',
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}

package/packages/voice-api/src/telegram.ts

@@ -0,0 +1,191 @@
+/**
+ * Telegram Voice Handler
+ * 
+ * Handles voice messages from Telegram bots.
+ */
+
+import type { VoiceMessage, TranscriptionResult } from './types.js';
+import { VoiceTranscriber } from './transcriber.js';
+
+/**
+ * Telegram voice message from bot update
+ */
+export interface TelegramVoiceUpdate {
+  update_id: number;
+  message?: {
+    message_id: number;
+    voice?: {
+      file_id: string;
+      duration: number;
+      mime_type: string;
+      file_size?: number;
+    };
+    chat: {
+      id: number;
+      type: string;
+    };
+  };
+}
+
+/**
+ * Telegram API response
+ */
+interface TelegramFileResponse {
+  ok: boolean;
+  result?: {
+    file_id: string;
+    file_unique_id: string;
+    file_path: string;
+  };
+  description?: string;
+}
+
+/**
+ * Telegram Voice Handler options
+ */
+export interface TelegramVoiceHandlerOptions {
+  /** Telegram bot token */
+  botToken: string;
+  /** Voice transcriber instance */
+  transcriber: VoiceTranscriber;
+  /** Base URL for Telegram API */
+  apiBase?: string;
+  /** Callback when voice is transcribed */
+  onTranscribed?: (chatId: string, text: string, result: TranscriptionResult) => Promise<void>;
+  /** Callback on error */
+  onError?: (chatId: string, error: Error) => void;
+}
+
+/**
+ * Telegram Voice Handler
+ * 
+ * Processes voice messages from Telegram and transcribes them.
+ */
+export class TelegramVoiceHandler {
+  private botToken: string;
+  private transcriber: VoiceTranscriber;
+  private apiBase: string;
+  private onTranscribed?: (chatId: string, text: string, result: TranscriptionResult) => Promise<void>;
+  private onError?: (chatId: string, error: Error) => void;
+
+  constructor(options: TelegramVoiceHandlerOptions) {
+    this.botToken = options.botToken;
+    this.transcriber = options.transcriber;
+    this.apiBase = options.apiBase || 'https://api.telegram.org';
+    this.onTranscribed = options.onTranscribed;
+    this.onError = options.onError;
+  }
+
+  /**
+   * Handle incoming update from Telegram webhook
+   */
+  async handleUpdate(update: TelegramVoiceUpdate): Promise<{ success: boolean; text?: string; error?: string }> {
+    const message = update.message;
+    const voice = message?.voice;
+
+    if (!message || !voice) {
+      return { success: false, error: 'No voice message in update' };
+    }
+
+    const chatId = String(message.chat.id);
+
+    try {
+      // Get file path from Telegram
+      const filePath = await this.getFilePath(voice.file_id);
+      if (!filePath) {
+        return { success: false, error: 'Failed to get file path' };
+      }
+
+      // Construct download URL
+      const fileUrl = `${this.apiBase}/file/bot${this.botToken}/${filePath}`;
+
+      // Transcribe
+      const result = await this.transcriber.transcribe({ audioUrl: fileUrl });
+
+      if (result.error) {
+        this.onError?.(chatId, new Error(result.error));
+        return { success: false, error: result.error };
+      }
+
+      // Notify callback
+      await this.onTranscribed?.(chatId, result.text, result);
+
+      return { success: true, text: result.text };
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      this.onError?.(chatId, err);
+      return { success: false, error: err.message };
+    }
+  }
+
+  /**
+   * Get file path from Telegram
+   */
+  private async getFilePath(fileId: string): Promise<string | null> {
+    try {
+      const response = await fetch(
+        `${this.apiBase}/bot${this.botToken}/getFile?file_id=${fileId}`
+      );
+
+      if (!response.ok) {
+        return null;
+      }
+
+      const data = await response.json() as TelegramFileResponse;
+      return data.result?.file_path || null;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Send text response to chat
+   */
+  async sendMessage(chatId: string, text: string): Promise<boolean> {
+    try {
+      const response = await fetch(
+        `${this.apiBase}/bot${this.botToken}/sendMessage`,
+        {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            chat_id: chatId,
+            text,
+            parse_mode: 'Markdown',
+          }),
+        }
+      );
+
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Create webhook handler middleware
+   */
+  createWebhookHandler() {
+    return async (request: Request): Promise<Response> => {
+      try {
+        const update = await request.json() as TelegramVoiceUpdate;
+        const result = await this.handleUpdate(update);
+
+        if (result.success && result.text) {
+          // Optionally echo back the transcription
+          // await this.sendMessage(String(update.message?.chat.id), `🎤: ${result.text}`);
+        }
+
+        return new Response(JSON.stringify({ ok: true }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      } catch (error) {
+        return new Response(JSON.stringify({ ok: false, error: String(error) }), {
+          status: 500,
+          headers: { 'Content-Type': 'application/json' },
+        });
+      }
+    };
+  }
+}