ugly-app 0.1.116 → 0.1.117

package/dist/cli/version.d.ts

@@ -1,2 +1,2 @@
-export declare const CLI_VERSION = "0.1.115";
+export declare const CLI_VERSION = "0.1.117";
 //# sourceMappingURL=version.d.ts.map

package/dist/cli/version.js

@@ -1,3 +1,3 @@
 // Auto-generated by prebuild — do not edit manually
-export const CLI_VERSION = "0.1.115";
+export const CLI_VERSION = "0.1.117";
 //# sourceMappingURL=version.js.map

package/dist/server/audio/stt/AudioStreamProcessor.d.ts

@@ -0,0 +1,226 @@
+import type { STTCorrectionEvent, STTMuteEvent, VADStatus } from '../../../shared/Audio.js';
+import type { STTProviderPriority, STTRequiredFeatures } from '../../../shared/Audio.js';
+type STTForcedModel = 'auto' | 'deepgram' | 'whisper' | 'groq_whisper';
+type UserLang = string;
+type UserLangAutoT = string;
+type STTMode = 'realtime' | 'batch' | 'auto';
+export interface AudioStreamConfig {
+    userId: string;
+    lang: UserLangAutoT;
+    mode: STTMode;
+    conversationId: string | null;
+    vadEnabled: boolean;
+    priority?: STTProviderPriority;
+    forcedModel?: STTForcedModel;
+    diarizeEnabled?: boolean;
+    noiseGateEnabled?: boolean;
+    serverNoiseGateEnabled?: boolean;
+    interimWhisperEnabled?: boolean;
+    interimWhisperIntervalMs?: number;
+    onTranscript: (text: string, isFinal: boolean, lang: UserLang) => void;
+    onCorrection?: (event: STTCorrectionEvent) => void;
+    onSpeechStart?: () => void;
+    onSpeechEnd?: () => void;
+    onVAD?: (probability: number, status: VADStatus) => void;
+    onDiarization?: (segmentCount: number, primarySpeaker: number, activeSpeaker: number | null) => void;
+    onMuteChange?: (event: STTMuteEvent) => void;
+    onProcessedAudio?: (samples: Float32Array) => void;
+    onError?: (error: Error) => void;
+    selectProvider?: (abstractModel: string, priority: STTProviderPriority, requiredFeatures?: STTRequiredFeatures, reserved?: undefined, durationMs?: number) => Promise<{
+        offering: {
+            provider: string;
+            providerModel: string;
+        };
+        reason: string;
+    } | null>;
+    createDeepgramProvider?: (config: {
+        lang: string;
+        userId: string;
+        conversationId: string | null;
+        onTranscript: (text: string, isFinal: boolean) => void;
+        onError: (error: Error) => void;
+        onClose?: () => void;
+        onSpeechFinal?: () => void;
+    }) => {
+        connect: () => Promise<void>;
+        send: (samples: Float32Array) => void;
+        close: () => void;
+        isConnected: () => boolean;
+    };
+    createWhisperProvider?: (userId: string) => {
+        transcribe: (samples: Float32Array, lang: string, conversationId: string | null) => Promise<{
+            text: string;
+            lang: string;
+        }>;
+    };
+    createGroqWhisperProvider?: (userId: string) => {
+        transcribe: (samples: Float32Array, lang: string, conversationId: string | null) => Promise<{
+            text: string;
+            lang: string;
+        }>;
+    };
+}
+/**
+ * Main orchestrator for server-side speech-to-text processing.
+ *
+ * This class coordinates:
+ * - Audio resampling (48kHz -> 16kHz)
+ * - Voice Activity Detection (Silero VAD)
+ * - Provider routing based on priority (price, speed, quality, multilingual)
+ * - Transcript delivery via callbacks
+ */
+export declare class AudioStreamProcessor {
+    private config;
+    private vad;
+    private resampler;
+    private segmentation;
+    private diarizeEnabled;
+    private noiseGateEnabled;
+    private serverNoiseGateEnabled;
+    private lastMuteState;
+    private deepgramProvider;
+    private noVadStreamingStarted;
+    private noVadBuffer;
+    private vadState;
+    private speakerHistory;
+    private readonly MIN_DURATION_TO_ESTABLISH;
+    private readonly TAKEOVER_ENERGY_MULTIPLIER;
+    private diarizationSilenceState;
+    private readonly DIARIZATION_CHECK_INTERVAL_MS;
+    private readonly DIARIZATION_SILENCE_THRESHOLD_MS;
+    private readonly vadChunkSize;
+    private vadRemainderBuffer;
+    private interimWhisper;
+    private readonly START_THRESHOLD;
+    private readonly STOP_THRESHOLD;
+    private readonly MIN_SPEECH_FRAMES;
+    private readonly SILENCE_FRAMES_TO_STOP;
+    private readonly PRE_ROLL_FRAMES;
+    private readonly MAX_UTTERANCE_FRAMES;
+    constructor(config: AudioStreamConfig);
+    private audioChunkCount;
+    private lastAudioLogTime;
+    /**
+     * Initialize the processor (loads VAD model if enabled)
+     */
+    init(): Promise<void>;
+    /**
+     * Start streaming without VAD (for bypass mode)
+     */
+    private startNoVadStreaming;
+    /**
+     * Process incoming audio samples from the client.
+     * Audio is expected at 48kHz mono Float32.
+     *
+     * @param samples48k PCM samples at 48kHz
+     */
+    processAudio(samples48k: Float32Array): Promise<void>;
+    /**
+     * Process audio without VAD - stream directly to provider
+     */
+    private processNoVadAudio;
+    private maxSpeechProb;
+    private lastVadLogTime;
+    /**
+     * Process a single VAD chunk and manage state machine transitions
+     */
+    private processVADChunk;
+    /**
+     * Start a new speech segment
+     */
+    private startSpeech;
+    /**
+     * End the current speech segment
+     */
+    private endSpeech;
+    /**
+     * Transcribe audio using the best available batch provider based on priority
+     */
+    private transcribeBatch;
+    /**
+     * Transcribe audio with a specific provider
+     */
+    private transcribeWithProvider;
+    /**
+     * Stream audio chunk to the active provider
+     */
+    private streamToProvider;
+    /**
+     * Determine the effective mode based on configuration, language, and priority
+     */
+    private getEffectiveMode;
+    /**
+     * Reset VAD state to idle
+     */
+    private resetToIdle;
+    /**
+     * Reset diarization silence detection state
+     */
+    private resetDiarizationSilenceState;
+    /**
+     * Reset interim Whisper state for a new speech segment
+     */
+    private resetInterimWhisperState;
+    /**
+     * Cancel any pending interim Whisper request
+     */
+    private cancelInterimWhisper;
+    /**
+     * Check if we should send an interim Whisper request
+     */
+    private shouldRunInterimWhisper;
+    /**
+     * Run interim Whisper transcription on accumulated audio
+     */
+    private runInterimWhisper;
+    /**
+     * Run full transcription in background and emit correction if different from interim
+     */
+    private runFullTranscriptionWithCorrection;
+    /**
+     * Emit mute change event if noise gate is enabled and state changed
+     */
+    private emitMuteChange;
+    /**
+     * Get recent audio from speech buffer for diarization analysis
+     */
+    private getRecentAudioBuffer;
+    /**
+     * Check if the primary speaker has stopped speaking (diarization-based silence)
+     * Returns true if we should trigger speech end due to primary speaker silence
+     */
+    private checkDiarizationSilence;
+    /**
+     * Concatenate multiple Float32Array buffers into one
+     */
+    private concatenateBuffers;
+    /**
+     * Calculate RMS energy for a segment of audio
+     */
+    private calculateSegmentEnergy;
+    /**
+     * Update speaker stats and select the user speaker with takeover logic
+     */
+    private updateAndSelectUserSpeaker;
+    /**
+     * Filter audio to keep only the established user speaker.
+     * Uses energy-based tracking to establish and maintain user identity.
+     */
+    private filterToPrimarySpeaker;
+    /**
+     * Force end the current utterance immediately — used for push-to-talk on button release.
+     * In no-VAD batch mode, transcribes the accumulated noVadBuffer right now.
+     * In VAD mode, triggers endSpeech() if currently speaking.
+     */
+    forceEndUtterance(): Promise<void>;
+    /**
+     * Clean up resources and finalize any pending transcriptions
+     */
+    destroy(): Promise<void>;
+    /**
+     * Get current VAD status
+     */
+    getStatus(): VADStatus;
+}
+export {};
+//# sourceMappingURL=AudioStreamProcessor.d.ts.map

package/dist/server/audio/stt/AudioStreamProcessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"AudioStreamProcessor.d.ts","sourceRoot":"","sources":["../../../../src/server/audio/stt/AudioStreamProcessor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,kBAAkB,EAClB,YAAY,EACZ,SAAS,EACV,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAQzF,KAAK,cAAc,GAAG,MAAM,GAAG,UAAU,GAAG,SAAS,GAAG,cAAc,CAAC;AACvE,KAAK,QAAQ,GAAG,MAAM,CAAC;AACvB,KAAK,aAAa,GAAG,MAAM,CAAC;AAC5B,KAAK,OAAO,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;AAI7C,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,aAAa,CAAC;IACpB,IAAI,EAAE,OAAO,CAAC;IACd,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,mBAAmB,CAAC;IAE/B,WAAW,CAAC,EAAE,cAAc,CAAC;IAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,sBAAsB,CAAC,EAAE,OAAO,CAAC;IACjC,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,KAAK,IAAI,CAAC;IACvE,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,kBAAkB,KAAK,IAAI,CAAC;IACnD,aAAa,CAAC,EAAE,MAAM,IAAI,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,IAAI,CAAC;IACzB,KAAK,CAAC,EAAE,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IACzD,aAAa,CAAC,EAAE,CACd,YAAY,EAAE,MAAM,EACpB,cAAc,EAAE,MAAM,EACtB,aAAa,EAAE,MAAM,GAAG,IAAI,KACzB,IAAI,CAAC;IACV,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IAC7C,gBAAgB,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;IACnD,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IAGjC,cAAc,CAAC,EAAE,CAAC,aAAa,EAAE,MAAM,EAAE,QAAQ,EAAE,mBAAmB,EAAE,gBAAgB,CAAC,EAAE,mBAAmB,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,QAAQ,EAAE;YAAE,QAAQ,EAAE,MAAM,CAAC;YAAC,aAAa,EAAE,MAAM,CAAA;SAAE,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;IACxP,sBAAsB,CAAC,EAAE,CAAC,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;QAAC,OAAO,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,IAAI,CAAA;KAAE,KAAK;QAAE,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAAC,IAAI,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;QAAC,KAAK,EAAE,MAAM,IAAI,CAAC;QAAC,WAAW,EAAE,MAAM,OAAO,CAAA;KAAE,CAAC;IACxW,qBAAqB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAC5K,yBAAyB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK;QAAE,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CACjL;AAiBD;;;;;;;;GAQG;AACH,qBAAa,oBAAoB;IA8EnB,OAAO,CAAC,MAAM;IA7E1B,OAAO,CAAC,GAAG,CAAY;IACvB,OAAO,CAAC,SAAS,CAAY;IAC7B,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,gBAAgB,CAAU;IAClC,OAAO,CAAC,sBAAsB,CAAU;IACxC,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,gBAAgB,CAAyC;IACjE,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,WAAW,CAAsB;IAEzC,OAAO,CAAC,QAAQ,CAMd;IAGF,OAAO,CAAC,cAAc,CAepB;IAGF,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAO;IACjD,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAO;IAGlD,OAAO,CAAC,uBAAuB,CAM7B;IAGF,OAAO,CAAC,QAAQ,CAAC,6BAA6B,CAAO;IACrD,OAAO,CAAC,QAAQ,CAAC,gCAAgC,CAAO;IACxD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAO;IACpC,OAAO,CAAC,kBAAkB,CAAqC;IAG/D,OAAO,CAAC,cAAc,CAUN;IAGhB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;gBAE1B,MAAM,EAAE,iBAAiB;IA4C7C,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,gBAAgB,CAAK;IAE7B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAoC3B;;OAEG;YACW,mBAAmB;IAmCjC;;;;;OAKG;IACG,YAAY,CAAC,UAAU,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IA8E3D;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,cAAc,CAAK;IAE3B;;OAEG;YACW,eAAe;IA8H7B;;OAEG;YACW,WAAW;IA2DzB;;OAEG;YACW,SAAS;IA8DvB;;OAEG;YACW,eAAe;IAyD7B;;OAEG;YACW,sBAAsB;IA6CpC;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAMxB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IA6BxB;;OAEG;IACH,OAAO,CAAC,WAAW;IAWnB;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAOpC;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAShC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA6C/B;;OAEG;YACW,iBAAiB;IAgE/B;;OAEG;YACW,kCAAkC;IAiChD;;OAEG;IACH,OAAO,CAAC,cAAc;IAWtB;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAuB5B;;;OAGG;YACW,uBAAuB;IAgErC;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAsB9B;;OAEG;IACH,OAAO,CAAC,0BAA0B;IAwGlC;;;OAGG;YACW,sBAAsB;IAyEpC;;;;OAIG;IACG,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BxC;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA+B9B;;OAEG;IACH,SAAS,IAAI,SAAS;CAGvB"}