uneeq-js 2.46.6 → 2.47.0

package/dist/index.js.LICENSE.txt

@@ -5,6 +5,12 @@
  * @license  MIT
  */
 
+/*!
+* ONNX Runtime Web v1.15.1
+* Copyright (c) Microsoft Corporation. All rights reserved.
+* Licensed under the MIT License.
+*/
+
 /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
 
 /*! queue-microtask. MIT License. Feross Aboukhadijeh <https://feross.org/opensource> */

package/dist/src/lib/vad/frame-processor.d.ts

@@ -0,0 +1,89 @@
+import { Message } from './messages';
+import { SpeechProbabilities } from './models';
+export interface FrameProcessorOptions {
+    /** Threshold over which values returned by the Silero VAD model will be considered as positively indicating speech.
+     * The Silero VAD model is run on each frame. This number should be between 0 and 1.
+     */
+    positiveSpeechThreshold: number;
+    /** Threshold under which values returned by the Silero VAD model will be considered as indicating an absence of speech.
+     * Note that the creators of the Silero VAD have historically set this number at 0.15 less than `positiveSpeechThreshold`.
+     */
+    negativeSpeechThreshold: number;
+    /** After a VAD value under the `negativeSpeechThreshold` is observed, the algorithm will wait `redemptionFrames` frames
+     * before running `onSpeechEnd`. If the model returns a value over `positiveSpeechThreshold` during this grace period, then
+     * the algorithm will consider the previously-detected "speech end" as having been a false negative.
+     */
+    redemptionFrames: number;
+    /** Number of audio samples (under a sample rate of 16000) to comprise one "frame" to feed to the Silero VAD model.
+     * The `frame` serves as a unit of measurement of lengths of audio segments and many other parameters are defined in terms of
+     * frames. The authors of the Silero VAD model offer the following warning:
+     * > WARNING! Silero VAD models were trained using 512, 1024, 1536 samples for 16000 sample rate and
+     * 256, 512, 768 samples for 8000 sample rate.
+     * > Values other than these may affect model performance!!
+     * In this context, audio fed to the VAD model always has sample rate 16000. It is probably a good idea to leave this at 1536.
+     */
+    frameSamples: number;
+    /** Number of frames to prepend to the audio segment that will be passed to `onSpeechEnd`. */
+    preSpeechPadFrames: number;
+    /** If an audio segment is detected as a speech segment according to initial algorithm but it has fewer than `minSpeechFrames`,
+     * it will be discarded and `onVADMisfire` will be run instead of `onSpeechEnd`.
+     */
+    minSpeechFrames: number;
+}
+export declare const defaultFrameProcessorOptions: FrameProcessorOptions;
+export declare function validateOptions(options: FrameProcessorOptions): void;
+export interface FrameProcessorInterface {
+    resume: () => void;
+    process: (arr: Float32Array) => Promise<{
+        probs?: SpeechProbabilities;
+        msg?: Message;
+        audio?: Float32Array;
+    }>;
+    endSegment: () => {
+        msg?: Message;
+        audio?: Float32Array;
+    };
+}
+export declare class FrameProcessor implements FrameProcessorInterface {
+    modelProcessFunc: (frame: Float32Array) => Promise<SpeechProbabilities>;
+    modelResetFunc: () => any;
+    options: FrameProcessorOptions;
+    speaking: boolean;
+    audioBuffer: Array<{
+        frame: Float32Array;
+        isSpeech: boolean;
+    }>;
+    redemptionCounter: number;
+    active: boolean;
+    constructor(modelProcessFunc: (frame: Float32Array) => Promise<SpeechProbabilities>, modelResetFunc: () => any, options: FrameProcessorOptions);
+    reset: () => void;
+    pause: () => void;
+    resume: () => void;
+    endSegment: () => {
+        msg: Message;
+        audio: Float32Array;
+    } | {
+        msg: Message;
+        audio?: undefined;
+    } | {
+        msg?: undefined;
+        audio?: undefined;
+    };
+    process: (frame: Float32Array) => Promise<{
+        probs?: undefined;
+        msg?: undefined;
+        audio?: undefined;
+    } | {
+        probs: SpeechProbabilities;
+        msg: Message;
+        audio?: undefined;
+    } | {
+        probs: SpeechProbabilities;
+        msg: Message;
+        audio: Float32Array;
+    } | {
+        probs: SpeechProbabilities;
+        msg?: undefined;
+        audio?: undefined;
+    }>;
+}

package/dist/src/lib/vad/messages.d.ts

@@ -0,0 +1,6 @@
+export declare enum Message {
+    AudioFrame = "AUDIO_FRAME",
+    SpeechStart = "SPEECH_START",
+    VADMisfire = "VAD_MISFIRE",
+    SpeechEnd = "SPEECH_END"
+}

package/dist/src/lib/vad/models.d.ts

@@ -0,0 +1,23 @@
+export type ONNXRuntimeAPI = any;
+export type ModelFetcher = () => Promise<ArrayBuffer>;
+export interface SpeechProbabilities {
+    notSpeech: number;
+    isSpeech: number;
+}
+export interface Model {
+    reset_state: () => void;
+    process: (arr: Float32Array) => Promise<SpeechProbabilities>;
+}
+export declare class Silero {
+    private ort;
+    private modelFetcher;
+    static new: (ort: ONNXRuntimeAPI, modelFetcher: ModelFetcher) => Promise<Silero>;
+    _session: any;
+    _h: any;
+    _c: any;
+    _sr: any;
+    constructor(ort: ONNXRuntimeAPI, modelFetcher: ModelFetcher);
+    init: () => Promise<void>;
+    reset_state: () => void;
+    process: (audioFrame: Float32Array) => Promise<SpeechProbabilities>;
+}

package/dist/src/lib/vad/real-time-vad.d.ts

@@ -0,0 +1,61 @@
+/**
+ * This VAD module is a fork from the following project:
+ * https://github.com/ricky0123/vad/tree/master/packages/web
+ *
+ * It has been modified to only include what is required for our
+ * own VAD implementation.
+ *
+ * This module is a wrapper to bring the following ML Model into
+ * JS: https://github.com/snakers4/silero-vad
+ */
+import { FrameProcessor, FrameProcessorOptions } from './frame-processor';
+import { SpeechProbabilities } from './models';
+interface RealTimeVADCallbacks {
+    /** Callback to run after each frame. The size (number of samples) of a frame is given by `frameSamples`. */
+    onFrameProcessed: (probabilities: SpeechProbabilities) => any;
+    /** Callback to run if speech start was detected but `onSpeechEnd` will not be run because the
+     * audio segment is smaller than `minSpeechFrames`.
+     */
+    onVADMisfire: () => any;
+    /** Callback to run when speech start is detected */
+    onSpeechStart: () => any;
+    /**
+     * Callback to run when speech end is detected.
+     * Takes as arg a Float32Array of audio samples between -1 and 1, sample rate 16000.
+     * This will not run if the audio segment is smaller than `minSpeechFrames`.
+     */
+    onSpeechEnd: (audio: Float32Array) => any;
+}
+interface RealTimeVADOptionsWithStream extends FrameProcessorOptions, RealTimeVADCallbacks {
+    stream: MediaStream;
+    assetBasePath: string;
+}
+export type RealTimeVADOptions = RealTimeVADOptionsWithStream;
+export declare const defaultRealTimeVADOptions: RealTimeVADOptions;
+export declare class MicVAD {
+    options: RealTimeVADOptions;
+    static new(options?: Partial<RealTimeVADOptions>): Promise<MicVAD>;
+    audioContext: AudioContext;
+    stream: MediaStream;
+    audioNodeVAD: AudioNodeVAD;
+    listening: boolean;
+    constructor(options: RealTimeVADOptions);
+    init: () => Promise<void>;
+    pause: () => void;
+    start: () => void;
+}
+export declare class AudioNodeVAD {
+    ctx: AudioContext;
+    options: RealTimeVADOptions;
+    static new(ctx: AudioContext, options?: Partial<RealTimeVADOptions>): Promise<AudioNodeVAD>;
+    frameProcessor: FrameProcessor;
+    entryNode: AudioNode;
+    constructor(ctx: AudioContext, options: RealTimeVADOptions);
+    pause: () => void;
+    start: () => void;
+    receive: (node: AudioNode) => void;
+    processFrame: (frame: Float32Array) => Promise<void>;
+    init: () => Promise<void>;
+    modelFetcher: () => Promise<ArrayBuffer>;
+}
+export {};

package/dist/src/service/speech-handler.d.ts

@@ -0,0 +1,24 @@
+import { SpeechHandlerOptions } from '../types/SpeechHandlerOptions';
+export declare class SpeechHandler {
+    private options;
+    private ws;
+    private mediaRecorder;
+    private logPrefix;
+    private speechBuffer;
+    private micVad;
+    private recordingLive;
+    private headerBlob;
+    constructor(options: SpeechHandlerOptions);
+    private initWebsocket;
+    private loadSavedAudioHeaders;
+    private initVoiceActivityDetection;
+    private onVadSpeechStart;
+    private onVadSpeechEnd;
+    private handleWebsocketMessage;
+    private sendStartMessage;
+    private sendStopMessage;
+    private initMicrophone;
+    private initMediaRecorder;
+    private mediaRecorderOnData;
+    private handleTranscriptionResult;
+}

package/dist/src/types/MessageTypes.d.ts

@@ -1,5 +1,6 @@
 import { ClientPerformanceStats } from './ClientPerformanceStats';
 import { PublishSubscribeState } from './PublishSubscribeState';
+import { SpeechTranscriptionResult } from './SpeechTranscriptionResult';
 import { Devices } from './types';
 export interface UneeqMessage {
     uneeqMessageType: UneeqMessageType;
@@ -32,6 +33,9 @@ export declare enum UneeqMessageType {
     FinishedSpeaking = "FinishedSpeaking",
     AvatarTextInputFinished = "AvatarTextInputFinished",
     AvatarQuestionText = "AvatarQuestionText",
+    SpeechTranscription = "SpeechTranscription",
+    UserStartedSpeaking = "UserStartedSpeaking",
+    UserStoppedSpeaking = "UserStoppedSpeaking",
     DevicePermissionAllowed = "DevicePermissionAllowed",
     RecordingStarted = "RecordingStarted",
     RecordingStopped = "RecordingStopped",
@@ -243,6 +247,28 @@ export declare class AvatarQuestionMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
     constructor(question: any, transcriptId: string);
 }
+/**
+ * The question the user asked
+ */
+export declare class SpeechTranscriptionMessage implements UneeqMessage {
+    readonly speechTranscription: SpeechTranscriptionResult;
+    uneeqMessageType: UneeqMessageType;
+    constructor(speechTranscription: SpeechTranscriptionResult);
+}
+/**
+ * Digital human has started speaking the message
+ */
+export declare class UserStartedSpeakingMessage implements UneeqMessage {
+    uneeqMessageType: UneeqMessageType;
+    constructor();
+}
+/**
+ * Digital human has started speaking the message
+ */
+export declare class UserStoppedSpeakingMessage implements UneeqMessage {
+    uneeqMessageType: UneeqMessageType;
+    constructor();
+}
 /**
  * Digital human has started speaking the message
  */

package/dist/src/types/SpeechHandlerOptions.d.ts

@@ -0,0 +1,11 @@
+import { Subject } from 'rxjs';
+import { API } from '../service/api';
+export interface SpeechHandlerOptions {
+    apiUrl: string;
+    sessionId: string;
+    api: API;
+    messages: Subject<any>;
+    assetBasePath?: string;
+    hintPhrases: string;
+    locales: string;
+}

package/dist/src/types/SpeechRecognitionMessageAction.d.ts

@@ -0,0 +1,4 @@
+export declare enum SpeechRecognitionMessageAction {
+    startTranscription = "startTranscription",
+    stopTranscription = "stopTranscription"
+}

package/dist/src/types/SpeechRecognitionStartMessage.d.ts

@@ -0,0 +1,9 @@
+import { SpeechRecognitionMessageAction } from './SpeechRecognitionMessageAction';
+export interface SpeechRecognitionStartMessage {
+    action: SpeechRecognitionMessageAction.startTranscription;
+    channels: number;
+    sampleRate: number;
+    interimResults: boolean;
+    lang: string;
+    phrases: string;
+}

package/dist/src/types/SpeechRecognitionStopMessage.d.ts

@@ -0,0 +1,4 @@
+import { SpeechRecognitionMessageAction } from './SpeechRecognitionMessageAction';
+export interface SpeechRecognitionStopMessage {
+    action: SpeechRecognitionMessageAction.stopTranscription;
+}

package/dist/src/types/SpeechTranscriptionResult.d.ts

@@ -0,0 +1,6 @@
+export interface SpeechTranscriptionResult {
+    confidence: number;
+    final: boolean;
+    language_code: string;
+    transcript: string;
+}

package/dist/src/types/UneeqOptions.d.ts

@@ -72,4 +72,13 @@ export interface UneeqOptions {
      * Locales should be separated with a colon e.g "en-US:en-GB:en-AU".
      */
     speechToTextLocales?: string;
+    /**
+     * Override the asset base path.
+     */
+    assetBasePath?: string;
+    /**
+     * Speech Recognition Hint Phrases
+     * A space separated list of phrases the speech recognition system should expect.
+     */
+    speechRecognitionHintPhrases?: string;
 }

package/dist/src/uneeq.d.ts

@@ -22,6 +22,7 @@ export declare class Uneeq {
     private messageHandler;
     private voiceInputManager;
     private unavailableAvatarMsgSent;
+    private speechHandler;
     private readonly options;
     private readonly messageCallback;
     private readonly startSessionData;