uneeq-js 3.0.16-preview.97 → 3.1.0

package/dist/types/SpeechRecognitionInterface.d.ts

@@ -0,0 +1,12 @@
+import { type PromptMetadata } from './PromptMetadata';
+/**
+ * Common interface for speech recognition providers.
+ * Both Google STT (SpeechRecognition) and Deepgram STT (DeepgramSTT) implement this interface.
+ */
+export interface SpeechRecognitionInterface {
+    startRecognition(): void | Promise<void>;
+    stopRecognition(): void | Promise<void>;
+    pause(): boolean | Promise<boolean>;
+    resume(): boolean | Promise<boolean>;
+    setChatMetadata(chatMetadata: PromptMetadata): void;
+}

package/dist/types/SpeechStreamEvents.d.ts

@@ -0,0 +1,5 @@
+export interface SpeechStreamEvent {
+    type: string;
+    streamId: number;
+    data: any;
+}

package/dist/types/UneeqConfig.d.ts

@@ -1,6 +1,10 @@
 import { type CameraAnchorDistance, type CameraAnchorHorizontal } from "../webrtc-data-channel/messages/CameraAnchor";
 import { type LogLevel } from "./LogLevel";
 import { type UneeqMessage } from "./UneeqMessages";
+export declare enum SpeechRecognitionProvider {
+    Google = "google",
+    Deepgram = "deepgram"
+}
 export interface UneeqConfig {
     sessionId: string;
     sessionToken: string;
@@ -16,6 +20,7 @@ export interface UneeqConfig {
     enableInterruptBySpeech?: boolean;
     cameraAnchorDistance: CameraAnchorDistance;
     cameraAnchorHorizontal: CameraAnchorHorizontal;
+    forceTURN?: boolean;
     /**
      * This option specifies up to four locales that the Digital Human should understand a person speaking (speech recognition)
      * The locales are specified in the BCP-47 format e.g "en-US".
@@ -33,6 +38,106 @@ export interface UneeqConfig {
      * An integer number between 0 and 20 that can be used to boost the hint phrases likely hood to match.
      */
     speechRecognitionHintPhrasesBoost?: number;
+    /**
+     * Speech recognition provider selection.
+     * @default SpeechRecognitionProvider.Google
+     *
+     * Supported values:
+     * - 'google': Uses UneeQ's speech-recognition-service (existing implementation)
+     * - 'deepgram': Uses Deepgram real-time STT with pause/mute support
+     *
+     * Any other value will log a warning and default to 'google'.
+     */
+    speechRecognitionProvider?: SpeechRecognitionProvider | 'google' | 'deepgram';
+    /**
+     * Configuration for Deepgram STT provider.
+     * These options only apply when using Deepgram - they are ignored when using Google STT provider.
+     */
+    deepgramConfig?: {
+        /**
+         * Deepgram model to use for transcription.
+         * @default 'nova-2'
+         */
+        model?: string;
+        /**
+         * Language code in BCP-47 format (e.g., 'en', 'en-US', 'es', 'fr').
+         * @default 'en'
+         */
+        language?: string;
+        /**
+         * Enable smart formatting (punctuation, capitalization).
+         * @default true
+         */
+        smartFormat?: boolean;
+        /**
+         * Enable interim results (partial transcripts).
+         * @default true
+         */
+        interimResults?: boolean;
+        /**
+         * Milliseconds of silence to detect end of utterance.
+         * @default 1000
+         */
+        utteranceEndMs?: number;
+        /**
+         * Enable VAD events (SpeechStarted, UtteranceEnd).
+         * @default true
+         */
+        vadEvents?: boolean;
+        /**
+         * Audio encoding format.
+         * @default 'linear16'
+         */
+        encoding?: string;
+        /**
+         * Audio sample rate in Hz.
+         * @default 16000
+         */
+        sampleRate?: number;
+        /**
+         * Number of audio channels.
+         * @default 1
+         */
+        channels?: number;
+        /**
+         * Microphone audio processing configuration.
+         */
+        microphone?: {
+            /**
+             * Enable echo cancellation (removes feedback from speakers).
+             * @default true
+             */
+            echoCancellation?: boolean;
+            /**
+             * Enable noise suppression (reduces background noise).
+             * @default true
+             */
+            noiseSuppression?: boolean;
+            /**
+             * Enable automatic gain control (normalizes volume levels).
+             * @default true
+             */
+            autoGainControl?: boolean;
+        };
+        /**
+         * Minimum number of words in interim transcript needed to interrupt digital human.
+         * This prevents background noise from interrupting, but allows real speech to interrupt.
+         * @default 1
+         */
+        interruptionWordThreshold?: number;
+        /**
+         * Enable filler word detection in transcripts.
+         * When enabled, Deepgram will identify filler words like "um", "uh", "like", etc.
+         * @default false
+         */
+        fillerWords?: boolean;
+        /**
+         * Silence detection threshold in milliseconds.
+         * Controls how long to wait for silence before finalizing a transcript.
+         * @default 10
+         */
+        endpointing?: number;
+    };
     customMetadata?: object;
     messageHandler?: (msg: UneeqMessage) => void;
     webRtcStatsEmitMessages?: boolean;

package/dist/types/UneeqErrorCodes.d.ts

@@ -0,0 +1 @@
+export declare const BackendErrorCodes: number[];

package/dist/types/UneeqMessages.d.ts

@@ -7,104 +7,45 @@ export interface UneeqMessage {
     uneeqMessageType: UneeqMessageType;
 }
 export declare enum UneeqMessageType {
-    Ready = "Ready",
-    DeviceNotFoundError = "DeviceNotFoundError",
-    SetCameraSuccess = "SetCameraSuccess",
-    SetMicSuccess = "SetMicSuccess",
-    SetSpeakerSuccess = "SetSpeakerSuccess",
-    SessionPaused = "SessionPaused",
-    SessionResumed = "SessionResumed",
-    SessionEnded = "SessionEnded",
-    AvatarAvailable = "AvatarAvailable",
-    AvatarUnavailable = "AvatarUnavailable",
-    ConnectionLost = "ConnectionLost",
-    ResumeSessionUnavailable = "ResumeSessionUnavailable",
-    ServiceUnavailable = "ServiceUnavailable",
-    OnlineStatusUpdate = "OnlineStatusUpdate",
-    MicActivityNotSupported = "MicActivityNotSupported",
-    VoiceActivityInputModeNotSupported = "VoiceActivityInputModeNotSupported",
-    MicActivityError = "MicActivityError",
-    MicActivity = "MicActivity",
-    DeviceError = "DeviceError",
-    DeviceListUpdated = "DeviceListUpdated",
     AvatarAnswer = "AvatarAnswer",
-    AvatarAnswerContent = "AvatarAnswerContent",
-    Instructions = "Instructions",
     AvatarStartedSpeaking = "AvatarStartedSpeaking",
     AvatarStoppedSpeaking = "AvatarStoppedSpeaking",
-    AvatarQuestionText = "AvatarQuestionText",
-    SpeechTranscription = "SpeechTranscription",
-    SpeechRecognitionInterrupt = "SpeechRecognitionInterrupt",
-    UserStartedSpeaking = "UserStartedSpeaking",
-    UserStoppedSpeaking = "UserStoppedSpeaking",
-    DevicePermissionAllowed = "DevicePermissionAllowed",
-    RecordingStarted = "RecordingStarted",
-    RecordingStopped = "RecordingStopped",
-    Warning = "Warning",
-    SessionLive = "SessionLive",
-    WebRtcData = "WebRtcData",
-    WebRtcStats = "WebRtcStats",
-    SessionError = "SessionError",
-    SessionBackendError = "SessionBackendError",
-    SessionInfo = "SessionInfo",
-    AvatarRequestCompleted = "AvatarRequestCompleted",
-    AvatarRequestIgnored = "AvatarRequestIgnored",
-    ClientMediaStreamUpdate = "ClientMediaStreamUpdate",
-    ClientPerformanceMessage = "ClientPerformanceMessage",
-    DigitalHumanVideoError = "DigitalHumanVideoError",
-    DigitalHumanFailedToPlay = "DigitalHumanFailedToPlay",
+    CallToActionDismissed = "CallToActionDismissed",
+    CustomMetadataUpdated = "CustomMetadataUpdated",
+    DataProcessingWarning = "DataProcessingWarning",
+    DeviceError = "DeviceError",
+    DigitalHumanMuted = "DigitalHumanMuted",
     DigitalHumanPlayedInMutedModeSuccess = "DigitalHumanPlayedInMutedModeSuccess",
     DigitalHumanUnmuted = "DigitalHumanUnmuted",
-    DigitalHumanMuted = "DigitalHumanMuted",
-    CustomMetadataUpdated = "CustomMetadataUpdated",
-    VadInterruptionAllowed = "VadInterruptionAllowed",
     EnableMicrophoneUpdated = "EnableMicrophoneUpdated",
-    PromptResult = "PromptResult",
+    FrameReady = "FrameReady",
+    FrameResize = "FrameResize",
+    MicPermissionDenied = "MicPermissionDenied",
+    Notification = "Notification",
     PromptRequest = "PromptRequest",
+    PromptResult = "PromptResult",
+    ReadyToStart = "ReadyToStart",
+    SceneReady = "SceneReady",
+    SessionBackendError = "SessionBackendError",
     SessionConnecting = "SessionConnecting",
     SessionDisconnected = "SessionDisconnected",
+    SessionEnded = "SessionEnded",
+    SessionError = "SessionError",
+    SessionLive = "SessionLive",
     SessionReconnecting = "SessionReconnecting",
+    SessionReconnectingFinished = "SessionReconnectingFinished",
+    SessionStateUpdate = "SessionStateUpdate",
+    SoftSwitchFinished = "SoftSwitchFinished",
+    SoftSwitchStarting = "SoftSwitchStarting",
+    SpeechEvent = "SpeechEvent",
+    SpeechTranscription = "SpeechTranscription",
+    UnhandledMessage = "UnhandledMessage",
+    UserStartedSpeaking = "UserStartedSpeaking",
+    UserStoppedSpeaking = "UserStoppedSpeaking",
+    VadInterruptionAllowed = "VadInterruptionAllowed",
+    VideoLayoutConfiguring = "VideoLayoutConfiguring",
     WaitingInQueue = "WaitingInQueue",
-    SpeechEvent = "SpeechEvent"
-}
-/**
- * Service is ready to be used
- */
-export declare class ReadyMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * Selected device is not found
- */
-export declare class DeviceNotFoundErrorMessage implements UneeqMessage {
-    readonly msg: string;
-    readonly deviceId: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(msg: string, deviceId: string);
-}
-/**
- * Users camera has been set successfully
- */
-export declare class SetCameraSuccessMessage implements UneeqMessage {
-    readonly deviceId: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(deviceId: string);
-}
-/**
- * Users microphone has been set successfully
- */
-export declare class SetMicSuccessMessage implements UneeqMessage {
-    readonly deviceId: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(deviceId: string);
-}
-/**
- * Users speaker has been set successfully
- */
-export declare class SetSpeakerSuccessMessage implements UneeqMessage {
-    readonly deviceId: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(deviceId: string);
+    WebRtcStats = "WebRtcStats"
 }
 /**
  * An error has occurred with the users device
@@ -114,76 +55,6 @@ export declare class DeviceErrorMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
     constructor(error: Error);
 }
-/**
- * Session has been paused successfully
- */
-export declare class SessionPausedMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * Session has been resumed successfully
- */
-export declare class SessionResumedMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * An avatar is available
- */
-export declare class AvatarAvailableMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * There are no avatars currently available
- */
-export declare class AvatarUnavailableMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * Service is unavailable
- */
-export declare class ServiceUnavailableMessage implements UneeqMessage {
-    readonly error: any;
-    uneeqMessageType: UneeqMessageType;
-    constructor(error: any);
-}
-/**
- * An error occured with mic activity messages
- */
-export declare class MicActivityErrorMessage implements UneeqMessage {
-    readonly error: Error;
-    uneeqMessageType: UneeqMessageType;
-    constructor(error: Error);
-}
-/**
- * Message is sent upon activity of the user microphone
- *
- * Visibility is controlled by UneeqOptions param micActivityMessages
- */
-export declare class MicActivityMessage implements UneeqMessage {
-    readonly level: number;
-    uneeqMessageType: UneeqMessageType;
-    constructor(level: number);
-}
-/**
- * Mic activity messages are not supported by the users browser
- *
- * Visibility is controlled by UneeqOptions param micActivityMessages
- */
-export declare class MicActivityNotSupportedMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * Mic activity input mode is not supported by the users browser
- */
-export declare class VoiceActivityInputModeNotSupportedMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * Resume Session Unavailable
- */
-export declare class ResumeSessionUnavailable implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
 /**
  * The avatar's answer to a question
  */
@@ -195,40 +66,6 @@ export declare class AvatarAnswerMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
     constructor(answer: string, answerAvatar: string, answerSpeech: string, transcriptId: string);
 }
-/**
- * The avatar's answer to a question (HTML Content)
- */
-export declare class AvatarAnswerContentMessage implements UneeqMessage {
-    readonly content: string;
-    readonly transcriptId: string;
-    readonly hasDisplayHtml: boolean;
-    uneeqMessageType: UneeqMessageType;
-    /**
-     * @param content Note: If `hasDisplayHtml` = true, this content will contain the HTML provided by the displayHTML instruction.
-     * If `hasDisplayHtml` = false, this content will contain the `answer` value
-     * @param transcriptId
-     * @param hasDisplayHtml
-     */
-    constructor(content: string, transcriptId: string, hasDisplayHtml?: boolean);
-}
-export declare class InstructionsMessage implements UneeqMessage {
-    readonly instructions: object;
-    uneeqMessageType: UneeqMessageType;
-    /**
-     *
-     * @param instructions is an object defined on the NLP which is passed to the client in response to certain questions
-     */
-    constructor(instructions: object);
-}
-/**
- * The question the user asked
- */
-export declare class AvatarQuestionMessage implements UneeqMessage {
-    readonly question: any;
-    readonly transcriptId: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(question: any, transcriptId: string);
-}
 /**
  * A speech transcription message containing transcript results
  */
@@ -237,12 +74,6 @@ export declare class SpeechTranscriptionMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
     constructor(speechTranscription: SpeechTranscriptionResult);
 }
-/**
- * The user interrupted the digital human with speech
- */
-export declare class SpeechRecognitionInterruptMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
 /**
  * VAD interruption status update
  */
@@ -275,26 +106,6 @@ export declare class AvatarStartedSpeakingMessage implements UneeqMessage {
 export declare class AvatarStoppedSpeakingMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
 }
-/**
- * User has started a recording successfully
- */
-export declare class RecordingStartedMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * User has stopped recording successfully
- */
-export declare class RecordingStoppedMessage implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-}
-/**
- * General Warning Message - refer to msg
- */
-export declare class WarningMessage implements UneeqMessage {
-    readonly msg: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(msg: string);
-}
 /**
  * Session is Live, avatar video is active
  */
@@ -311,14 +122,6 @@ export declare class WaitingInQueueMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
     constructor(position: number);
 }
-/**
- * WebRTC data channel message
- */
-export declare class WebRtcDataMessage implements UneeqMessage {
-    readonly data: any;
-    uneeqMessageType: UneeqMessageType;
-    constructor(data: any);
-}
 /**
  * WebRTC stats message (diagnostics)
  */
@@ -350,14 +153,6 @@ export interface SessionBackendError {
     category: string;
     session_id: string;
 }
-/**
- * Avatar request completed: the Avatar has completed processing a text or instruction request
- */
-export declare class AvatarRequestCompleted implements UneeqMessage {
-    readonly reason: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(reason: string);
-}
 export declare class SpeechEventMessage implements UneeqMessage {
     readonly speechEvent: SpeechEvent;
     uneeqMessageType: UneeqMessageType;
@@ -368,53 +163,6 @@ export declare class SessionEndedMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
     constructor(reason: string);
 }
-/**
- * Avatar request ignored: the Avatar has ignored a request to speak
- */
-export declare class AvatarRequestIgnored implements UneeqMessage {
-    readonly reason: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(reason: string);
-}
-/**
- * Session Info Messages: Primarily used for testing, e.g. the active behaviour tag
- */
-export declare class SessionInfoMessage implements UneeqMessage {
-    readonly info: string;
-    uneeqMessageType: UneeqMessageType;
-    constructor(info: string);
-}
-/**
- * Clients network status has changed. {online: true|false}
- */
-export declare class OnlineStatusUpdateMessage implements UneeqMessage {
-    readonly online: boolean;
-    uneeqMessageType: UneeqMessageType;
-    constructor(online: boolean);
-}
-export declare class ClientMediaStreamUpdateMessage implements UneeqMessage {
-    readonly clientStreamingAudio: boolean;
-    readonly clientStreamingVideo: boolean;
-    readonly serverStreamingAudio: boolean;
-    readonly serverStreamingVideo: boolean;
-    uneeqMessageType: UneeqMessageType;
-    constructor(clientStreamingAudio: boolean, clientStreamingVideo: boolean, serverStreamingAudio: boolean, serverStreamingVideo: boolean);
-}
-/**
- * DigitalHuman Video Error has occurred
- */
-export declare class DigitalHumanVideoError implements UneeqMessage {
-    readonly error: any;
-    uneeqMessageType: UneeqMessageType;
-    constructor(error: any);
-}
-/**
- * DigitalHuman Failed to play
- */
-export declare class DigitalHumanFailedToPlay implements UneeqMessage {
-    uneeqMessageType: UneeqMessageType;
-    msg: string;
-}
 /**
  * DigitalHuman was successfully played in muted mode
  */
@@ -458,7 +206,14 @@ export declare class PromptResultMessage implements UneeqMessage {
     constructor(promptResult: PromptResult);
 }
 /**
- * Session Disconnected
+ * Scene is ready
+ * Indicates that the avatar and scene are fully loaded and ready to start interaction
+ */
+export declare class SceneReadyMessage implements UneeqMessage {
+    uneeqMessageType: UneeqMessageType;
+}
+/**
+ * Session Connecting
  */
 export declare class SessionConnectingMessage implements UneeqMessage {
     readonly rendererId: string;
@@ -477,3 +232,27 @@ export declare class SessionDisconnectedMessage implements UneeqMessage {
 export declare class SessionReconnectingMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
 }
+/**
+ * Session Reconnecting Finished
+ */
+export declare class SessionReconnectingFinishedMessage implements UneeqMessage {
+    uneeqMessageType: UneeqMessageType;
+}
+/**
+ * Soft Switch Starting
+ */
+export declare class SoftSwitchStartingMessage implements UneeqMessage {
+    uneeqMessageType: UneeqMessageType;
+}
+/**
+ * Soft Switch Finished
+ */
+export declare class SoftSwitchStoppingMessage implements UneeqMessage {
+    uneeqMessageType: UneeqMessageType;
+}
+/**
+ * Video Layout Configuring
+ */
+export declare class VideoLayoutConfiguringMessage implements UneeqMessage {
+    uneeqMessageType: UneeqMessageType;
+}

package/dist/types/dhop-messages.d.ts

@@ -2,5 +2,7 @@ export declare enum DhopMessageType {
     IndexInQueue = "index_in_queue",
     RendererTaken = "renderer_taken",
     RendererGone = "renderer_gone",
-    RendererError = "renderer_error"
+    RendererError = "renderer_error",
+    ClientGone = "client_gone",
+    SwitchRendererRequest = "switch_renderer_request"
 }

package/dist/uneeq.d.ts

@@ -8,6 +8,10 @@ export declare class Uneeq {
     private ensureSessionExists;
     chatPrompt(chatPrompt: string): void;
     speak(speech: string): void;
+    openAudioStream(): void;
+    closeAudioStream(): void;
+    interruptAudioStream(): void;
+    speakAudio(audio: string | Uint8Array | Blob): Promise<void>;
     cameraAnchorDistance(position: CameraAnchorDistance, duration?: number): void;
     cameraAnchorHorizontal(position: CameraAnchorHorizontal, duration?: number): void;
     endSession(): void;
@@ -16,7 +20,7 @@ export declare class Uneeq {
     stopSpeaking(): void;
     setCustomPromptMetadata(metadata: object): void;
     enableMicrophone(enable?: boolean): void;
-    pauseSpeechRecognition(): boolean;
-    resumeSpeechRecognition(): boolean;
+    pauseSpeechRecognition(): Promise<boolean>;
+    resumeSpeechRecognition(): Promise<boolean>;
     setWebRtcStatsEnabled(emitMessages: boolean, logMessages: boolean): void;
 }

package/dist/uneeq.spec.d.ts

@@ -0,0 +1 @@
+export {};