uneeq-js 2.54.0 → 2.55.0

package/dist/src/service/speech-handler.d.ts

@@ -16,12 +16,14 @@ export declare class SpeechHandler {
     private wsReconnectMessageQueue;
     private storedTranscription?;
     private awaitingFinalTranscript;
+    private interruptionBlocked;
     constructor(options: SpeechHandlerOptions);
     startRecognition(): void;
     stopRecognition(): void;
     pause(): boolean;
     resume(): boolean;
     setChatMetadata(chatMetadata: ChatMetadata): void;
+    pauseVadIfInterruptNotAllowed(): void;
     private loadScript;
     private loadScripts;
     private handleAppMessages;
@@ -29,6 +31,8 @@ export declare class SpeechHandler {
     private sendWebsocketMessageQueue;
     private loadSavedAudioHeaders;
     private initVoiceActivityDetection;
+    private startMicVad;
+    private pauseMicVad;
     private onVadSpeechStart;
     private onVadSpeechEnd;
     private sendStoredTranscriptionIfReady;
@@ -44,4 +48,5 @@ export declare class SpeechHandler {
     private mediaRecorderOnData;
     private handlePttTranscriptionResult;
     private handleVadTranscriptionResult;
+    private sendChatPrompt;
 }

package/dist/src/types/MessageTypes.d.ts

@@ -54,7 +54,8 @@ export declare enum UneeqMessageType {
     DigitalHumanFailedToPlay = "DigitalHumanFailedToPlay",
     DigitalHumanPlayedInMutedModeSuccess = "DigitalHumanPlayedInMutedModeSuccess",
     DigitalHumanUnmuted = "DigitalHumanUnmuted",
-    CustomMetadataUpdated = "CustomMetadataUpdated"
+    CustomMetadataUpdated = "CustomMetadataUpdated",
+    VadInterruptionAllowed = "VadInterruptionAllowed"
 }
 /**
  * Service is ready to be used
@@ -257,6 +258,14 @@ export declare class SpeechTranscriptionMessage implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
     constructor(speechTranscription: SpeechTranscriptionResult);
 }
+/**
+ * VAD interruption status update
+ */
+export declare class VadInterruptionAllowedMessage implements UneeqMessage {
+    readonly interruptionAllowed: boolean;
+    uneeqMessageType: UneeqMessageType;
+    constructor(interruptionAllowed: boolean);
+}
 /**
  * Digital human has started speaking the message
  */

package/dist/src/types/SpeechHandlerOptions.d.ts

@@ -13,4 +13,5 @@ export interface SpeechHandlerOptions {
     jwtToken: string;
     chatMetadata: ChatMetadata;
     enableVad?: boolean;
+    enableInterrupt?: boolean;
 }

package/dist/src/types/UneeqOptions.d.ts

@@ -98,4 +98,11 @@ export interface UneeqOptions {
      * This can be set to any stringified value.
      */
     customMetadata?: string;
+    /**
+     * Enable Interrupt By Speech
+     * When using SPEECH_RECOGNITION mode, this option will allow the user to interrupt the Digital Human by speaking over them.
+     * If this value is false, the Digital Human will not be interrupted by the user. The default value is false - users cannot
+     * interrupt the Digital Human. Note: Sending a text message will interrupt the digital human irrespective of this setting.
+     */
+    enableInterruptBySpeech?: boolean;
 }

package/dist/umd/index.js

@@ -3880,6 +3880,9 @@ class Uneeq {
     sendTranscript(text) {
         if (this.serviceReadyCheck('sendTranscript')) {
             if (!this.sessionPaused) {
+                if (this.speechHandler) {
+                    this.speechHandler.pauseVadIfInterruptNotAllowed();
+                }
                 this.api.avatarAsk(text, this.chatMetadata);
             }
             else {
@@ -4001,6 +4004,9 @@ class Uneeq {
      * Trigger this conversation's welcome message
      */
     playWelcomeMessage() {
+        if (this.speechHandler) {
+            this.speechHandler.pauseVadIfInterruptNotAllowed();
+        }
         return this.api.avatarWelcome(this.chatMetadata);
     }
     /* Unmute the digital human video - this might be called if the video started in muted mode.*/
@@ -4073,7 +4079,8 @@ class Uneeq {
                     hintPhrasesBoost: this.options.speechRecognitionHintPhrasesBoost,
                     jwtToken: token,
                     chatMetadata: this.chatMetadata,
-                    enableVad: this.options.enableVad
+                    enableVad: this.options.enableVad,
+                    enableInterrupt: this.options.enableInterruptBySpeech
                 });
             }
             else {
@@ -13313,7 +13320,7 @@ exports.race = race;
 /***/ ((module) => {
 
 "use strict";
-module.exports = JSON.parse('{"name":"uneeq-js","version":"2.54.0","description":"","main":"dist/index.js","types":"dist/src/index.d.ts","scripts":{"start":"npx webpack -w","test-local":"npx karma start karma.conf.js -logLevel=DEBUG","test":"npx karma start --browsers ChromeHeadless --single-run","test:windows":"karma start karma.conf.js","build":"webpack --config webpack.config.prod.js && webpack --config webpack.config.umd.js","lint":"npx tslint -p tsconfig.json --fix","docs":"npx typedoc --options"},"files":["dist","!dist/test"],"author":"","license":"ISC","dependencies":{"@stomp/stompjs":"^6.0.0","@uehreka/seriously":"^1.0.1","fast-text-encoding":"^1.0.6","intrinsic-scale":"^3.0.4","onnxruntime-web":"^1.15.1","promjs":"^0.4.1","ring-buffer-ts":"^1.2.0","rxjs":"^7.8.1","rxjs-compat":"^6.6.7","simple-peer":"^9.11.1","webrtc-adapter":"^8.2.3"},"devDependencies":{"@types/dom-mediacapture-record":"^1.0.16","@types/jasmine":"^2.8.8","@types/node":"^10.9.4","fetch-mock":"7.7.3","ignore-styles":"^5.0.1","jasmine":"^5.1.0","jasmine-core":"^5.1.0","karma":"6.4.3","karma-chrome-launcher":"^3.2.0","karma-firefox-launcher":"^2.1.2","karma-jasmine":"^5.1.0","karma-jasmine-html-reporter":"^2.1.0","karma-requirejs":"^1.1.0","karma-typescript":"^5.5.4","karma-typescript-es6-transform":"^5.5.4","nock":"^9.6.1","requirejs":"^2.3.6","ts-loader":"^9.4.4","ts-node":"^7.0.1","tslint":"^5.11.0","tslint-no-focused-test":"^0.5.0","typedoc":"^0.18.0","typescript":"^5.1.6","webpack":"^5.88.2","webpack-cli":"^5.1.4"}}');
+module.exports = JSON.parse('{"name":"uneeq-js","version":"2.55.0","description":"","main":"dist/index.js","types":"dist/src/index.d.ts","scripts":{"start":"npx webpack -w","test-local":"npx karma start karma.conf.js -logLevel=DEBUG","test":"npx karma start --browsers ChromeHeadless --single-run","test:windows":"karma start karma.conf.js","build":"webpack --config webpack.config.prod.js && webpack --config webpack.config.umd.js","lint":"npx tslint -p tsconfig.json","lint-fix":"npx tslint -p tsconfig.json --fix","docs":"npx typedoc --options"},"files":["dist","!dist/test"],"author":"","license":"ISC","dependencies":{"@stomp/stompjs":"^6.0.0","@uehreka/seriously":"^1.0.1","fast-text-encoding":"^1.0.6","intrinsic-scale":"^3.0.4","onnxruntime-web":"^1.15.1","promjs":"^0.4.1","ring-buffer-ts":"^1.2.0","rxjs":"^7.8.1","rxjs-compat":"^6.6.7","simple-peer":"^9.11.1","webrtc-adapter":"^8.2.3"},"devDependencies":{"@types/dom-mediacapture-record":"^1.0.16","@types/jasmine":"^2.8.8","@types/node":"^10.9.4","fetch-mock":"7.7.3","ignore-styles":"^5.0.1","jasmine":"^5.1.0","jasmine-core":"^5.1.0","karma":"6.4.3","karma-chrome-launcher":"^3.2.0","karma-firefox-launcher":"^2.1.2","karma-jasmine":"^5.1.0","karma-jasmine-html-reporter":"^2.1.0","karma-requirejs":"^1.1.0","karma-typescript":"^5.5.4","karma-typescript-es6-transform":"^5.5.4","nock":"^9.6.1","requirejs":"^2.3.6","ts-loader":"^9.4.4","ts-node":"^7.0.1","tslint":"^5.11.0","tslint-no-focused-test":"^0.5.0","typedoc":"^0.18.0","typescript":"^5.1.6","webpack":"^5.88.2","webpack-cli":"^5.1.4"}}');
 
 /***/ }),
 /* 242 */
@@ -29975,7 +29982,7 @@ exports.DigitalHumanVideoPlayManager = DigitalHumanVideoPlayManager;
 "use strict";
 
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.CustomMetadataUpdated = exports.DigitalHumanUnmuted = exports.DigitalHumanPlayedInMutedModeSuccess = exports.DigitalHumanFailedToPlay = exports.DigitalHumanVideoError = exports.ClientPerformanceMessage = exports.ClientMediaStreamUpdateMessage = exports.OnlineStatusUpdateMessage = exports.SessionInfoMessage = exports.AvatarRequestIgnored = exports.AvatarRequestCompleted = exports.SessionErrorMessage = exports.WebRtcStatsMessage = exports.WebRtcDataMessage = exports.SessionLiveMessage = exports.WarningMessage = exports.RecordingStoppedMessage = exports.RecordingStartedMessage = exports.DevicePermissionAllowedMessage = exports.AvatarTextInputFinishedMessage = exports.FinishedSpeakingMessage = exports.StartedSpeakingMessage = exports.UserStoppedSpeakingMessage = exports.UserStartedSpeakingMessage = exports.SpeechTranscriptionMessage = exports.AvatarQuestionMessage = exports.InstructionsMessage = exports.AvatarAnswerContentMessage = exports.AvatarAnswerMessage = exports.ResumeSessionUnavailable = exports.ConnectionLostMessage = exports.VoiceActivityInputModeNotSupportedMessage = exports.MicActivityNotSupportedMessage = exports.MicActivityMessage = exports.MicActivityErrorMessage = exports.ServiceUnavailableMessage = exports.AvatarUnavailableMessage = exports.AvatarAvailableMessage = exports.SessionEndedMessage = exports.SessionResumedMessage = exports.SessionPausedMessage = exports.DeviceListUpdatedMessage = exports.DeviceErrorMessage = exports.SetSpeakerSuccessMessage = exports.SetMicSuccessMessage = exports.SetCameraSuccessMessage = exports.DeviceNotFoundErrorMessage = exports.ReadyMessage = exports.UneeqMessageType = void 0;
+exports.CustomMetadataUpdated = exports.DigitalHumanUnmuted = exports.DigitalHumanPlayedInMutedModeSuccess = exports.DigitalHumanFailedToPlay = exports.DigitalHumanVideoError = exports.ClientPerformanceMessage = exports.ClientMediaStreamUpdateMessage = exports.OnlineStatusUpdateMessage = exports.SessionInfoMessage = exports.AvatarRequestIgnored = exports.AvatarRequestCompleted = exports.SessionErrorMessage = exports.WebRtcStatsMessage = exports.WebRtcDataMessage = exports.SessionLiveMessage = exports.WarningMessage = exports.RecordingStoppedMessage = exports.RecordingStartedMessage = exports.DevicePermissionAllowedMessage = exports.AvatarTextInputFinishedMessage = exports.FinishedSpeakingMessage = exports.StartedSpeakingMessage = exports.UserStoppedSpeakingMessage = exports.UserStartedSpeakingMessage = exports.VadInterruptionAllowedMessage = exports.SpeechTranscriptionMessage = exports.AvatarQuestionMessage = exports.InstructionsMessage = exports.AvatarAnswerContentMessage = exports.AvatarAnswerMessage = exports.ResumeSessionUnavailable = exports.ConnectionLostMessage = exports.VoiceActivityInputModeNotSupportedMessage = exports.MicActivityNotSupportedMessage = exports.MicActivityMessage = exports.MicActivityErrorMessage = exports.ServiceUnavailableMessage = exports.AvatarUnavailableMessage = exports.AvatarAvailableMessage = exports.SessionEndedMessage = exports.SessionResumedMessage = exports.SessionPausedMessage = exports.DeviceListUpdatedMessage = exports.DeviceErrorMessage = exports.SetSpeakerSuccessMessage = exports.SetMicSuccessMessage = exports.SetCameraSuccessMessage = exports.DeviceNotFoundErrorMessage = exports.ReadyMessage = exports.UneeqMessageType = void 0;
 var UneeqMessageType;
 (function (UneeqMessageType) {
     UneeqMessageType["Ready"] = "Ready";
@@ -30026,6 +30033,7 @@ var UneeqMessageType;
     UneeqMessageType["DigitalHumanPlayedInMutedModeSuccess"] = "DigitalHumanPlayedInMutedModeSuccess";
     UneeqMessageType["DigitalHumanUnmuted"] = "DigitalHumanUnmuted";
     UneeqMessageType["CustomMetadataUpdated"] = "CustomMetadataUpdated";
+    UneeqMessageType["VadInterruptionAllowed"] = "VadInterruptionAllowed";
 })(UneeqMessageType || (exports.UneeqMessageType = UneeqMessageType = {}));
 /**
  * Service is ready to be used
@@ -30276,6 +30284,16 @@ class SpeechTranscriptionMessage {
     }
 }
 exports.SpeechTranscriptionMessage = SpeechTranscriptionMessage;
+/**
+ * VAD interruption status update
+ */
+class VadInterruptionAllowedMessage {
+    constructor(interruptionAllowed) {
+        this.interruptionAllowed = interruptionAllowed;
+        this.uneeqMessageType = UneeqMessageType.VadInterruptionAllowed;
+    }
+}
+exports.VadInterruptionAllowedMessage = VadInterruptionAllowedMessage;
 /**
  * Digital human has started speaking the message
  */
@@ -51271,6 +51289,7 @@ const websocketReconnectionQueueLimit = 200;
 const transcriptionStabilityThreshold = 0.5;
 class SpeechHandler {
     constructor(options) {
+        var _a, _b, _c;
         this.options = options;
         this.logPrefix = 'UneeQ: Speech Recognition: ';
         this.speechBuffer = new ring_buffer_ts_1.RingBuffer(speechBufferLength);
@@ -51279,8 +51298,10 @@ class SpeechHandler {
         this.reconnectWs = true;
         this.wsReconnectMessageQueue = [];
         this.awaitingFinalTranscript = false;
-        this.options.assetBasePath = this.options.assetBasePath || defaultAssetPath;
-        this.options.enableVad = this.options.enableVad === undefined ? true : this.options.enableVad;
+        this.interruptionBlocked = false;
+        this.options.assetBasePath = (_a = this.options.assetBasePath) !== null && _a !== void 0 ? _a : defaultAssetPath;
+        this.options.enableVad = (_b = this.options.enableVad) !== null && _b !== void 0 ? _b : true;
+        this.options.enableInterrupt = (_c = this.options.enableInterrupt) !== null && _c !== void 0 ? _c : false;
         this.loadSavedAudioHeaders();
         this.loadScripts();
         this.handleAppMessages();
@@ -51297,15 +51318,13 @@ class SpeechHandler {
                 track.stop();
             });
         }
-        if (this.micVad) {
-            this.micVad.pause();
-        }
+        this.pauseMicVad();
     }
     pause() {
         if (this.stream && this.micVad) {
             this.sendStoredTranscriptionIfReady();
             this.stream.getTracks().forEach((track) => track.enabled = false);
-            this.micVad.pause();
+            this.pauseMicVad();
             this.onVadSpeechEnd();
             return true;
         }
@@ -51321,8 +51340,7 @@ class SpeechHandler {
         this.storedTranscription = undefined;
         if (this.stream && this.micVad) {
             this.stream.getTracks().forEach((track) => track.enabled = true);
-            this.micVad.start();
-            return true;
+            return this.startMicVad();
         }
         logger_1.logger.warn(this.logPrefix + 'Could not resume speech recognition.');
         return false;
@@ -51330,6 +51348,15 @@ class SpeechHandler {
     setChatMetadata(chatMetadata) {
         this.options.chatMetadata = chatMetadata;
     }
+    pauseVadIfInterruptNotAllowed() {
+        if (this.micVad && !this.options.enableInterrupt) {
+            this.micVad.pause();
+            this.options.messages.next(new MessageTypes_1.VadInterruptionAllowedMessage(false));
+        }
+        if (!this.options.enableInterrupt) {
+            this.interruptionBlocked = true;
+        }
+    }
     loadScript(url) {
         return new Promise((resolve) => {
             const script = document.createElement('script');
@@ -51356,9 +51383,15 @@ class SpeechHandler {
             switch (msg.uneeqMessageType) {
                 case MessageTypes_1.UneeqMessageType.StartedSpeaking:
                     this.digitalHumanSpeaking = true;
+                    this.pauseVadIfInterruptNotAllowed();
                     break;
                 case MessageTypes_1.UneeqMessageType.FinishedSpeaking:
                     this.digitalHumanSpeaking = false;
+                    this.interruptionBlocked = false;
+                    this.startMicVad();
+                    if (!this.options.enableInterrupt) {
+                        this.options.messages.next(new MessageTypes_1.VadInterruptionAllowedMessage(true));
+                    }
                     break;
                 case MessageTypes_1.UneeqMessageType.SessionEnded:
                     this.reconnectWs = false;
@@ -51422,12 +51455,35 @@ class SpeechHandler {
             stream: stream,
         }).then((vad) => {
             this.micVad = vad;
-            this.micVad.start();
+            // Don't start VAD if interruption has been blocked
+            if (!this.interruptionBlocked) {
+                this.startMicVad();
+            }
+            else {
+                this.options.messages.next(new MessageTypes_1.VadInterruptionAllowedMessage(false));
+            }
             logger_1.logger.log(this.logPrefix + 'VAD module has started');
         }).catch((err) => {
             console.error(this.logPrefix + 'Could not initialize VAD module', err);
         });
     }
+    startMicVad() {
+        if (this.micVad) {
+            if (!this.digitalHumanSpeaking) {
+                this.micVad.start();
+                return true;
+            }
+        }
+        logger_1.logger.log(this.logPrefix + 'startMicVad: VAD has not been started.');
+        return false;
+    }
+    pauseMicVad() {
+        if (this.micVad) {
+            this.micVad.pause();
+        }
+        logger_1.logger.log(this.logPrefix + 'pauseMicVad: VAD has not been paused.');
+        return false;
+    }
     onVadSpeechStart() {
         logger_1.logger.log(this.logPrefix + 'User started speaking');
         this.options.messages.next(new MessageTypes_1.UserStartedSpeakingMessage());
@@ -51454,7 +51510,7 @@ class SpeechHandler {
     sendStoredTranscriptionIfReady() {
         if (!this.options.enableVad && this.storedTranscription !== undefined && !this.awaitingFinalTranscript) {
             // send the transcript
-            this.options.api.avatarAsk(this.storedTranscription.transcript, this.options.chatMetadata);
+            this.sendChatPrompt(this.storedTranscription.transcript);
             this.options.messages.next(new MessageTypes_1.SpeechTranscriptionMessage(this.storedTranscription));
             this.storedTranscription = undefined;
         }
@@ -51598,7 +51654,7 @@ class SpeechHandler {
                         result.transcript = this.storedTranscription.transcript + result.transcript;
                     }
                     this.options.chatMetadata.userSpokenLocale = result.language_code;
-                    this.options.api.avatarAsk(result.transcript, this.options.chatMetadata);
+                    this.sendChatPrompt(result.transcript);
                     this.options.messages.next(new MessageTypes_1.SpeechTranscriptionMessage(result));
                     this.storedTranscription = undefined;
                 }
@@ -51616,18 +51672,24 @@ class SpeechHandler {
         }
     }
     handleVadTranscriptionResult(result) {
-        if (result.final) {
-            this.options.chatMetadata.userSpokenLocale = result.language_code;
-            this.options.api.avatarAsk(result.transcript, this.options.chatMetadata);
-        }
-        else if (this.digitalHumanSpeaking) {
-            this.options.api.stopSpeaking();
-            this.digitalHumanSpeaking = false;
+        if (result.transcript !== '') {
+            if (result.final) {
+                this.options.chatMetadata.userSpokenLocale = result.language_code;
+                this.sendChatPrompt(result.transcript);
+            }
+            else if (this.digitalHumanSpeaking) {
+                this.options.api.stopSpeaking();
+                this.digitalHumanSpeaking = false;
+            }
         }
         if (result.stability > transcriptionStabilityThreshold || result.final) {
             this.options.messages.next(new MessageTypes_1.SpeechTranscriptionMessage(result));
         }
     }
+    sendChatPrompt(transcript) {
+        this.pauseVadIfInterruptNotAllowed();
+        this.options.api.avatarAsk(transcript, this.options.chatMetadata);
+    }
 }
 exports.SpeechHandler = SpeechHandler;
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uneeq-js",
-  "version": "2.54.0",
+  "version": "2.55.0",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/src/index.d.ts",
@@ -10,7 +10,8 @@
     "test": "npx karma start --browsers ChromeHeadless --single-run",
     "test:windows": "karma start karma.conf.js",
     "build": "webpack --config webpack.config.prod.js && webpack --config webpack.config.umd.js",
-    "lint": "npx tslint -p tsconfig.json --fix",
+    "lint": "npx tslint -p tsconfig.json",
+    "lint-fix": "npx tslint -p tsconfig.json --fix",
     "docs": "npx typedoc --options"
   },
   "files": [

package/readme.md

@@ -8,6 +8,12 @@ https://docs.uneeq.io/build-your-own-experience
 To use uneeq-js with a typescript version < 3.8, 'skipLibCheck' must be set to true.
 
 ## Release Notes
+#### 2.55.0
+* Added a new option 'enableInterrupt' that can be used to enable interruption of the digital human. The default value is false, users cannot interrupt the digital human.
+
+#### 2.54.1
+* Fixed an edge case that caused the digital human to stop speaking when a blank STT result was received.
+
 #### 2.54.0
 * Re-enabled echoCancellation and fixed an issue that caused speech recognition to perform poorly in high latency environments.