uneeq-js 2.47.5 → 2.47.7

package/dist/src/service/api.d.ts

@@ -29,6 +29,7 @@ export declare class API {
     setAvatarDebug(enabled: boolean): Promise<any>;
     sendMediaUnavailable(data: MediaUnavailableRequest): Promise<any>;
     getEdgeNodeId(turnServerAddress: string): Promise<any>;
+    getSessionJwt(personaId: string): Promise<any>;
     getTimeZone(): string;
     private makeRequest;
     private getRequest;

package/dist/src/service/speech-handler.d.ts

@@ -10,7 +10,11 @@ export declare class SpeechHandler {
     private headerBlob;
     private scriptsLoadedPromise;
     private digitalHumanSpeaking;
+    private stream;
     constructor(options: SpeechHandlerOptions);
+    stopRecognition(): void;
+    pause(): boolean;
+    resume(): boolean;
     private loadScript;
     private loadScripts;
     private handleAppMessages;

package/dist/src/types/SpeechHandlerOptions.d.ts

@@ -8,4 +8,5 @@ export interface SpeechHandlerOptions {
     assetBasePath?: string;
     hintPhrases: string;
     locales: string;
+    jwtToken: string;
 }

package/dist/src/uneeq.d.ts

@@ -27,6 +27,7 @@ export declare class Uneeq {
     private readonly messageCallback;
     private readonly startSessionData;
     private metricsService;
+    private sessionJwt;
     private startSessionTime;
     private availableResponseTime;
     private streamManager;
@@ -102,6 +103,16 @@ export declare class Uneeq {
      * When successful a corresponding AvatarRequestCompleted should be received.
      */
     stopSpeaking(): Promise<void>;
+    /**
+     * Pause speech recognition. The users speech will not be processed, or sent off their device.
+     * This is the equivilent of muting the users microphone.
+     */
+    pauseSpeechRecognition(): boolean;
+    /**
+     * Resume speech recognition. The users speech will being being processed again.
+     * This is the equivilent of unmuting the users microphone.
+     */
+    resumeSpeechRecognition(): boolean;
     /**
      * Pauses a live session. Local video and audio will stop being sent and remote avatar video audio will be stopped.
      * On success, SessionPausedMessage will be sent.

package/dist/umd/index.js

@@ -3902,6 +3902,26 @@ class Uneeq {
     stopSpeaking() {
         return this.api.stopSpeaking();
     }
+    /**
+     * Pause speech recognition. The users speech will not be processed, or sent off their device.
+     * This is the equivilent of muting the users microphone.
+     */
+    pauseSpeechRecognition() {
+        if (this.speechHandler) {
+            return this.speechHandler.pause();
+        }
+        return false;
+    }
+    /**
+     * Resume speech recognition. The users speech will being being processed again.
+     * This is the equivilent of unmuting the users microphone.
+     */
+    resumeSpeechRecognition() {
+        if (this.speechHandler) {
+            return this.speechHandler.resume();
+        }
+        return false;
+    }
     /**
      * Pauses a live session. Local video and audio will stop being sent and remote avatar video audio will be stopped.
      * On success, SessionPausedMessage will be sent.
@@ -4011,34 +4031,41 @@ class Uneeq {
         });
     }
     initDeviceManager() {
-        if (this.options.voiceInputMode === VoiceInputMode_1.VoiceInputMode.SPEECH_RECOGNITION) {
-            this.speechHandler = new speech_handler_1.SpeechHandler({
-                api: this.api,
-                apiUrl: this.options.url,
-                sessionId: this.session.id,
+        this.sessionJwt
+            .then(({ token }) => {
+            if (this.options.voiceInputMode === VoiceInputMode_1.VoiceInputMode.SPEECH_RECOGNITION) {
+                this.speechHandler = new speech_handler_1.SpeechHandler({
+                    api: this.api,
+                    apiUrl: this.options.url,
+                    sessionId: this.session.id,
+                    messages: this.messages,
+                    assetBasePath: this.options.assetBasePath,
+                    locales: this.options.speechToTextLocales || '',
+                    hintPhrases: this.options.speechRecognitionHintPhrases || '',
+                    jwtToken: token
+                });
+            }
+            else {
+                // Legacy implementation
+                // Init Voice Input Manager
+                this.voiceInputManager = new voice_input_manager_1.VoiceInputManager(this.options, this.api, this.messages, (enable) => { this.enableMicrophone(enable); });
+            }
+            this.deviceManager = new device_manager_1.DeviceManager({
+                userMessages: this.messages,
+                session: this.session,
                 messages: this.messages,
-                assetBasePath: this.options.assetBasePath,
-                locales: this.options.speechToTextLocales || '',
-                hintPhrases: this.options.speechRecognitionHintPhrases || ''
-            });
-        }
-        else {
-            // Legacy implementation
-            // Init Voice Input Manager
-            this.voiceInputManager = new voice_input_manager_1.VoiceInputManager(this.options, this.api, this.messages, (enable) => { this.enableMicrophone(enable); });
-        }
-        this.deviceManager = new device_manager_1.DeviceManager({
-            userMessages: this.messages,
-            session: this.session,
-            messages: this.messages,
-            internalMessages$: this.internalMessages$,
-            messaging: this.messaging,
-            uneeqOptions: this.options,
-            streamManager: this.streamManager,
-            voiceInputManager: this.voiceInputManager,
-            digitalHumanVideoElement: this.digitalHumanVideoElement
-        }, this.api);
-        this.messageHandler.setDeviceManager(this.deviceManager);
+                internalMessages$: this.internalMessages$,
+                messaging: this.messaging,
+                uneeqOptions: this.options,
+                streamManager: this.streamManager,
+                voiceInputManager: this.voiceInputManager,
+                digitalHumanVideoElement: this.digitalHumanVideoElement
+            }, this.api);
+            this.messageHandler.setDeviceManager(this.deviceManager);
+        })
+            .catch((err) => {
+            this.messages.next(new MessageTypes_1.SessionErrorMessage(err));
+        });
     }
     sessionLiveHandler() {
         // Send the SessionLive message and play welcome (optional)
@@ -4116,6 +4143,7 @@ class Uneeq {
         if (this.session) {
             this.initPromMetricsListener();
             this.api.callId = this.session.id;
+            this.sessionJwt = this.api.getSessionJwt(this.options.conversationId);
             this.initAvatar();
             const messagingOptions = {
                 maxReconnectAttempts: 3,
@@ -13240,7 +13268,7 @@ exports.race = race;
 /***/ ((module) => {
 
 "use strict";
-module.exports = JSON.parse('{"name":"uneeq-js","version":"2.47.5","description":"","main":"dist/index.js","types":"dist/src/index.d.ts","scripts":{"start":"npx webpack -w","test-local":"npx karma start karma.conf.js -logLevel=DEBUG","test":"npx karma start --browsers ChromeHeadless --single-run","test:windows":"karma start karma.conf.js","build":"webpack --config webpack.config.prod.js && webpack --config webpack.config.umd.js","lint":"npx tslint -p tsconfig.json --fix","docs":"npx typedoc --options"},"files":["dist","!dist/test"],"author":"","license":"ISC","dependencies":{"@stomp/stompjs":"^6.0.0","@uehreka/seriously":"^1.0.1","fast-text-encoding":"^1.0.0","intrinsic-scale":"^3.0.4","onnxruntime-web":"^1.15.1","promjs":"^0.4.1","ring-buffer-ts":"^1.2.0","rxjs":"^7.8.1","rxjs-compat":"^6.6.7","simple-peer":"^9.11.1","webrtc-adapter":"^8.2.3"},"devDependencies":{"@types/dom-mediacapture-record":"^1.0.16","@types/jasmine":"^2.8.8","@types/node":"^10.9.4","fetch-mock":"7.7.3","ignore-styles":"^5.0.1","jasmine":"^5.1.0","jasmine-core":"^5.1.0","karma":"^6.4.2","karma-chrome-launcher":"^3.2.0","karma-firefox-launcher":"^2.1.2","karma-jasmine":"^5.1.0","karma-jasmine-html-reporter":"^2.1.0","karma-requirejs":"^1.1.0","karma-typescript":"^5.5.4","karma-typescript-es6-transform":"^5.5.4","nock":"^9.6.1","requirejs":"^2.3.6","ts-loader":"^9.4.4","ts-node":"^7.0.1","tslint":"^5.11.0","tslint-no-focused-test":"^0.5.0","typedoc":"^0.18.0","typescript":"^5.1.6","webpack":"^5.88.2","webpack-cli":"^5.1.4"}}');
+module.exports = JSON.parse('{"name":"uneeq-js","version":"2.47.7","description":"","main":"dist/index.js","types":"dist/src/index.d.ts","scripts":{"start":"npx webpack -w","test-local":"npx karma start karma.conf.js -logLevel=DEBUG","test":"npx karma start --browsers ChromeHeadless --single-run","test:windows":"karma start karma.conf.js","build":"webpack --config webpack.config.prod.js && webpack --config webpack.config.umd.js","lint":"npx tslint -p tsconfig.json --fix","docs":"npx typedoc --options"},"files":["dist","!dist/test"],"author":"","license":"ISC","dependencies":{"@stomp/stompjs":"^6.0.0","@uehreka/seriously":"^1.0.1","fast-text-encoding":"^1.0.0","intrinsic-scale":"^3.0.4","onnxruntime-web":"^1.15.1","promjs":"^0.4.1","ring-buffer-ts":"^1.2.0","rxjs":"^7.8.1","rxjs-compat":"^6.6.7","simple-peer":"^9.11.1","webrtc-adapter":"^8.2.3"},"devDependencies":{"@types/dom-mediacapture-record":"^1.0.16","@types/jasmine":"^2.8.8","@types/node":"^10.9.4","fetch-mock":"7.7.3","ignore-styles":"^5.0.1","jasmine":"^5.1.0","jasmine-core":"^5.1.0","karma":"^6.4.2","karma-chrome-launcher":"^3.2.0","karma-firefox-launcher":"^2.1.2","karma-jasmine":"^5.1.0","karma-jasmine-html-reporter":"^2.1.0","karma-requirejs":"^1.1.0","karma-typescript":"^5.5.4","karma-typescript-es6-transform":"^5.5.4","nock":"^9.6.1","requirejs":"^2.3.6","ts-loader":"^9.4.4","ts-node":"^7.0.1","tslint":"^5.11.0","tslint-no-focused-test":"^0.5.0","typedoc":"^0.18.0","typescript":"^5.1.6","webpack":"^5.88.2","webpack-cli":"^5.1.4"}}');
 
 /***/ }),
 /* 242 */
@@ -13316,6 +13344,19 @@ class API {
         });
         return this.makeRequest(request);
     }
+    getSessionJwt(personaId) {
+        const request = new Request(this.apiUrl + '/v1/jwt/token', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+                session_id: this._callId,
+                persona_id: personaId
+            })
+        });
+        return this.makeRequest(request);
+    }
     getTimeZone() {
         return Intl.DateTimeFormat().resolvedOptions().timeZone;
     }
@@ -51175,13 +51216,44 @@ class SpeechHandler {
         this.speechBuffer = new ring_buffer_ts_1.RingBuffer(speechBufferLength);
         this.recordingLive = false;
         this.digitalHumanSpeaking = false;
-        console.warn('UneeQ: SPEECH_RECOGNITION input mode is in development, you may experience issues.');
+        console.warn(this.logPrefix + 'SPEECH_RECOGNITION input mode is in development, you may experience issues.');
         this.options.assetBasePath = this.options.assetBasePath || defaultAssetPath;
         this.loadSavedAudioHeaders();
         this.loadScripts();
         this.initWebsocket(this.options);
         this.handleAppMessages();
     }
+    stopRecognition() {
+        if (this.mediaRecorder) {
+            this.mediaRecorder.stop();
+        }
+        if (this.stream) {
+            this.stream.getTracks().forEach((track) => {
+                track.stop();
+            });
+        }
+        if (this.micVad) {
+            this.micVad.pause();
+        }
+    }
+    pause() {
+        if (this.stream && this.micVad) {
+            this.stream.getTracks().forEach((track) => track.enabled = false);
+            this.micVad.pause();
+            return true;
+        }
+        logger_1.logger.warn(this.logPrefix + 'Could not pause speech recognition.');
+        return false;
+    }
+    resume() {
+        if (this.stream && this.micVad) {
+            this.stream.getTracks().forEach((track) => track.enabled = true);
+            this.micVad.start();
+            return true;
+        }
+        logger_1.logger.warn(this.logPrefix + 'Could not resume speech recognition.');
+        return false;
+    }
     loadScript(url) {
         return new Promise((resolve) => {
             const script = document.createElement('script');
@@ -51212,13 +51284,16 @@ class SpeechHandler {
                 case MessageTypes_1.UneeqMessageType.FinishedSpeaking:
                     this.digitalHumanSpeaking = false;
                     break;
+                case MessageTypes_1.UneeqMessageType.SessionEnded:
+                    this.stopRecognition();
+                    break;
                 default:
             }
         });
     }
     initWebsocket(options) {
-        let connectionUrl = options.apiUrl.replace('https://api.', 'wss://speech-recognition-service.');
-        connectionUrl += '/ws/recognize?session_id=' + options.sessionId;
+        let connectionUrl = options.apiUrl.replace('https://', 'wss://');
+        connectionUrl += `/speech-recognition/ws/recognize?jwt=${options.jwtToken}&session_id=${options.sessionId}`;
         logger_1.logger.log(this.logPrefix + 'Initializing speech recognition websocket to ' + connectionUrl);
         this.ws = new WebSocket(connectionUrl);
         this.ws.onopen = () => {
@@ -51319,6 +51394,7 @@ class SpeechHandler {
     initMicrophone() {
         navigator.mediaDevices.getUserMedia({ audio: { sampleRate: audioSampleRate, channelCount: audioChannelCount } })
             .then((stream) => {
+            this.stream = stream;
             this.initVoiceActivityDetection(stream);
             this.initMediaRecorder(stream);
         })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uneeq-js",
-  "version": "2.47.5",
+  "version": "2.47.7",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/src/index.d.ts",

package/readme.md

@@ -8,6 +8,12 @@ https://docs.uneeq.io/build-your-own-experience
 To use uneeq-js with a typescript version < 3.8, 'skipLibCheck' must be set to true.
 
 ## Release Notes
+#### 2.47.7
+* added jwt security to speech recognition
+
+#### 2.47.6
+* Speech Recognition: added support for pause and resume.
+
 #### 2.47.5
 * Speech Recognition mode: Stop the digital human from speaking when an interim speech result is received.