uneeq-js 2.47.10 → 2.48.0

package/dist/src/service/api.d.ts

@@ -1,4 +1,5 @@
 import { AvatarAvailableCountResponse } from '../types/AvatarAvailableCountResponse';
+import { ChatMetadata } from '../types/ChatMetadata';
 import { MediaUnavailableRequest } from '../types/MediaUnavailableRequest';
 import { NewSessionRequest } from '../types/NewSessionRequest';
 import { NewSessionResponse } from '../types/NewSessionResponse';
@@ -19,8 +20,8 @@ export declare class API {
     startSession(data: NewSessionRequest): Promise<NewSessionResponse>;
     endCall(): Promise<void>;
     avatarStart(data: PersonaStartRequest): Promise<void>;
-    avatarWelcome(): Promise<void>;
-    avatarAsk(text: string): Promise<void>;
+    avatarWelcome(metadata?: ChatMetadata): Promise<void>;
+    avatarAsk(text: string, metadata?: ChatMetadata): Promise<void>;
     avatarStartRecording(): Promise<void>;
     avatarStopRecording(): Promise<void>;
     stopSpeaking(): Promise<void>;

package/dist/src/service/speech-handler.d.ts

@@ -1,6 +1,7 @@
+import { ChatMetadata } from '../types/ChatMetadata';
 import { SpeechHandlerOptions } from '../types/SpeechHandlerOptions';
 export declare class SpeechHandler {
-    private options;
+    options: SpeechHandlerOptions;
     private ws;
     private mediaRecorder;
     private logPrefix;
@@ -17,6 +18,7 @@ export declare class SpeechHandler {
     stopRecognition(): void;
     pause(): boolean;
     resume(): boolean;
+    setChatMetadata(chatMetadata: ChatMetadata): void;
     private loadScript;
     private loadScripts;
     private handleAppMessages;

package/dist/src/types/ChatMetadata.d.ts

@@ -0,0 +1,9 @@
+export interface ChatMetadata {
+    userSpokenLocale?: string;
+    browserDetectedLocales?: string;
+    userTimezone?: string;
+    userScreenWidth?: number;
+    userScreenHeight?: number;
+    userAgent?: string;
+    custom?: string;
+}

package/dist/src/types/MessageTypes.d.ts

@@ -1,3 +1,4 @@
+import { ChatMetadata } from './ChatMetadata';
 import { ClientPerformanceStats } from './ClientPerformanceStats';
 import { PublishSubscribeState } from './PublishSubscribeState';
 import { SpeechTranscriptionResult } from './SpeechTranscriptionResult';
@@ -52,7 +53,8 @@ export declare enum UneeqMessageType {
     DigitalHumanVideoError = "DigitalHumanVideoError",
     DigitalHumanFailedToPlay = "DigitalHumanFailedToPlay",
     DigitalHumanPlayedInMutedModeSuccess = "DigitalHumanPlayedInMutedModeSuccess",
-    DigitalHumanUnmuted = "DigitalHumanUnmuted"
+    DigitalHumanUnmuted = "DigitalHumanUnmuted",
+    CustomMetadataUpdated = "CustomMetadataUpdated"
 }
 /**
  * Service is ready to be used
@@ -432,3 +434,8 @@ export declare class DigitalHumanUnmuted implements UneeqMessage {
     uneeqMessageType: UneeqMessageType;
     constructor();
 }
+export declare class CustomMetadataUpdated implements UneeqMessage {
+    readonly chatMetadata: ChatMetadata;
+    uneeqMessageType: UneeqMessageType;
+    constructor(chatMetadata: ChatMetadata);
+}

package/dist/src/types/SpeechHandlerOptions.d.ts

@@ -1,5 +1,6 @@
 import { Subject } from 'rxjs';
 import { API } from '../service/api';
+import { ChatMetadata } from './ChatMetadata';
 export interface SpeechHandlerOptions {
     apiUrl: string;
     sessionId: string;
@@ -9,4 +10,5 @@ export interface SpeechHandlerOptions {
     hintPhrases: string;
     locales: string;
     jwtToken: string;
+    chatMetadata: ChatMetadata;
 }

package/dist/src/types/UneeqOptions.d.ts

@@ -81,4 +81,10 @@ export interface UneeqOptions {
      * A space separated list of phrases the speech recognition system should expect.
      */
     speechRecognitionHintPhrases?: string;
+    /**
+     * Custom Metadata
+     * A stringified value that will be sent to BYO NLPs with all chat calls like playWelcomeMessage and sendTranscript requests.
+     * This can be set to any stringified value.
+     */
+    customMetadata?: string;
 }

package/dist/src/uneeq.d.ts

@@ -34,6 +34,7 @@ export declare class Uneeq {
     private digitalHumanVideoElement;
     private sessionWasResumed;
     private uneeqLocalStorage;
+    private chatMetadata;
     constructor(options: UneeqOptions);
     setLoggerEnabled(enabled: boolean): void;
     /**
@@ -83,6 +84,13 @@ export declare class Uneeq {
      * @param enable Defaults to true
      */
     enableMicrophoneAndCamera(enable?: boolean): void;
+    /**
+     * Set the custom field in chat metadata to a string value that will be sent
+     * to BYO NLPs with all chat calls like playWelcomeMessage and sendTranscript requests.
+     *
+     * @param customMetadata
+     */
+    setCustomChatMetadata(customMetadata: string): void;
     /**
      * Send a text based message to the avatar.
      * @param text
@@ -168,4 +176,5 @@ export declare class Uneeq {
      * Warn implementer if unavailable
      */
     private checkIfResumeSessionIsAvailable;
+    private initChatMetadata;
 }

package/dist/umd/index.js

@@ -3727,6 +3727,7 @@ class Uneeq {
         if (this.options.customData) {
             this.startSessionData.customData = this.options.customData;
         }
+        this.chatMetadata = this.initChatMetadata(this.options);
         logger_1.logger.log(`Uneeq Created (version ${this.version})`);
         this.streamManager = new stream_manager_1.StreamManager();
         this.initDigitalHumanVideoElement();
@@ -3849,6 +3850,21 @@ class Uneeq {
             this.deviceManager.enableLocalDevices(enable, enable);
         }
     }
+    /**
+     * Set the custom field in chat metadata to a string value that will be sent
+     * to BYO NLPs with all chat calls like playWelcomeMessage and sendTranscript requests.
+     *
+     * @param customMetadata
+     */
+    setCustomChatMetadata(customMetadata) {
+        if (this.chatMetadata) {
+            this.chatMetadata.custom = customMetadata;
+            this.messages.next(new MessageTypes_1.CustomMetadataUpdated(this.chatMetadata));
+        }
+        else {
+            this.messages.next(new MessageTypes_1.WarningMessage('Call to set custom chat metadata will be ignored as metadata does not exist yet.'));
+        }
+    }
     /**
      * Send a text based message to the avatar.
      * @param text
@@ -3856,7 +3872,7 @@ class Uneeq {
     sendTranscript(text) {
         if (this.serviceReadyCheck('sendTranscript')) {
             if (!this.sessionPaused) {
-                this.api.avatarAsk(text);
+                this.api.avatarAsk(text, this.chatMetadata);
             }
             else {
                 this.messages.next(new MessageTypes_1.WarningMessage('Call to sendTranscript() will be ignored. Session is paused.'));
@@ -3977,7 +3993,7 @@ class Uneeq {
      * Trigger this conversation's welcome message
      */
     playWelcomeMessage() {
-        return this.api.avatarWelcome();
+        return this.api.avatarWelcome(this.chatMetadata);
     }
     /* Unmute the digital human video - this might be called if the video started in muted mode.*/
     unmuteDigitalHuman() {
@@ -4042,7 +4058,8 @@ class Uneeq {
                     assetBasePath: this.options.assetBasePath,
                     locales: this.options.speechToTextLocales || '',
                     hintPhrases: this.options.speechRecognitionHintPhrases || '',
-                    jwtToken: token
+                    jwtToken: token,
+                    chatMetadata: this.chatMetadata
                 });
             }
             else {
@@ -4072,7 +4089,7 @@ class Uneeq {
         if (!this.sessionStarted) {
             this.messages.next(new MessageTypes_1.SessionLiveMessage(this.session.showUneeqLogo || false, this.sessionWasResumed));
             if (this.options.playWelcome) {
-                this.api.avatarWelcome();
+                this.playWelcomeMessage();
             }
             // Metric the session start time
             const startTimeMetric = {
@@ -4210,6 +4227,17 @@ class Uneeq {
             this.options.resumeSession = false;
         }
     }
+    initChatMetadata(options) {
+        return {
+            userSpokenLocale: '',
+            browserDetectedLocales: options.speechToTextLocales,
+            userTimezone: this.api.getTimeZone(),
+            userScreenWidth: window.screen.availWidth,
+            userScreenHeight: window.screen.availHeight,
+            userAgent: navigator.userAgent,
+            custom: options.customMetadata
+        };
+    }
 }
 exports.Uneeq = Uneeq;
 
@@ -13271,7 +13299,7 @@ exports.race = race;
 /***/ ((module) => {
 
 "use strict";
-module.exports = JSON.parse('{"name":"uneeq-js","version":"2.47.10","description":"","main":"dist/index.js","types":"dist/src/index.d.ts","scripts":{"start":"npx webpack -w","test-local":"npx karma start karma.conf.js -logLevel=DEBUG","test":"npx karma start --browsers ChromeHeadless --single-run","test:windows":"karma start karma.conf.js","build":"webpack --config webpack.config.prod.js && webpack --config webpack.config.umd.js","lint":"npx tslint -p tsconfig.json --fix","docs":"npx typedoc --options"},"files":["dist","!dist/test"],"author":"","license":"ISC","dependencies":{"@stomp/stompjs":"^6.0.0","@uehreka/seriously":"^1.0.1","fast-text-encoding":"^1.0.0","intrinsic-scale":"^3.0.4","onnxruntime-web":"^1.15.1","promjs":"^0.4.1","ring-buffer-ts":"^1.2.0","rxjs":"^7.8.1","rxjs-compat":"^6.6.7","simple-peer":"^9.11.1","webrtc-adapter":"^8.2.3"},"devDependencies":{"@types/dom-mediacapture-record":"^1.0.16","@types/jasmine":"^2.8.8","@types/node":"^10.9.4","fetch-mock":"7.7.3","ignore-styles":"^5.0.1","jasmine":"^5.1.0","jasmine-core":"^5.1.0","karma":"^6.4.2","karma-chrome-launcher":"^3.2.0","karma-firefox-launcher":"^2.1.2","karma-jasmine":"^5.1.0","karma-jasmine-html-reporter":"^2.1.0","karma-requirejs":"^1.1.0","karma-typescript":"^5.5.4","karma-typescript-es6-transform":"^5.5.4","nock":"^9.6.1","requirejs":"^2.3.6","ts-loader":"^9.4.4","ts-node":"^7.0.1","tslint":"^5.11.0","tslint-no-focused-test":"^0.5.0","typedoc":"^0.18.0","typescript":"^5.1.6","webpack":"^5.88.2","webpack-cli":"^5.1.4"}}');
+module.exports = JSON.parse('{"name":"uneeq-js","version":"2.48.0","description":"","main":"dist/index.js","types":"dist/src/index.d.ts","scripts":{"start":"npx webpack -w","test-local":"npx karma start karma.conf.js -logLevel=DEBUG","test":"npx karma start --browsers ChromeHeadless --single-run","test:windows":"karma start karma.conf.js","build":"webpack --config webpack.config.prod.js && webpack --config webpack.config.umd.js","lint":"npx tslint -p tsconfig.json --fix","docs":"npx typedoc --options"},"files":["dist","!dist/test"],"author":"","license":"ISC","dependencies":{"@stomp/stompjs":"^6.0.0","@uehreka/seriously":"^1.0.1","fast-text-encoding":"^1.0.0","intrinsic-scale":"^3.0.4","onnxruntime-web":"^1.15.1","promjs":"^0.4.1","ring-buffer-ts":"^1.2.0","rxjs":"^7.8.1","rxjs-compat":"^6.6.7","simple-peer":"^9.11.1","webrtc-adapter":"^8.2.3"},"devDependencies":{"@types/dom-mediacapture-record":"^1.0.16","@types/jasmine":"^2.8.8","@types/node":"^10.9.4","fetch-mock":"7.7.3","ignore-styles":"^5.0.1","jasmine":"^5.1.0","jasmine-core":"^5.1.0","karma":"^6.4.2","karma-chrome-launcher":"^3.2.0","karma-firefox-launcher":"^2.1.2","karma-jasmine":"^5.1.0","karma-jasmine-html-reporter":"^2.1.0","karma-requirejs":"^1.1.0","karma-typescript":"^5.5.4","karma-typescript-es6-transform":"^5.5.4","nock":"^9.6.1","requirejs":"^2.3.6","ts-loader":"^9.4.4","ts-node":"^7.0.1","tslint":"^5.11.0","tslint-no-focused-test":"^0.5.0","typedoc":"^0.18.0","typescript":"^5.1.6","webpack":"^5.88.2","webpack-cli":"^5.1.4"}}');
 
 /***/ }),
 /* 242 */
@@ -13312,11 +13340,16 @@ class API {
     avatarStart(data) {
         return this.postRequest('/api/v1/avatar/' + this._callId + '/start', data);
     }
-    avatarWelcome() {
-        return this.postRequest('/api/v1/avatar/' + this._callId + '/welcome');
+    avatarWelcome(metadata = {}) {
+        return this.postRequest('/api/v1/avatar/' + this._callId + '/welcome', {
+            metadata: JSON.stringify(metadata)
+        });
     }
-    avatarAsk(text) {
-        return this.postRequest('/api/v1/avatar/' + this._callId + '/ask', { text });
+    avatarAsk(text, metadata = {}) {
+        return this.postRequest('/api/v1/avatar/' + this._callId + '/ask', {
+            text,
+            metadata: JSON.stringify(metadata)
+        });
     }
     avatarStartRecording() {
         return this.postRequest('/api/v1/avatar/' + this._callId + '/audioCapture/start');
@@ -29928,7 +29961,7 @@ exports.DigitalHumanVideoPlayManager = DigitalHumanVideoPlayManager;
 "use strict";
 
 Object.defineProperty(exports, "__esModule", ({ value: true }));
-exports.DigitalHumanUnmuted = exports.DigitalHumanPlayedInMutedModeSuccess = exports.DigitalHumanFailedToPlay = exports.DigitalHumanVideoError = exports.ClientPerformanceMessage = exports.ClientMediaStreamUpdateMessage = exports.OnlineStatusUpdateMessage = exports.SessionInfoMessage = exports.AvatarRequestIgnored = exports.AvatarRequestCompleted = exports.SessionErrorMessage = exports.WebRtcStatsMessage = exports.WebRtcDataMessage = exports.SessionLiveMessage = exports.WarningMessage = exports.RecordingStoppedMessage = exports.RecordingStartedMessage = exports.DevicePermissionAllowedMessage = exports.AvatarTextInputFinishedMessage = exports.FinishedSpeakingMessage = exports.StartedSpeakingMessage = exports.UserStoppedSpeakingMessage = exports.UserStartedSpeakingMessage = exports.SpeechTranscriptionMessage = exports.AvatarQuestionMessage = exports.InstructionsMessage = exports.AvatarAnswerContentMessage = exports.AvatarAnswerMessage = exports.ResumeSessionUnavailable = exports.ConnectionLostMessage = exports.VoiceActivityInputModeNotSupportedMessage = exports.MicActivityNotSupportedMessage = exports.MicActivityMessage = exports.MicActivityErrorMessage = exports.ServiceUnavailableMessage = exports.AvatarUnavailableMessage = exports.AvatarAvailableMessage = exports.SessionEndedMessage = exports.SessionResumedMessage = exports.SessionPausedMessage = exports.DeviceListUpdatedMessage = exports.DeviceErrorMessage = exports.SetSpeakerSuccessMessage = exports.SetMicSuccessMessage = exports.SetCameraSuccessMessage = exports.DeviceNotFoundErrorMessage = exports.ReadyMessage = exports.UneeqMessageType = void 0;
+exports.CustomMetadataUpdated = exports.DigitalHumanUnmuted = exports.DigitalHumanPlayedInMutedModeSuccess = exports.DigitalHumanFailedToPlay = exports.DigitalHumanVideoError = exports.ClientPerformanceMessage = exports.ClientMediaStreamUpdateMessage = exports.OnlineStatusUpdateMessage = exports.SessionInfoMessage = exports.AvatarRequestIgnored = exports.AvatarRequestCompleted = exports.SessionErrorMessage = exports.WebRtcStatsMessage = exports.WebRtcDataMessage = exports.SessionLiveMessage = exports.WarningMessage = exports.RecordingStoppedMessage = exports.RecordingStartedMessage = exports.DevicePermissionAllowedMessage = exports.AvatarTextInputFinishedMessage = exports.FinishedSpeakingMessage = exports.StartedSpeakingMessage = exports.UserStoppedSpeakingMessage = exports.UserStartedSpeakingMessage = exports.SpeechTranscriptionMessage = exports.AvatarQuestionMessage = exports.InstructionsMessage = exports.AvatarAnswerContentMessage = exports.AvatarAnswerMessage = exports.ResumeSessionUnavailable = exports.ConnectionLostMessage = exports.VoiceActivityInputModeNotSupportedMessage = exports.MicActivityNotSupportedMessage = exports.MicActivityMessage = exports.MicActivityErrorMessage = exports.ServiceUnavailableMessage = exports.AvatarUnavailableMessage = exports.AvatarAvailableMessage = exports.SessionEndedMessage = exports.SessionResumedMessage = exports.SessionPausedMessage = exports.DeviceListUpdatedMessage = exports.DeviceErrorMessage = exports.SetSpeakerSuccessMessage = exports.SetMicSuccessMessage = exports.SetCameraSuccessMessage = exports.DeviceNotFoundErrorMessage = exports.ReadyMessage = exports.UneeqMessageType = void 0;
 var UneeqMessageType;
 (function (UneeqMessageType) {
     UneeqMessageType["Ready"] = "Ready";
@@ -29978,6 +30011,7 @@ var UneeqMessageType;
     UneeqMessageType["DigitalHumanFailedToPlay"] = "DigitalHumanFailedToPlay";
     UneeqMessageType["DigitalHumanPlayedInMutedModeSuccess"] = "DigitalHumanPlayedInMutedModeSuccess";
     UneeqMessageType["DigitalHumanUnmuted"] = "DigitalHumanUnmuted";
+    UneeqMessageType["CustomMetadataUpdated"] = "CustomMetadataUpdated";
 })(UneeqMessageType || (exports.UneeqMessageType = UneeqMessageType = {}));
 /**
  * Service is ready to be used
@@ -30454,6 +30488,13 @@ class DigitalHumanUnmuted {
     }
 }
 exports.DigitalHumanUnmuted = DigitalHumanUnmuted;
+class CustomMetadataUpdated {
+    constructor(chatMetadata) {
+        this.chatMetadata = chatMetadata;
+        this.uneeqMessageType = UneeqMessageType.CustomMetadataUpdated;
+    }
+}
+exports.CustomMetadataUpdated = CustomMetadataUpdated;
 
 
 /***/ }),
@@ -51246,6 +51287,7 @@ class SpeechHandler {
         if (this.stream && this.micVad) {
             this.stream.getTracks().forEach((track) => track.enabled = false);
             this.micVad.pause();
+            this.onVadSpeechEnd();
             return true;
         }
         logger_1.logger.warn(this.logPrefix + 'Could not pause speech recognition.');
@@ -51260,6 +51302,9 @@ class SpeechHandler {
         logger_1.logger.warn(this.logPrefix + 'Could not resume speech recognition.');
         return false;
     }
+    setChatMetadata(chatMetadata) {
+        this.options.chatMetadata = chatMetadata;
+    }
     loadScript(url) {
         return new Promise((resolve) => {
             const script = document.createElement('script');
@@ -51295,6 +51340,9 @@ class SpeechHandler {
                     this.stopRecognition();
                     this.ws.close();
                     break;
+                case MessageTypes_1.UneeqMessageType.CustomMetadataUpdated:
+                    this.options.chatMetadata = msg.chatMetadata;
+                    break;
                 default:
             }
         });
@@ -51480,7 +51528,8 @@ class SpeechHandler {
     handleTranscriptionResult(result) {
         logger_1.logger.log(this.logPrefix + `Speech transcription (${result.confidence} confidence): ${result.transcript}`);
         if (result.final) {
-            this.options.api.avatarAsk(result.transcript);
+            this.options.chatMetadata.userSpokenLocale = result.language_code;
+            this.options.api.avatarAsk(result.transcript, this.options.chatMetadata);
         }
         else if (this.digitalHumanSpeaking) {
             this.options.api.stopSpeaking();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "uneeq-js",
-  "version": "2.47.10",
+  "version": "2.48.0",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/src/index.d.ts",

package/readme.md

@@ -8,6 +8,12 @@ https://docs.uneeq.io/build-your-own-experience
 To use uneeq-js with a typescript version < 3.8, 'skipLibCheck' must be set to true.
 
 ## Release Notes
+#### 2.48.0
+* Add client and custom metadata that is sent during chats with a digital human. Developers can specify anything they wish in the custom metadata using the new function setCustomChatMetadata.
+
+#### 2.47.11
+* Speech Recognition: Bug fix: Pausing recording during active speech causes recognition state to be lost.
+
 #### 2.47.10
 * Speech Recognition: Added support for reconnecting speech recognition upon disconnection.