sentifyd-bot 1.4.0 → 1.5.0

package/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.5.0] - 2026-01-24
+
+### Added
+- **Azure Video Avatars**: Support for photorealistic video avatars powered by Azure Speech, as an alternative to 3D avatars.
+
+## [1.4.0] - 2026-01-XX
+
+### Added
+- Major UI improvement and various stability and performance improvements.
+
 ## [1.3.1] - 2025-12-18
 
 ### Changed

package/dist/AzureTalkingAvatar-DwFpnwON.js

@@ -0,0 +1 @@
+import{C as t,c as e}from"./index-DY_KYJkq.js";import{AvatarVideoFormat as n,AvatarConfig as i,AvatarSynthesizer as s,ResultReason as o}from"microsoft-cognitiveservices-speech-sdk";import{L as c}from"./LiveCaptions-CxsuwP4M.js";const a={BASE_URL:"/",DEV:!1,MODE:"production",PROD:!0,SSR:!1};class r{constructor(t={}){this.backendUrl=t.backendUrl||"",this.character=t.character||"lisa",this.style=t.style||"casual-sitting",this.voice=t.voice||"en-US-AvaNeural",this.speechConfig=t.speechConfig,this.auth=t.auth,this.peerConnection=null,this.avatarSynthesizer=null,this.videoElement=null,this.audioElement=null,this.onConnected=t.onConnected||(()=>{}),this.onDisconnected=t.onDisconnected||(()=>{}),this.onTrack=t.onTrack||(()=>{}),this.onError=t.onError||(t=>{}),this.onSessionLost=t.onSessionLost||(()=>{}),this.onWebRtcEvent=t.onWebRtcEvent||(()=>{}),this._connected=!1,this._connecting=!1,this._reconnecting=!1,this._permanentFailure=!1,this._wasThrottled=!1,this._eventDataChannel=null,this._onDataChannelHandler=null,this._connectPromise=null,this._connectionAttemptId=0,this._lastDisconnectTime=null,this._autoplayBlocked=!1}async _safePlayMediaElement(t,e){try{if(!t)return;const e=t.play?.();return e&&"function"==typeof e.then&&await e,!0}catch(n){return this._autoplayBlocked=!0,!1}}get wasAutoplayBlocked(){return this._autoplayBlocked}async resumeMediaPlayback(){if(!this._autoplayBlocked)return!0;let t=!0;if(this.videoElement?.paused&&this.videoElement?.srcObject){const e=await this._safePlayMediaElement(this.videoElement,"video");t=t&&e}if(this.audioElement?.paused&&this.audioElement?.srcObject){const e=await this._safePlayMediaElement(this.audioElement,"audio");t=t&&e}return t&&(this._autoplayBlocked=!1),t}get isConnected(){return this._connected}get isConnecting(){return this._connecting}get isPermanentlyFailed(){return this._permanentFailure}setVideoElement(t){this.videoElement=t}setAudioElement(t){this.audioElement=t}_buildApiUrl(t){try{const e=new URL(this.backendUrl);return e.pathname=e.pathname.replace(/\/$/,"")+t,e.toString()}catch(e){return this.backendUrl.split("?")[0].replace(/\/$/,"")+t}}async getIceToken(){if(this.auth?.callSentifydApi){const t=await this.auth.callSentifydApi("/api/azure-avatar/ice-token","GET");return t?.data??t}const t=this._buildApiUrl("/api/azure-avatar/ice-token"),e=await fetch(t);if(404===e.status)throw this._permanentFailure=!0,new Error("Azure Avatar API endpoint not found - backend may not support Azure Avatars");if(500===e.status){const t=await e.text().catch((()=>"Internal server error"));(t.includes("not implemented")||t.includes("not supported")||t.includes("not configured"))&&(this._permanentFailure=!0);const n=t.includes("not configured")?" (service not configured)":t.includes("not implemented")?" (not implemented)":t.includes("not supported")?" (not supported)":"";throw new Error(`Azure Avatar backend error${n}`)}if(501===e.status||503===e.status)throw this._permanentFailure=!0,new Error("Azure Avatar service not configured on backend");if(!e.ok)throw new Error(`Failed to get ICE token: ${e.status}`);return await e.json()}async connect(){return this._connectPromise||(this._connectPromise=(async()=>{if(this._connected)return this._connectPromise=null,!0;if(this._permanentFailure)return this._connectPromise=null,!1;this._connectionAttemptId++;const t=this._connectionAttemptId;this._connecting=!0;try{return await this._doConnect(t)}finally{this._connectionAttemptId===t&&(this._connectPromise=null)}})()),this._connectPromise}async _doConnect(t){try{if(!this.speechConfig)throw new Error("speechConfig is required to start AvatarSynthesizer");if(this._lastDisconnectTime){const t=5e3-(Date.now()-this._lastDisconnectTime);t>0&&await new Promise((e=>setTimeout(e,t)))}const t=await this.getIceToken(),c="true"===(a||{}).VITE_ENABLE_PUBLIC_STUN_FALLBACKS,r=(Array.isArray(t?.Urls)?t.Urls:[]).filter((t=>"string"==typeof t)).map((t=>t.trim())).filter((t=>/^(turns?):/i.test(t)));if(0===r.length)throw new Error("ICE token did not include any TURN URLs (expected turn:/turns: only)");const h=[{urls:r,username:t.Username,credential:t.Password}];c&&h.push({urls:"stun:stun.l.google.com:19302"},{urls:"stun:stun1.l.google.com:19302"},{urls:"stun:global.stun.twilio.com:3478"}),this.peerConnection=new RTCPeerConnection({iceServers:h,iceTransportPolicy:c?"all":"relay"}),this._onDataChannelHandler=t=>{try{const e=t?.channel;if(!e)return;this._eventDataChannel=e,e.onmessage=async t=>{try{const e=t?.data;let n=null;if("string"==typeof e?n=e:e&&"undefined"!=typeof Blob&&e instanceof Blob?n=await e.text():e instanceof ArrayBuffer&&(n=new TextDecoder("utf-8").decode(new Uint8Array(e))),!n)return;const i=JSON.parse(n),s=i?.event?.eventType||i?.eventType||null;if(!s)return;this.onWebRtcEvent({eventType:s,raw:i})}catch(e){}}}catch(e){}},this.peerConnection.addEventListener("datachannel",this._onDataChannelHandler);try{this.peerConnection.createDataChannel("eventChannel")}catch(e){}this.peerConnection.ontrack=t=>{"video"===t.track.kind&&this.videoElement?(this.videoElement.srcObject=t.streams[0],this._safePlayMediaElement(this.videoElement,"video")):"audio"===t.track.kind&&this.audioElement&&(this.audioElement.srcObject=t.streams[0],this._safePlayMediaElement(this.audioElement,"audio")),this.onTrack(t)},this.peerConnection.onconnectionstatechange=()=>{const t=this.peerConnection?.connectionState;"connected"===t?(this._connected=!0,this.onConnected()):"disconnected"!==t&&"failed"!==t||(this._connected=!1,this.onDisconnected())},this.peerConnection.oniceconnectionstatechange=()=>{},this.peerConnection.addTransceiver("video",{direction:"sendrecv"}),this.peerConnection.addTransceiver("audio",{direction:"sendrecv"}),this.speechConfig.speechSynthesisVoiceName=this.voice;const d=new n,l=new i(this.character,this.style,d);this.avatarSynthesizer=new s(this.speechConfig,l);const u=await this.avatarSynthesizer.startAvatarAsync(this.peerConnection);if(u?.reason===o.Canceled)throw new Error(u?.errorDetails||"Avatar start canceled");return this._connecting=!1,this._wasThrottled=!1,!0}catch(c){this._connecting=!1;try{await this.disconnect({suppressCallback:!0})}catch(e){}throw(c?.message?.includes("4429")||c?.message?.toLowerCase().includes("throttl")||c?.message?.toLowerCase().includes("concurrent request limit"))&&(this._wasThrottled=!0),this.onError(c),c}}async speak(t,e=null){if(!this.avatarSynthesizer)throw new Error("Cannot speak - AvatarSynthesizer not initialized");e&&(this.voice=e,this.speechConfig.speechSynthesisVoiceName=this.voice);const n=await this.avatarSynthesizer.speakTextAsync(t);if(n?.reason===o.Canceled)throw new Error(n?.errorDetails||"Speech synthesis canceled");return n}async stopSpeaking(){if(this.avatarSynthesizer)try{await this.avatarSynthesizer.stopSpeakingAsync()}catch(t){}}async disconnect(t={}){const{suppressCallback:e=!1}=t;this._connected=!1,this._connecting=!1;try{this._eventDataChannel&&(this._eventDataChannel.onmessage=null,this._eventDataChannel.close?.())}catch(n){}this._eventDataChannel=null;try{this.avatarSynthesizer&&(await this.avatarSynthesizer.stopAvatarAsync(),await this.avatarSynthesizer.close())}catch(i){}finally{this.avatarSynthesizer=null}try{if(this.peerConnection){if(this.peerConnection.ontrack=null,this.peerConnection.onconnectionstatechange=null,this.peerConnection.oniceconnectionstatechange=null,this._onDataChannelHandler)try{this.peerConnection.removeEventListener("datachannel",this._onDataChannelHandler)}catch(n){}this.peerConnection.close()}}catch(i){}finally{this.peerConnection=null,this._onDataChannelHandler=null}if(this.videoElement)try{this.videoElement.srcObject=null}catch(n){}if(this.audioElement)try{this.audioElement.srcObject=null}catch(n){}this._lastDisconnectTime=Date.now(),e||this.onDisconnected()}async reconnect(){if(this._permanentFailure)return!1;if(this._reconnecting)return!1;if(this._connectPromise)try{return await this._connectPromise}catch(t){}this._reconnecting=!0;try{await this.disconnect({suppressCallback:!0});const t=5e3,e=7e3,n=this._wasThrottled?e:t,i=this._lastDisconnectTime?Date.now()-this._lastDisconnectTime:0,s=Math.max(0,n-i);if(s>0&&await new Promise((t=>setTimeout(t,s))),!(await this.connect()))return!1;const o=async(t=1e4)=>{const e=Date.now();for(;Date.now()-e<t;){if(this._connected)return!0;const t=this.peerConnection?.connectionState;if("failed"===t||"closed"===t)return!1;await new Promise((t=>setTimeout(t,200)))}return this._connected};return await o()}catch(e){return!1}finally{this._reconnecting=!1}}}class h extends t{static isAzureAvatar=!0;isAzureAvatar=!0;isVideoAvatar=!0;constructor(t,e){super("talkingAvatar",{autoReconnect:!1}),this.bus=t,this.avatarOrchestrator=e,this.connection=null,this.videoElement=null,this.audioElement=null,this.currentTurnId=null,this.isStoppingSpeaking=!1,this._endedTurns=new Set,this.conversationStreamActive=!1,this._audioStartSent=!1,this._busHandlers=[],this._isSpeaking=!1,this._audioTrack=null,this._audioContext=null,this._audioAnalyser=null,this._audioSourceNode=null,this._speechMonitorIntervalId=null,this._speechMonitorState=null,this._speechCompletionTimeoutId=null,this._speechHardTimeoutId=null,this._lastWebRtcEventType=null,this._initializing=!1,this._initRetryCount=0,this._maxInitRetries=3,this._initRetryDelayMs=5e3,this._initRetryTimeoutId=null,this._pendingInitParams=null,this._reconnectRetryCount=0,this._maxReconnectRetries=3,this._reconnectRetryDelayMs=5e3,this._reconnectRetryTimeoutId=null,this._lastDisconnectTime=null,this.suspendTime=null,this._lastConnectedTime=null,this._connectionCooldownMs=5e3,this._idleDisconnectTimeoutId=null,this._idleDisconnectArmSeq=0,this._idleDisconnectArmId=null,this._idleDisconnectArmedAt=null,this._disconnectReason=null,this._disconnecting=!1,this._suspendGracePeriodTimeoutId=null,this._suspendGracePeriodMs=3e4,this._pendingSuspendReason=null,this._pendingMessage=null,this._readyToSpeak=!1,this.liveCaptions=new c,this.backendUrl=null,this.character="lisa",this.style="casual-sitting",this.voice="en-US-AvaNeural"}_videoDebugEnabled(){try{return!0===e?.videoAvatar?.debug||!0===e?.test?.enableDiagnostics}catch(t){return!1}}_logIdleTimer(t,e=null){this._videoDebugEnabled()}_isStreamActive(){const t=this.connection;if(!t)return!1;const e=t?.peerConnection?.connectionState,n=t?.peerConnection?.iceConnectionState;return!0===t.isConnected||"connected"===e||"connected"===n}_clearIdleDisconnectTimer(){if(!this._idleDisconnectTimeoutId)return;const t=this._idleDisconnectArmId,e=this._idleDisconnectArmedAt;try{clearTimeout(this._idleDisconnectTimeoutId)}catch(i){}this._idleDisconnectTimeoutId=null,this._idleDisconnectArmId=null,this._idleDisconnectArmedAt=null;const n="number"==typeof e?Date.now()-e:null;this._logIdleTimer("AzureTalkingAvatar: Cleared idle disconnect timer",{armId:t,elapsedMs:n})}_clearSuspendGracePeriodTimer(){if(this._suspendGracePeriodTimeoutId){try{clearTimeout(this._suspendGracePeriodTimeoutId)}catch(t){}this._suspendGracePeriodTimeoutId=null,this._pendingSuspendReason=null}}disarmIdleDisconnect(){this._clearIdleDisconnectTimer()}armIdleDisconnect(t=null){const n=e?.videoAvatar??{};if(!1===n.idleDisconnectEnabled)return;if(null!==t&&(!Number.isFinite(t)||t<=0))return;const i=Number(t??n.idleDisconnectMs??6e4);if(!Number.isFinite(i)||i<=0)return;this._clearIdleDisconnectTimer(),this._idleDisconnectArmSeq=(this._idleDisconnectArmSeq||0)+1,this._idleDisconnectArmId=`idle-${this._idleDisconnectArmSeq}-${Date.now()}`,this._idleDisconnectArmedAt=Date.now();const s=this._idleDisconnectArmId;this._logIdleTimer("AzureTalkingAvatar: Armed idle disconnect timer",{armId:s,idleMs:i,conversationStreamActive:this.conversationStreamActive,disconnectReason:this._disconnectReason,isStreamActive:this._isStreamActive?.()}),this._idleDisconnectTimeoutId=setTimeout((()=>{try{if(this._logIdleTimer("AzureTalkingAvatar: Idle disconnect timer fired",{armId:s,idleMs:i,conversationStreamActive:this.conversationStreamActive,disconnectReason:this._disconnectReason,isStreamActive:this._isStreamActive?.()}),!this._isStreamActive())return;this.disconnectForIdle().catch((()=>{}))}catch(t){}}),i)}async disconnectForIdle(){this._disconnectReason="idle",this.autoReconnect=!1,this._clearIdleDisconnectTimer(),this._clearSuspendGracePeriodTimer(),this.blocking=!1;try{await(this.interruptSpeaking?.())}catch(t){}try{this.connection&&await this.connection.disconnect()}catch(t){}this.setDisconnected();try{this.bus?.emit?.("event_status_Avatar_RenderingStateChanged",{running:!1,reason:"idle"})}catch(t){}try{this.bus?.emit?.("event_status_VideoAvatar_IdleDisconnected",{reason:"idle"})}catch(t){}}async disconnectForVisibility(t="visibility"){if(this.connection?.isConnected||this._isStreamActive()){this._disconnecting=!0,this._disconnectReason=t,"visibility"!==t&&(this.blocking=!1),this.autoReconnect=!1,this._clearIdleDisconnectTimer();try{await(this.interruptSpeaking?.())}catch(e){}try{this.connection&&await this.connection.disconnect()}catch(e){}this.setDisconnected(),this._lastDisconnectTime=Date.now(),this._disconnecting=!1;try{this.bus?.emit?.("event_status_Avatar_RenderingStateChanged",{running:!1,reason:t})}catch(e){}}else"visibility"!==t&&(this.blocking=!1)}async userReconnect(){if(this._disconnectReason=null,this.autoReconnect=!0,this.blocking=!0,this._clearIdleDisconnectTimer(),this._clearSuspendGracePeriodTimer(),!this.conversationStreamActive){const t=this.avatarOrchestrator?.getSnapshot?.()?.matches?.("lifecycle.conversation.inProgress");t&&(this.conversationStreamActive=!0)}await this.reconnect()}_stopSpeechMonitor(){try{this._speechMonitorIntervalId&&clearInterval(this._speechMonitorIntervalId)}catch(t){}this._speechMonitorIntervalId=null,this._speechMonitorState=null}_clearSpeechTimeouts(){try{this._speechCompletionTimeoutId&&clearTimeout(this._speechCompletionTimeoutId)}catch(t){}try{this._speechHardTimeoutId&&clearTimeout(this._speechHardTimeoutId)}catch(t){}this._speechCompletionTimeoutId=null,this._speechHardTimeoutId=null}_ensureAudioAnalyserFromStream(t){if(t&&!(this._audioAnalyser&&this._audioContext&&this._audioSourceNode))try{const e=window.AudioContext||window.webkitAudioContext;if(!e)return;this._audioContext=this._audioContext||new e;const n=this._audioContext.createMediaStreamSource(t),i=this._audioContext.createAnalyser();i.fftSize=1024,i.smoothingTimeConstant=.8,n.connect(i),this._audioSourceNode=n,this._audioAnalyser=i}catch(e){}}_startSpeechMonitor(t){if(this._stopSpeechMonitor(),!this._audioAnalyser)return;const e=new Uint8Array(this._audioAnalyser.fftSize),n={seenLoud:!1,lastLoudAt:Date.now(),startedAt:Date.now(),turnId:t};this._speechMonitorState=n;const i=()=>{this._audioAnalyser.getByteTimeDomainData(e);let t=0;for(let n=0;n<e.length;n++){const i=(e[n]-128)/128;t+=i*i}return Math.sqrt(t/e.length)};this._speechMonitorIntervalId=setInterval((()=>{try{if(!this._isSpeaking)return;const t=this.currentTurnId;if(n.turnId&&t&&n.turnId!==t)return;const e=i(),s=Date.now();if(e>=.005)return n.seenLoud=!0,void(n.lastLoudAt=s);n.seenLoud&&s-n.lastLoudAt>=800&&(this._isSpeaking=!1,this._stopSpeechMonitor(),this._clearSpeechTimeouts(),this.handleAudioEndOnce(n.turnId??t,"silence_detected"))}catch(t){this._stopSpeechMonitor()}}),100)}_armSpeechFailSafes(t,e){this._clearSpeechTimeouts();const n=Date.now();this._speechHardTimeoutId=setTimeout((()=>{const e=this.currentTurnId;this._isSpeaking&&(t&&e&&t!==e||(this._isSpeaking=!1,this._stopSpeechMonitor(),this._clearSpeechTimeouts(),this.handleAudioEndOnce(t??e,"hard_timeout")))}),9e4),this._speechMonitorState={...this._speechMonitorState||{},startedAt:n,turnId:t,textLength:"string"==typeof e?e.length:null}}async init(t,e,n={}){if(!this._initializing&&!this.connection?.isConnected){this._initializing=!0;try{if(this.connection){try{await this.connection.disconnect()}catch(i){}this.connection=null}this.backendUrl=t.azureBackendUrl||"http://localhost:5000",this.character=t.azureCharacter||"lisa",this.style=t.azureStyle||"casual-sitting",this.voice=t.voice||"en-US-AvaNeural",this._createVideoElements(e);const s=n?.speechConfig,o=n?.auth;if(!s)throw new Error("AzureTalkingAvatar requires speechConfig. Initialize VoiceBot before initializing the Azure avatar.");this.connection=new r({backendUrl:this.backendUrl,character:this.character,style:this.style,voice:this.voice,speechConfig:s,auth:o,onConnected:()=>this._onConnected(),onDisconnected:()=>this._onDisconnected(),onError:t=>this._onError(t),onTrack:t=>this._onTrack(t),onWebRtcEvent:t=>this._onWebRtcEvent(t),onSessionLost:()=>this._onSessionLost()}),this.connection.setVideoElement(this.videoElement),this.connection.setAudioElement(this.audioElement),await this.connection.connect(),this.addEventListeners(),this.setConnected()}catch(s){if((s?.message?.includes("4429")||s?.message?.toLowerCase().includes("throttl")||s?.message?.toLowerCase().includes("concurrent request limit"))&&this._initRetryCount<this._maxInitRetries){this._initRetryCount++;const i=this._initRetryDelayMs*this._initRetryCount;return this._pendingInitParams={avatarItem:t,avatarContainer:e,initOptions:n},this.blocking=!1,this.setConnecting(),void(this._initRetryTimeoutId=setTimeout((async()=>{if(this._initRetryTimeoutId=null,this._initializing=!1,this._pendingInitParams){const{avatarItem:e,avatarContainer:n,initOptions:i}=this._pendingInitParams;this._pendingInitParams=null,this.blocking=!0;try{await this.init(e,n,i)}catch(t){}}}),i))}throw this.setFailed(),new Error("Error initializing AzureTalkingAvatar: "+s.message)}finally{this._initRetryTimeoutId||(this._initializing=!1)}}}_createVideoElements(t){t.innerHTML="",t.style.cssText="\n            width: 100%;\n            height: 100%;\n            position: relative;\n            overflow: hidden;\n        ";const e=document.createElement("div");e.className="azure-avatar-wrapper",e.style.cssText="\n            width: 100%;\n            height: 100%;\n            position: absolute;\n            top: 0;\n            left: 0;\n            background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%);\n            overflow: hidden;\n        ",this.videoElement=document.createElement("video"),this.videoElement.autoplay=!0,this.videoElement.playsInline=!0,this.videoElement.muted=!1,this.videoElement.style.cssText="\n            width: 100%;\n            height: 100%;\n            object-fit: cover;\n            position: absolute;\n            top: 50%;\n            left: 50%;\n            transform: translate(-50%, -50%);\n            min-width: 100%;\n            min-height: 100%;\n        ",this.audioElement=document.createElement("audio"),this.audioElement.autoplay=!0,this.audioElement.style.display="none",e.appendChild(this.videoElement),e.appendChild(this.audioElement),t.appendChild(e)}_onConnected(){this.conversationStreamActive=!0,this._isSpeaking=!1,this._clearIdleDisconnectTimer();try{this.bus?.emit?.("event_status_AzureAvatar_WebRtcConnectionChanged",{connected:!0})}catch(t){}this._lastConnectedTime=Date.now(),this._initRetryCount=0,this._reconnectRetryCount=0,this._pendingInitParams=null,this._reconnectRetryTimeoutId&&(clearTimeout(this._reconnectRetryTimeoutId),this._reconnectRetryTimeoutId=null),this.blocking=!0,this.setConnected(),this._emitRenderingStateWhenVideoReady()}_emitRenderingStateWhenVideoReady(){const t=()=>{if(this.isRenderingActive())try{this.bus?.emit?.("event_status_Avatar_RenderingStateChanged",{running:!0,reason:"connected"})}catch(e){}else this.connection?.isConnected&&setTimeout(t,100)};t()}_onDisconnected(){"idle"===this._disconnectReason||"visibility"===this._disconnectReason||"session_lost"===this._disconnectReason||(this.conversationStreamActive=!1),this._isSpeaking=!1,this._clearIdleDisconnectTimer();try{this.bus?.emit?.("event_status_AzureAvatar_WebRtcConnectionChanged",{connected:!1})}catch(t){}this.blocking=!1,this.setDisconnected()}_onError(t){this._isSpeaking=!1,this._clearIdleDisconnectTimer(),this.bus?.emit?.("event_status_Avatar_Error",{error:t.message}),this.setFailed()}async _onSessionLost(){this._isSpeaking=!1,this._stopSpeechMonitor(),this._clearSpeechTimeouts(),this._clearIdleDisconnectTimer(),this._clearSuspendGracePeriodTimer(),this.disarmIdleDisconnect(),this._disconnectReason="session_lost",this.autoReconnect=!1,this.blocking=!0,this._reconnectRetryTimeoutId&&(clearTimeout(this._reconnectRetryTimeoutId),this._reconnectRetryTimeoutId=null);try{await(this.connection?.disconnect?.())}catch(t){}this._lastDisconnectTime=Date.now();try{this.bus?.emit?.("event_status_Avatar_Error",{error:"Avatar session lost. Click reconnect to resume.",recoverable:!0})}catch(t){}this.setDisconnected()}_onTrack(t){try{if("audio"===t?.track?.kind){this._audioTrack=t.track;const e=Array.isArray(t.streams)?t.streams[0]:null;e&&this._ensureAudioAnalyserFromStream(e),this._audioTrack.onended=()=>{try{this._isSpeaking&&(this._isSpeaking=!1,this._stopSpeechMonitor(),this.handleAudioEndOnce(this.currentTurnId,"track_ended"))}catch(t){}}}}catch(e){}}_onWebRtcEvent(t){try{const e=t?.eventType;if(!e)return;if(this._lastWebRtcEventType=e,"EVENT_TYPE_TURN_END"===e||"EVENT_TYPE_SWITCH_TO_IDLE"===e||"EVENT_TYPE_SESSION_END"===e){const t=this.currentTurnId;if(!t)return;if(this._endedTurns.has(t??"__null__"))return;this._isSpeaking=!1,this._stopSpeechMonitor(),this._clearSpeechTimeouts(),this.handleAudioEndOnce(t,`webrtc_${e.toLowerCase()}`)}}catch(e){}}isRenderingActive(){const t=this.connection;if(!t)return!1;const e=t?.peerConnection?.connectionState,n=t?.peerConnection?.iceConnectionState;if(!0!==t.isConnected&&"connected"!==e&&"connected"!==n)return!1;const i=this.videoElement;if(!i)return!1;const s=!!i.srcObject,o="number"!=typeof i.videoWidth||"number"!=typeof i.videoHeight||i.videoWidth>0&&i.videoHeight>0,c="number"!=typeof i.readyState||i.readyState>=2;return s&&c&&o}async startConversationStream(){if(this.conversationStreamActive||(this.conversationStreamActive=!0),!this.isRenderingActive()){this._disconnectReason=null,this._clearIdleDisconnectTimer(),this._clearSuspendGracePeriodTimer(),this.blocking=!0;try{await this.reconnect();const t=5e3,e=100;let n=0;for(;!this.isRenderingActive()&&n<t;)await new Promise((t=>setTimeout(t,e))),n+=e;this.isRenderingActive()}catch(t){}}}async stopConversationStream(){if(this.conversationStreamActive=!1,this.disarmIdleDisconnect(),this._clearSuspendGracePeriodTimer(),this.connection)try{await this.connection.disconnect()}catch(t){}this.bus?.emit?.("event_status_Avatar_RenderingStateChanged",{running:!1,reason:"conversation_ended"})}startTurn(t){this.currentTurnId=t,this._audioStartSent=!1,this._endedTurns.delete(t)}setPendingMessage(t,e){if(this._pendingMessage={text:t,turnId:e},this._readyToSpeak&&this.isRenderingActive()&&!this._isSpeaking){const t=this._pendingMessage;this._pendingMessage=null,this._readyToSpeak=!1,this.currentTurnId=t?.turnId??this.currentTurnId,this._audioStartSent=!1,this._endedTurns.delete(t?.turnId??"__null__"),this.speak(t?.text,t?.turnId)}}async startStreaming(t,e,n){this.currentTurnId=n,this.isStoppingSpeaking=!1,this._audioStartSent=!1,this._endedTurns.delete(n??"__null__"),this._readyToSpeak=!0;try{t?.cancel?.()}catch(i){}try{e?.cancel?.()}catch(i){}if(!this.isRenderingActive()){const t=5e3,e=100;let i=0;for(;!this.isRenderingActive()&&i<t;)await new Promise((t=>setTimeout(t,e))),i+=e;if(!this.isRenderingActive())return this.handleAudioEndOnce(n,"suppressed_timeout"),this.currentTurnId=null,void(this._pendingMessage=null)}if(this._pendingMessage&&this._pendingMessage.text){const{text:t}=this._pendingMessage;return this._pendingMessage=null,this._readyToSpeak=!1,void(await this.speak(t,n))}}async speak(t,e=null){if(!this.connection?.isConnected)return;const n=e||this.currentTurnId;this._audioStartSent=!1,this._endedTurns.delete(n??"__null__"),this._stopSpeechMonitor(),this._clearSpeechTimeouts();try{this._isSpeaking=!0,this._armSpeechFailSafes(n,t),this._startSpeechMonitor(n),this._notifyAudioStart(n),await this.connection.speak(t,this.voice),this._speechCompletionTimeoutId=setTimeout((()=>{const t=this.currentTurnId;this._isSpeaking&&(n&&t&&n!==t||(this._isSpeaking=!1,this._stopSpeechMonitor(),this._clearSpeechTimeouts(),this.handleAudioEndOnce(n??t,"completed_fallback")))}),1500)}catch(i){this._isSpeaking=!1,this._stopSpeechMonitor(),this._clearSpeechTimeouts(),this.handleAudioEndOnce(n,"error")}}startStreamPump(t){}startLipsyncPump(t){}_notifyAudioStart(t,e=null){if(this._audioStartSent)return;this._audioStartSent=!0;const n=t?{turnId:t,audioStartTime:e}:{audioStartTime:e||Date.now()},i=()=>{try{this.avatarOrchestrator?.sendEvent("AUDIO_START",n)}catch(t){}};"function"==typeof queueMicrotask?queueMicrotask(i):setTimeout(i,0)}handleAudioEndOnce(t,e){const n=t??"__null__";this._endedTurns.has(n)||(this._endedTurns.add(n),this._stopSpeechMonitor(),this._clearSpeechTimeouts(),this.avatarOrchestrator.sendEvent("AUDIO_END",{turnId:t,reason:e}))}async interruptSpeaking(){if(this.isStoppingSpeaking)return;if(this.isStoppingSpeaking=!0,this._isSpeaking=!1,this._stopSpeechMonitor(),this._clearSpeechTimeouts(),this.connection?.isConnected)try{await this.connection.stopSpeaking()}catch(e){}try{this.liveCaptions?.clearCaptions?.()}catch(n){}try{this.bus?.emit?.("event_command_UI_ClearCaptions")}catch(n){}const t=this.currentTurnId;t&&(this.handleAudioEndOnce(t,"interrupted"),this.currentTurnId=null),this.isStoppingSpeaking=!1}async prepareForNewUserTurn(){(this._isSpeaking||this.connection?.isConnected)&&await this.interruptSpeaking()}finalizeTurn(){}async suspendRendering(t="hidden"){if(!this.connection?.isConnected&&!this._isStreamActive())return"visibility"!==t&&(this.blocking=!1),void(this._disconnectReason=t);this._suspendGracePeriodTimeoutId?this._pendingSuspendReason=t:(this.suspendTime=Date.now(),this._pendingSuspendReason=t,this._suspendGracePeriodTimeoutId=setTimeout((async()=>{this._suspendGracePeriodTimeoutId=null;const e=this._pendingSuspendReason||t;this._pendingSuspendReason=null,await this.disconnectForVisibility(e)}),this._suspendGracePeriodMs))}async resumeRendering(t={}){if(!this._resumeInProgress){this._resumeInProgress=!0;try{await this._doResumeRendering(t)}finally{this._resumeInProgress=!1}}}async _doResumeRendering(t={}){const{forceReconnect:e=!1}=t,n=!!this._suspendGracePeriodTimeoutId,i=this._disconnectReason;if(n&&this._clearSuspendGracePeriodTimer(),this.blocking=!0,this._pendingSuspendReason=null,this._disconnectReason=null,n&&(this.connection?.isConnected||this._isStreamActive()))return;if(this._disconnecting){const t=2e3,e=50;let n=0;for(;this._disconnecting&&n<t;)await new Promise((t=>setTimeout(t,e))),n+=e}if((this.connection?.isConnected||this.connection?.isConnecting)&&this._isStreamActive())return;if(!this._isStreamActive())if(e||"visibility"===i||"conditions-change"===i)try{await this.reconnect({forceReconnect:!!e})}catch(r){if(this.conversationStreamActive)try{this.bus?.emit?.("event_status_VideoAvatar_IdleDisconnected",{reason:"reconnect_failed"})}catch(h){}}else if("idle"===i&&this.conversationStreamActive)try{this.bus?.emit?.("event_status_VideoAvatar_IdleDisconnected",{reason:"idle_persist"})}catch(h){}const s=Date.now(),o=!!(this.suspendTime&&s-this.suspendTime>15e3);this.suspendTime=null;const c=t=>new Promise((e=>setTimeout(e,t)));try{const t=this.isRenderingActive();!o&&t||this.bus?.emit?.("event_status_Avatar_RenderingStateChanged",{running:!1,reason:"resume_start"})}catch(h){}const a=async t=>{if(!t)return{played:!1,error:null};try{return await t.play(),{played:!t.paused,error:null}}catch(e){return{played:!1,error:e}}};try{const t=await a(this.videoElement);await a(this.audioElement),this.videoElement&&(this.videoElement.paused||this.videoElement.readyState<2)&&((t=>{if(t)try{const e=t.srcObject;if(!e)return;t.srcObject=null,t.srcObject=e}catch(h){}})(this.videoElement),await a(this.videoElement));let e=!1;if(o&&this.videoElement&&!this.videoElement.paused)try{const t=this.videoElement.currentTime;await c(250),e=!(this.videoElement.currentTime>t+.01)}catch(h){}const n=!(!this.videoElement||!(this.videoElement.paused||this.videoElement.readyState<2)),i=(()=>{const e=t?.error,n=e?.name;return"NotAllowedError"===n||"NotSupportedError"===n})(),s=this.connection?.peerConnection?.connectionState,r=this.connection?.peerConnection?.iceConnectionState,d="connected"===s&&("connected"===r||"completed"===r||void 0===r),l="disconnected"===s||"failed"===s||"closed"===s||"disconnected"===r||"failed"===r;if(null==this._disconnectReason&&!i&&(o||n||e)&&this.connection&&(l||!d))try{await Promise.race([this.connection.reconnect(),c(8e3)])}catch(h){}}finally{try{const t=this.isRenderingActive();this.bus?.emit?.("event_status_Avatar_RenderingStateChanged",{running:t,reason:"resume"})}catch(h){}}}get wasAutoplayBlocked(){return this.connection?.wasAutoplayBlocked??!1}resumeAudioContext(){this.connection?.resumeMediaPlayback?this.connection.resumeMediaPlayback().catch((()=>{})):(this.videoElement?.paused&&this.videoElement?.srcObject&&this.videoElement.play().catch((t=>{})),this.audioElement?.paused&&this.audioElement?.srcObject&&this.audioElement.play().catch((t=>{})))}getSpeechAnalyzerNode(){return null}onResize(){}setView(t){}addEventListeners(){const t=(t,e)=>{try{this.bus.on(t,e),this._busHandlers.push([t,e])}catch(n){}};t("event_command_Avatar_DisconnectForIdle",(async()=>{try{await this.disconnectForIdle()}catch(t){}})),t("event_command_Avatar_ReconnectAfterIdle",(async()=>{try{await this.userReconnect()}catch(t){}})),t("event_command_Avatar_UserReconnect",(async()=>{try{await this.userReconnect()}catch(t){}})),t("event_command_Avatar_SetAvatarView",(t=>{this.setView(t.view)})),t("event_command_Avatar_SetAvatarMood",(t=>{})),t("event_command_Avatar_PlayOrStopGesture",(t=>{})),t("event_command_Avatar_PlayOrStopAnimation",(t=>{})),t("event_command_Avatar_ControlAvatarLookAt",(t=>{})),t("event_command_UI_ClearCaptions",(()=>{try{this.liveCaptions?.clearCaptions?.()}catch(t){}}))}async reconnect(t={}){const e=!!t?.forceReconnect;if((this.conversationStreamActive||e)&&!this._initializing&&!this.connection?.isConnecting&&!this.connection?.isPermanentlyFailed)if(this.connection?.isConnected)this.setConnected();else if(this.connection){if(this._lastDisconnectTime){const t=Date.now()-this._lastDisconnectTime,e=5e3;if(t<e){const n=e-t;await new Promise((t=>setTimeout(t,n)))}}try{await this.connection.reconnect()?(this._reconnectRetryCount=0,this.setConnected()):this.connection.isPermanentlyFailed?this.setFailed():this._scheduleReconnectRetryIfThrottled()}catch(n){n?.message?.includes("4429")||n?.message?.toLowerCase().includes("throttl")||n?.message?.toLowerCase().includes("concurrent request limit")?this._scheduleReconnectRetryIfThrottled():this.connection.isPermanentlyFailed||this.setFailed()}}else this.setFailed()}_scheduleReconnectRetryIfThrottled(){if(this._reconnectRetryCount>=this._maxReconnectRetries)return void this.setFailed();this._reconnectRetryCount++;const t=this._reconnectRetryDelayMs*this._reconnectRetryCount;this.blocking=!1,this.setConnecting(),this._reconnectRetryTimeoutId&&clearTimeout(this._reconnectRetryTimeoutId),this._reconnectRetryTimeoutId=setTimeout((async()=>{this._reconnectRetryTimeoutId=null;try{await this.reconnect()}catch(t){}}),t)}destroy(){this._clearIdleDisconnectTimer(),this._clearSuspendGracePeriodTimer(),this._initializing=!1,this._initRetryTimeoutId&&(clearTimeout(this._initRetryTimeoutId),this._initRetryTimeoutId=null),this._pendingInitParams=null,this._reconnectRetryTimeoutId&&(clearTimeout(this._reconnectRetryTimeoutId),this._reconnectRetryTimeoutId=null),this._stopSpeechMonitor();try{this._audioTrack=null}catch(t){}try{this._audioSourceNode?.disconnect?.()}catch(t){}try{this._audioAnalyser=null}catch(t){}try{this._audioSourceNode=null}catch(t){}try{this._audioContext?.close?.()}catch(t){}this._audioContext=null;for(const[e,n]of this._busHandlers)try{this.bus.off(e,n)}catch(t){}this._busHandlers=[],this.connection&&(this.connection.disconnect().catch((()=>{})),this.connection=null),this.videoElement=null,this.audioElement=null,this.setDisconnected()}}export{h as AzureTalkingAvatar};

package/dist/LiveCaptions-CxsuwP4M.js

@@ -0,0 +1 @@
+import{c as t}from"./index-DY_KYJkq.js";import"socket.io-client";import"microsoft-cognitiveservices-speech-sdk";import"js-cookie";import"jwt-decode";import"xstate";const e={maxChars:t.captions.maxCharactersPerCaption,minChars:t.captions.minCharactersPerCaption,softChars:Math.floor(.75*t.captions.maxCharactersPerCaption),pauseBreakMs:350,preferBreakAfter:/[.!?…]|—|–|;/,softBreakAfter:/,|:/};class i{constructor(){this.currentCaptions=[],this.currentCaptionIndex=0,this.displayTimeoutId=null,this.durationTimeoutId=null,this.settleTimeoutId=null,this._playAttemptActive=!1}isBoundaryTypeValid(t){return"SpeechSynthesisBoundaryType.Word"===t.boundaryType||"SpeechSynthesisBoundaryType.Punctuation"===t.boundaryType}handleWordBoundaryEvent(t){if(!this.isBoundaryTypeValid(t))return;const i=t.text,s=t.audioOffset,r=t.duration;let n=this.currentCaptions[this.currentCaptions.length-1],a=null;if(n&&Number.isFinite(n._lastEndOffsetMs)&&(a=s-n._lastEndOffsetMs),!n||function(t,i,s){const r=t.text,n=r.length,a=e.preferBreakAfter.test(r.slice(-1)),o=e.softBreakAfter.test(r.slice(-1));return!!t.isDisplayed||null!=s&&s>=e.pauseBreakMs&&n>=e.minChars||(u=n)+(u?1:0)+i.length>e.maxChars||!!(a&&n>=e.minChars)||!!(n>=e.softChars&&o);var u}(n,i,a)){const t={text:i,audioOffset:s,duration:r,isDisplayed:!1,_lastEndOffsetMs:s+r,get renderText(){return(t=this.text)?` ${t.trim()} `:"  ";var t}};this.currentCaptions.push(t)}else n.text=(u=i,((o=n.text).length>0&&!/\s$/.test(o)&&!/^[\s.,!?;:…)]/.test(u)?o+" "+u:o+u).replace(/\s{2,}/g," ")),n.duration+=r,n._lastEndOffsetMs=s+r;var o,u;if(/^[\s.,!?;:…()\[\]{}'"—–-]$/.test(i)){const t=this.currentCaptions[this.currentCaptions.length-1];t&&(t.text=t.text.replace(/\s+([.,!?;:…)\]}])/g,"$1").replace(/([([{])\s+/g,"$1").replace(/\s{2,}/g," ").trim())}}playCaptions(t,e){this._playAttemptActive=!0;const i=async()=>{this._playAttemptActive&&(this.currentCaptions.length>0&&!this.currentCaptions[0].isDisplayed?(this._playAttemptActive=!1,this.startCaptionDisplay(t,e)):(await this.wait(100),i()))};i()}startCaptionDisplay(t,e){this.currentCaptionIndex=0,this.displayTimeoutId&&clearTimeout(this.displayTimeoutId),this.durationTimeoutId&&clearTimeout(this.durationTimeoutId),this.settleTimeoutId&&clearTimeout(this.settleTimeoutId),this.displayTimeoutId=null,this.durationTimeoutId=null,this.settleTimeoutId=null;const i=()=>{if(this.currentCaptionIndex>=this.currentCaptions.length)return void(this.settleTimeoutId=setTimeout((()=>{this.settleTimeoutId=null,this.currentCaptionIndex>=this.currentCaptions.length?t(!1):i()}),100));const s=this.currentCaptions[this.currentCaptionIndex];if(!s)return void t(!1);const r=Math.max(0,s.audioOffset-(performance.now()-e));this.displayTimeoutId=setTimeout((()=>{this.displayTimeoutId=null,this.currentCaptions[this.currentCaptionIndex]?(s.isDisplayed=!0,t(!0,s.text),this.durationTimeoutId=setTimeout((()=>{this.durationTimeoutId=null,this.currentCaptionIndex++,i()}),s.duration)):t(!1)}),r)};i()}clearCaptions(){this._playAttemptActive=!1,this.displayTimeoutId&&clearTimeout(this.displayTimeoutId),this.durationTimeoutId&&clearTimeout(this.durationTimeoutId),this.settleTimeoutId&&clearTimeout(this.settleTimeoutId),this.displayTimeoutId=null,this.durationTimeoutId=null,this.settleTimeoutId=null,this.currentCaptions=[],this.currentCaptionIndex=0}wait(t){return new Promise((e=>setTimeout(e,t)))}destroy(){this.clearCaptions()}}export{i as L};

package/dist/TalkingAvatar--N4jOsLi.js

@@ -0,0 +1 @@
+import{TalkingHead as t}from"@met4citizen/talkinghead";import{C as e,c as i}from"./index-DY_KYJkq.js";import{L as a}from"./LiveCaptions-CxsuwP4M.js";const n={dance:["dance\\M_Dances_007.fbx","dance\\M_Dances_008.fbx","dance\\M_Dances_009.fbx","dance\\M_Dances_011.fbx","dance\\F_Dances_001.fbx","dance\\F_Dances_004.fbx","dance\\F_Dances_005.fbx","dance\\F_Dances_006.fbx","dance\\F_Dances_007.fbx","dance\\M_Dances_001.fbx","dance\\M_Dances_002.fbx","dance\\M_Dances_003.fbx","dance\\M_Dances_004.fbx","dance\\M_Dances_005.fbx","dance\\M_Dances_006.fbx"],expression:["expression\\M_Standing_Expressions_016.fbx","expression\\M_Standing_Expressions_017.fbx","expression\\M_Standing_Expressions_018.fbx","expression\\M_Talking_Variations_001.fbx","expression\\M_Talking_Variations_002.fbx","expression\\M_Talking_Variations_003.fbx","expression\\M_Talking_Variations_004.fbx","expression\\M_Talking_Variations_005.fbx","expression\\M_Talking_Variations_006.fbx","expression\\M_Talking_Variations_007.fbx","expression\\M_Talking_Variations_008.fbx","expression\\M_Talking_Variations_009.fbx","expression\\M_Talking_Variations_010.fbx","expression\\F_Talking_Variations_001.fbx","expression\\F_Talking_Variations_002.fbx","expression\\F_Talking_Variations_003.fbx","expression\\F_Talking_Variations_004.fbx","expression\\F_Talking_Variations_005.fbx","expression\\F_Talking_Variations_006.fbx","expression\\M_Standing_Expressions_001.fbx","expression\\M_Standing_Expressions_002.fbx","expression\\M_Standing_Expressions_004.fbx","expression\\M_Standing_Expressions_005.fbx","expression\\M_Standing_Expressions_006.fbx","expression\\M_Standing_Expressions_007.fbx","expression\\M_Standing_Expressions_008.fbx","expression\\M_Standing_Expressions_009.fbx","expression\\M_Standing_Expressions_010.fbx","expression\\M_Standing_Expressions_011.fbx","expression\\M_Standing_Expressions_012.fbx","expression\\M_Standing_Expressions_013.fbx","expression\\M_Standing_Expressions_014.fbx","expression\\M_Standing_Expressions_015.fbx"],locomotion:["locomotion\\M_CrouchedWalk_Backwards_002.fbx","locomotion\\M_Falling_Idle_002.fbx","locomotion\\M_Jog_001.fbx","locomotion\\M_Jog_003.fbx","locomotion\\M_Jog_Backwards_001.fbx","locomotion\\M_Jog_Jump_001.fbx","locomotion\\M_Jog_Jump_002.fbx","locomotion\\M_Jog_Strafe_Left_001.fbx","locomotion\\M_Jog_Strafe_Right_001.fbx","locomotion\\M_Run_001.fbx","locomotion\\M_Run_Backwards_002.fbx","locomotion\\M_Run_Jump_001.fbx","locomotion\\M_Run_Jump_002.fbx","locomotion\\M_Run_Strafe_Left_002.fbx","locomotion\\M_Run_Strafe_Right_002.fbx","locomotion\\M_Walk_001.fbx","locomotion\\M_Walk_002.fbx","locomotion\\M_Walk_Backwards_001.fbx","locomotion\\M_Walk_Jump_001.fbx","locomotion\\M_Walk_Jump_002.fbx","locomotion\\M_Walk_Jump_003.fbx","locomotion\\M_Walk_Strafe_Left_002.fbx","locomotion\\M_Walk_Strafe_Right_002.fbx","locomotion\\F_Crouch_Strafe_Left.fbx","locomotion\\F_Crouch_Strafe_Right.fbx","locomotion\\F_Crouch_Walk_001.fbx","locomotion\\F_CrouchedWalk_Backwards_001.fbx","locomotion\\F_Falling_Idle_000.fbx","locomotion\\F_Falling_Idle_001.fbx","locomotion\\F_Jog_001.fbx","locomotion\\F_Jog_Backwards_001.fbx","locomotion\\F_Jog_Jump_Small_001.fbx","locomotion\\F_Jog_Strafe_Left_002.fbx","locomotion\\F_Jog_Strafe_Right_002.fbx","locomotion\\F_Run_001.fbx","locomotion\\F_Run_Backwards_001.fbx","locomotion\\F_Run_Jump_001.fbx","locomotion\\F_Run_Strafe_Left_001.fbx","locomotion\\F_Run_Strafe_Right_001.fbx","locomotion\\F_Walk_002.fbx","locomotion\\F_Walk_003.fbx","locomotion\\F_Walk_Backwards_001.fbx","locomotion\\F_Walk_Jump_001.fbx","locomotion\\F_Walk_Jump_002.fbx","locomotion\\F_Walk_Strafe_Left_001.fbx","locomotion\\F_Walk_Strafe_Right_001.fbx","locomotion\\M_Crouch_Strafe_Left_002.fbx","locomotion\\M_Crouch_Strafe_Right_002.fbx","locomotion\\M_Crouch_Walk_003.fbx"],idle:["idle\\M_Standing_Idle_Variations_001.fbx","idle\\M_Standing_Idle_Variations_002.fbx","idle\\M_Standing_Idle_Variations_003.fbx","idle\\M_Standing_Idle_Variations_004.fbx","idle\\M_Standing_Idle_Variations_005.fbx","idle\\M_Standing_Idle_Variations_006.fbx","idle\\M_Standing_Idle_Variations_007.fbx","idle\\M_Standing_Idle_Variations_008.fbx","idle\\M_Standing_Idle_Variations_009.fbx","idle\\M_Standing_Idle_Variations_010.fbx","idle\\F_Standing_Idle_001.fbx","idle\\F_Standing_Idle_Variations_001.fbx","idle\\F_Standing_Idle_Variations_002.fbx","idle\\F_Standing_Idle_Variations_003.fbx","idle\\F_Standing_Idle_Variations_004.fbx","idle\\F_Standing_Idle_Variations_005.fbx","idle\\F_Standing_Idle_Variations_006.fbx","idle\\F_Standing_Idle_Variations_007.fbx","idle\\F_Standing_Idle_Variations_008.fbx","idle\\F_Standing_Idle_Variations_009.fbx","idle\\M_Standing_Idle_001.fbx","idle\\M_Standing_Idle_002.fbx"]};function s(t){return t&&0!==t.length?t[Math.floor(Math.random()*t.length)]:null}function o(t,e,...i){return t.filter((t=>!!t.includes(`${e}_`)&&i.every((e=>t.includes(e)))))}const r=["eyeBlinkLeft","eyeLookDownLeft","eyeLookInLeft","eyeLookOutLeft","eyeLookUpLeft","eyeSquintLeft","eyeWideLeft","eyeBlinkRight","eyeLookDownRight","eyeLookInRight","eyeLookOutRight","eyeLookUpRight","eyeSquintRight","eyeWideRight","jawForward","jawLeft","jawRight","jawOpen","mouthClose","mouthFunnel","mouthPucker","mouthLeft","mouthRight","mouthSmileLeft","mouthSmileRight","mouthFrownLeft","mouthFrownRight","mouthDimpleLeft","mouthDimpleRight","mouthStretchLeft","mouthStretchRight","mouthRollLower","mouthRollUpper","mouthShrugLower","mouthShrugUpper","mouthPressLeft","mouthPressRight","mouthLowerDownLeft","mouthLowerDownRight","mouthUpperUpLeft","mouthUpperUpRight","browDownLeft","browDownRight","browInnerUp","browOuterUpLeft","browOuterUpRight","cheekPuff","cheekSquintLeft","cheekSquintRight","noseSneerLeft","noseSneerRight","tongueOut","headRotateZ"],_=["sil","aa","aa","O","E","RR","I","U","O","O","O","I","kk","RR","nn","SS","CH","TH","FF","DD","kk","PP"];class l extends e{isVideoAvatar=!1;constructor(t,e){super("talkingAvatar"),this.bus=t,this.avatarOrchestrator=e,this.talkingAvatar=null,this.isFullBody=!0,this.bodyGender="M",this.resetSpeakData(),this.prevViseme=null,this.frameDuration=1e3/i.constants.FRAME_RATE,this.currentTurnId=null,this.isStoppingSpeaking=!1,this._endedTurns=new Set,this.conversationStreamActive=!1,this._pendingEndReason=null,this._audioStartSent=!1,this._busHandlers=[],this.liveCaptions=new a}resetSpeakData(){this.speak={audio:[],visemes:[],vtimes:[],vdurations:[],words:[],wtimes:[],wdurations:[],markers:[],mtimes:[],anim:{}},this.azureBlendShapes={frames:[],sbuffer:[],orderBuffer:{}},this.prevViseme=null}finalizeTurn(){this.handlePendingVisemes(),this.talkingAvatar.streamNotifyEnd(),this.resetSpeakData()}async init(e,i){try{this.talkingAvatar=new t(i,{ttsEndpoint:"/gtts/",cameraZoomEnable:!1,cameraPanEnable:!1,cameraRotateEnable:!1,cameraView:"full",cameraY:.25,cameraDistance:.5,avatarMood:"happy"});const a="male"===e.gender?"M":"F";this.bodyGender=a,this.currentAvatarConfig={url:e.url,body:a},await this.talkingAvatar.showAvatar(this.currentAvatarConfig),this.addEventListeners(),this.setConnected(),this.lipsyncType=e.lipsyncType}catch(a){throw this.setFailed(),new Error("Error initializing TalkingAvatar: "+a)}}isRenderingActive(){return!!this.talkingAvatar?.isRunning}async suspendRendering(t="hidden"){if(this.isRenderingActive()){this.suspendTime=Date.now();try{this.talkingAvatar?.streamInterrupt?.()}catch(e){}try{this.audioReader?.cancel()}catch(e){}try{this.lipsyncReader?.cancel()}catch(e){}if(this.conversationStreamActive)try{await this.stopConversationStream()}catch(e){}try{this.talkingAvatar?.stop?.()}catch(e){}try{this.liveCaptions?.clearCaptions?.()}catch(e){}try{this.bus?.emit?.("event_command_UI_ClearCaptions")}catch(e){}try{const e=this.isRenderingActive();this.bus?.emit?.("event_status_Avatar_RenderingStateChanged",{running:e,reason:t})}catch(e){}}}async resumeRendering(){if(!this.isRenderingActive()){this.suspendTime&&Date.now()-this.suspendTime>15e3&&this.currentAvatarConfig&&await this.refreshAvatar();try{this.talkingAvatar?.start?.()}catch(t){}try{const t=this.isRenderingActive();this.bus?.emit?.("event_status_Avatar_RenderingStateChanged",{running:t,reason:"resume"})}catch(t){}}}async refreshAvatar(){if(!this.currentAvatarConfig)return!1;try{try{this.talkingAvatar?.streamInterrupt?.()}catch(t){}return await this.talkingAvatar.showAvatar(this.currentAvatarConfig),!0}catch(e){return!1}}resumeAudioContext(){try{const t=this.talkingAvatar?.audioCtx;if(!t)return;"suspended"!==t.state&&"interrupted"!==t.state||t.resume().then((()=>{})).catch((t=>{}))}catch(t){}}getSpeechAnalyzerNode(){return this.talkingAvatar?.audioAnalyzerNode??null}startStreamPump(t){try{this.audioReader=t.readable.getReader()}catch(i){return}const e=this.currentTurnId;(async()=>{try{for(;;){const{value:i,done:a}=await this.audioReader.read();if(a)return;if(this.currentTurnId!==e){try{this.audioReader.cancel()}catch(t){}return}this._notifyAudioStart(e),this.handleSpeechSynthesizingChunk({audio:i})}}catch(i){"AbortError"===i.name||i.message.includes("cancel")}})().catch((t=>{}))}startLipsyncPump(t){this.lipsyncReader=t.readable.getReader();const e=this.currentTurnId;(async()=>{try{for(;;){const{value:i,done:a}=await this.lipsyncReader.read();if(a)break;if(this.currentTurnId!==e){try{this.lipsyncReader.cancel()}catch(t){}break}switch(i.type){case"viseme":this.handleAddVisemeToStreaming(i);break;case"word":this.handleReceivedWordBoundaryEvent(i);break;case"blend":this.handleAddBlendShapestoStreaming(i)}}}catch(i){"AbortError"===i.name||i.message.includes("cancel")}})().catch((t=>{}))}handleAudioEndOnce(t,e){const i=t??"__null__";this._endedTurns.has(i)||(this._endedTurns.add(i),this.avatarOrchestrator.sendEvent("AUDIO_END",{turnId:t,reason:e}))}async startConversationStream(){if(this.isRenderingActive()&&!this.conversationStreamActive)try{await this.talkingAvatar.streamStart({lipsyncType:this.lipsyncType,sampleRate:24e3},(({turnId:t,audioStartTime:e}={})=>{const i=t??this.currentTurnId;this._notifyAudioStart(i,e)}),(()=>{const t=this._pendingEndReason||"completed";this._pendingEndReason=null,this.handleAudioEndOnce(this.currentTurnId,t),this.currentTurnId&&(this.currentTurnId=null)})),this.conversationStreamActive=!0}catch(t){throw this.conversationStreamActive=!1,t}}async stopConversationStream(){if(this.conversationStreamActive)try{this.talkingAvatar.streamStop(),this.conversationStreamActive=!1}catch(t){this.conversationStreamActive=!1}}addEventListeners(){const t=(t,e)=>{this.bus.on(t,e),this._busHandlers.push([t,e])};t("event_command_Avatar_SetAvatarView",(t=>{this.setView(t.view)})),t("event_command_Avatar_SetAvatarMood",(t=>{this.talkingAvatar.setMood(t.mood)})),t("event_command_Avatar_PlayOrStopGesture",(t=>{"stop"!==t.gesture?this.talkingAvatar.playGesture(t.gesture,t.duration,t.mirror,t.ms):this.talkingAvatar.stopGesture()})),t("event_command_Avatar_PlayOrStopAnimation",(t=>{if("stop"===t.animation)return void this.talkingAvatar.stopAnimation();const e=function(t,e="M"){let i=[];switch(t){case"dance":i=o(n.dance,e,"Dances");break;case"expression_talking":i=o(n.expression,e,"Talking_Variations");break;case"expression_standing":i=o(n.expression,e,"Standing_Expressions");break;case"idle_standing":i=o(n.idle,e,"Standing_Idle");break;case"locomotion_walk":i=o(n.locomotion,e,"Walk_");break;case"locomotion_walk_backwards":i=o(n.locomotion,e,"Walk_Backwards");break;case"locomotion_walk_jump":i=o(n.locomotion,e,"Walk_Jump");break;case"locomotion_walk_strafe_left":i=o(n.locomotion,e,"Walk_Strafe_Left");break;case"locomotion_walk_strafe_right":i=o(n.locomotion,e,"Walk_Strafe_Right");break;case"locomotion_run":i=o(n.locomotion,e,"Run_");break;case"locomotion_run_backwards":i=o(n.locomotion,e,"Run_Backwards");break;case"locomotion_run_jump":i=o(n.locomotion,e,"Run_Jump");break;case"locomotion_run_strafe_left":i=o(n.locomotion,e,"Run_Strafe_Left");break;case"locomotion_run_strafe_right":i=o(n.locomotion,e,"Run_Strafe_Right");break;case"locomotion_jog":i=o(n.locomotion,e,"Jog_");break;case"locomotion_jog_backwards":i=o(n.locomotion,e,"Jog_Backwards");break;case"locomotion_jog_jump":i=o(n.locomotion,e,"Jog_Jump");break;case"locomotion_jog_strafe_left":i=o(n.locomotion,e,"Jog_Strafe_Left");break;case"locomotion_jog_strafe_right":i=o(n.locomotion,e,"Jog_Strafe_Right");break;case"locomotion_crouch_walk":i=o(n.locomotion,e,"Crouch_Walk");break;case"locomotion_crouch_strafe_left":i=o(n.locomotion,e,"Crouch_Strafe_Left");break;case"locomotion_crouch_strafe_right":i=o(n.locomotion,e,"Crouch_Strafe_Right");break;case"locomotion_crouchedwalk_backwards":i=o(n.locomotion,e,"CrouchedWalk_Backwards");break;case"locomotion_falling_idle":i=o(n.locomotion,e,"Falling_Idle");break;default:i=[]}return s(i)||s(n.locomotion.filter((t=>t.includes(`${e}_Walk_`))))||"locomotion\\M_Walk_001.fbx"}(t.animation,this.bodyGender);this.playAnimation(e,t.dur)})),t("event_command_Avatar_ControlAvatarLookAt",(t=>{"ahead"!==t.target?"point"!==t.target?"camera"!==t.target?"eye_contact"!==t.target||this.talkingAvatar.makeEyeContact(t.t):this.talkingAvatar.lookAtCamera(t.t):this.talkingAvatar.lookAt(t.x,t.y,t.t):this.talkingAvatar.lookAhead(t.t)})),t("event_command_UI_ClearCaptions",(()=>{try{this.liveCaptions?.clearCaptions?.()}catch(t){}}))}onResize(){this.talkingAvatar&&this.talkingAvatar.onResize()}handleSpeechSynthesizingChunk(t){switch(this.lipsyncType){case"blendshapes":this.talkingAvatar.streamAudio({audio:t.audio,anims:this.azureBlendShapes?.sbuffer.splice(0,this.azureBlendShapes?.sbuffer.length)});break;case"visemes":this.talkingAvatar.streamAudio({audio:t.audio,visemes:this.speak.visemes.splice(0,this.speak.visemes.length),vtimes:this.speak.vtimes.splice(0,this.speak.vtimes.length),vdurations:this.speak.vdurations.splice(0,this.speak.vdurations.length)});break;case"words":this.talkingAvatar.streamAudio({audio:t.audio,words:this.speak.words.splice(0,this.speak.words.length),wtimes:this.speak.wtimes.splice(0,this.speak.wtimes.length),wdurations:this.speak.wdurations.splice(0,this.speak.wdurations.length)})}}handleAddVisemeToStreaming(t){const e=_[t?.visemeId||0],i=t?.audioOffset||0;if(this.prevViseme){const t=this.prevViseme;let e=i-t.vtime;e<40&&(e=40),this.speak.visemes.push(t.viseme),this.speak.vtimes.push(t.vtime),this.speak.vdurations.push(e)}this.prevViseme={viseme:e,vtime:i}}handleAddBlendShapestoStreaming(t){const e=t?.BlendShapes;if(!e||0===e.length)return;const i=this._getBlendColumns(),a=e.length,n=Object.create(null);for(let o=0;o<i.length;o++){const[t,s]=i[o],r=new Float32Array(a);for(let i=0;i<a;i++){const a=e[i];r[i]=Array.isArray(a)||ArrayBuffer.isView(a)?a[t]??0:0}n[s]=r}const s=null!=t?.AudioOffset?Math.round(t.AudioOffset/1e4):(t?.FrameIndex||0)*this.frameDuration;this.azureBlendShapes.sbuffer.push({name:"blendshapes",delay:s,dt:this._getDtArray(a),vs:n})}handleReceivedWordBoundaryEvent(t){try{this.liveCaptions.handleWordBoundaryEvent(t)}catch(s){}const e=this.speak,i=t.text,a=t.audioOffset,n=t.duration;"SpeechSynthesisBoundaryType.Punctuation"===t.boundaryType&&e.words.length?e.words[e.words.length-1]+=i:"SpeechSynthesisBoundaryType.Word"===t.boundaryType||"SpeechSynthesisBoundaryType.Punctuation"===t.boundaryType?(e.words.push(i),e.wtimes.push(a),e.wdurations.push(n)):"SpeechSynthesisBoundaryType.Sentence"===t.boundaryType&&a>500&&(e.markers.push((()=>{this.talkingAvatar.lookAtCamera(500)})),e.mtimes.push(a-500))}handlePendingVisemes(t=null){if("visemes"===this.lipsyncType&&this.prevViseme){const e=this.prevViseme;t||(t=100),this.speak.visemes.push(e.viseme),this.speak.vtimes.push(e.vtime),this.speak.vdurations.push(t),this.prevViseme=null}"visemes"===this.lipsyncType&&this.speak.visemes.length&&this.talkingAvatar.streamAudio({visemes:this.speak.visemes.splice(0,this.speak.visemes.length),vtimes:this.speak.vtimes.splice(0,this.speak.vtimes.length),vdurations:this.speak.vdurations.splice(0,this.speak.vdurations.length)}),"blendshapes"===this.lipsyncType&&this.azureBlendShapes.sbuffer.length&&this.talkingAvatar.streamAudio({anims:this.azureBlendShapes.sbuffer.splice(0,this.azureBlendShapes.sbuffer.length)})}async startStreaming(t,e,i){if(this.currentTurnId=i,this.isStoppingSpeaking=!1,this._audioStartSent=!1,!this.isRenderingActive()){try{t?.cancel?.()}catch(n){}try{e?.cancel?.()}catch(n){}return this.handleAudioEndOnce(i,"suppressed"),void(this.currentTurnId=null)}const a=i;this._endedTurns.delete(a??"__null__");try{this.startStreamPump(t),this.startLipsyncPump(e),this.conversationStreamActive||await this.startConversationStream()}catch(s){this.handleAudioEndOnce(a,"error"),this.currentTurnId===a&&(this.currentTurnId=null),this.isStoppingSpeaking=!1}}_notifyAudioStart(t,e=null){if(this._audioStartSent)return;this._audioStartSent=!0;const i=e??("undefined"!=typeof performance?performance.now():Date.now()),a=t??this.currentTurnId,n=a||0===a?{turnId:a,audioStartTime:i}:{audioStartTime:i},s=()=>{try{this.avatarOrchestrator?.sendEvent("AUDIO_START",n)}catch(t){}};"function"==typeof queueMicrotask?queueMicrotask(s):setTimeout(s,0)}interruptSpeaking(){if(!this.isStoppingSpeaking){this.isStoppingSpeaking=!0,this.conversationStreamActive&&this.talkingAvatar.streamInterrupt(),this.audioReader?.cancel(),this.lipsyncReader?.cancel(),this._pendingEndReason="interrupted";try{this.liveCaptions?.clearCaptions?.()}catch(t){}try{this.bus?.emit?.("event_command_UI_ClearCaptions")}catch(t){}this.isStoppingSpeaking=!1}}setView(t){switch(t){case"full":this.talkingAvatar.setView("full");break;case"mid":this.talkingAvatar.setView("mid");break;case"upper":this.talkingAvatar.setView("upper");break;case"head":this.talkingAvatar.setView("head")}}playAnimation(t,e){const i=`https://sentifydfrontend.blob.core.windows.net/animationclips/${t}`;this.setView("full"),this.talkingAvatar.playAnimation(i,null,e)}reconnect(){}async destroy(){for(const[t,e]of this._busHandlers)try{this.bus.off(t,e)}catch{}this._busHandlers.length=0,this.isStoppingSpeaking=!1,this.conversationStreamActive&&await this.stopConversationStream(),this.talkingAvatar?.stop(),this.talkingAvatar=null}_getBlendColumns(){if(!this._blendCols){this._blendCols=[];for(let t=0;t<r.length;t++){const e=r[t];e&&this._blendCols.push([t,e])}}return this._blendCols}_getDtArray(t){this._dtCache||(this._dtCache=new Map);let e=this._dtCache.get(t);return e||(e=new Array(t).fill(this.frameDuration),this._dtCache.set(t,e)),e}}export{l as TalkingAvatar};