uneeq-js 3.0.16-preview.99 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -39
- package/README.npm.md +15 -0
- package/README.repo.md +45 -0
- package/dist/591.index.js +1 -0
- package/dist/611.index.js +1 -0
- package/dist/deepgram-stt.d.ts +67 -0
- package/dist/deepgram-stt.spec.d.ts +1 -0
- package/dist/elevenlabs-stt.d.ts +68 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -1
- package/dist/lib/audio-speak.d.ts +7 -0
- package/dist/lib/audio-speak.spec.d.ts +1 -0
- package/dist/lib/logger.d.ts +3 -1
- package/dist/node_modules_ws_browser_js.index.js +22 -0
- package/dist/session-transfer-manager.d.ts +2 -2
- package/dist/session.d.ts +16 -8
- package/dist/signaling.d.ts +30 -13
- package/dist/speech-recognition.d.ts +6 -4
- package/dist/speech-recognition.spec.d.ts +1 -0
- package/dist/src/elevenlabs-stt.d.ts +57 -0
- package/dist/src/index.d.ts +8 -0
- package/dist/src/lib/audio-speak.d.ts +7 -0
- package/dist/src/lib/audio-speak.spec.d.ts +1 -0
- package/dist/src/session-transfer-manager.d.ts +10 -0
- package/dist/src/session-transfer-manager.spec.d.ts +1 -0
- package/dist/src/session.d.ts +55 -0
- package/dist/src/session.spec.d.ts +1 -0
- package/dist/src/signaling.d.ts +64 -0
- package/dist/src/signaling.spec.d.ts +1 -0
- package/dist/src/types/UneeqConfig.d.ts +138 -0
- package/dist/src/uneeq.d.ts +26 -0
- package/dist/src/uneeq.spec.d.ts +1 -0
- package/dist/stats.d.ts +11 -0
- package/dist/stats.spec.d.ts +1 -0
- package/dist/types/SessionEnded.d.ts +7 -6
- package/dist/types/SpeechRecognitionInterface.d.ts +12 -0
- package/dist/types/SpeechStreamEvents.d.ts +5 -0
- package/dist/types/UneeqConfig.d.ts +104 -0
- package/dist/types/UneeqErrorCodes.d.ts +1 -0
- package/dist/types/UneeqMessages.d.ts +60 -282
- package/dist/types/dhop-messages.d.ts +3 -1
- package/dist/uneeq.d.ts +6 -2
- package/dist/uneeq.spec.d.ts +1 -0
- package/dist/vendors-node_modules_elevenlabs_client_dist_lib_modern_js.index.js +32 -0
- package/dist/webrtc-data-channel/DataChannelAction.d.ts +3 -1
- package/dist/webrtc-data-channel/DataChannelActionType.d.ts +4 -2
- package/dist/webrtc-data-channel/DataChannelResponse.d.ts +4 -1
- package/dist/webrtc-data-channel/messages/CameraAnchor.d.ts +2 -1
- package/dist/webrtc-data-channel/messages/ChatPrompt.d.ts +2 -1
- package/dist/webrtc-data-channel/messages/InterimTranscript.d.ts +20 -0
- package/dist/webrtc-data-channel/messages/Speak.d.ts +2 -1
- package/dist/webrtc-data-channel/messages/SpeechStream.d.ts +21 -0
- package/dist/webrtc-data-channel/messages/StopSpeaking.d.ts +2 -0
- package/dist/webrtc-data-channel/messages/UserSpeaking.d.ts +2 -1
- package/package.json +3 -2
- package/dist/constants/reconnection-ui-style.d.ts +0 -1
- package/dist/constants/video-container-style.d.ts +0 -1
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Session } from "../session";
|
|
2
|
+
/**
|
|
3
|
+
* Sends audio data over the session's signaling channel.
|
|
4
|
+
* @param session The session instance (must have audioStreamId and signaling)
|
|
5
|
+
* @param audio The audio data (base64 string, Uint8Array, or Blob)
|
|
6
|
+
*/
|
|
7
|
+
export declare function speakAudioSend(session: Session | undefined, audio: string | Uint8Array | Blob): Promise<void>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/lib/logger.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import pino from
|
|
1
|
+
import pino from "pino";
|
|
2
2
|
export default class Logger {
|
|
3
3
|
private static instance;
|
|
4
|
+
private static formatLogMessage;
|
|
4
5
|
static fatal(log: string, ...args: unknown[]): void;
|
|
5
6
|
static error(log: string, ...args: unknown[]): void;
|
|
6
7
|
static warn(log: string, ...args: unknown[]): void;
|
|
@@ -9,4 +10,5 @@ export default class Logger {
|
|
|
9
10
|
static trace(log: string, ...args: unknown[]): void;
|
|
10
11
|
static getLogger(): pino.Logger;
|
|
11
12
|
static setLevel(level: string): void;
|
|
13
|
+
static serialiseError(err: unknown): string;
|
|
12
14
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* ATTENTION: The "eval" devtool has been used (maybe by default in mode: "development").
|
|
4
|
+
* This devtool is neither made for production nor for readable output files.
|
|
5
|
+
* It uses "eval()" calls to create a separate source file in the browser devtools.
|
|
6
|
+
* If you are trying to read the output file, select a different devtool (https://webpack.js.org/configuration/devtool/)
|
|
7
|
+
* or disable the default devtool with "devtool: false".
|
|
8
|
+
* If you are looking for production-ready output files, see mode: "production" (https://webpack.js.org/configuration/mode/).
|
|
9
|
+
*/
|
|
10
|
+
(Object(typeof self !== "undefined" ? self : this)["webpackChunkUneeq"] = Object(typeof self !== "undefined" ? self : this)["webpackChunkUneeq"] || []).push([["node_modules_ws_browser_js"],{
|
|
11
|
+
|
|
12
|
+
/***/ "./node_modules/ws/browser.js":
|
|
13
|
+
/*!************************************!*\
|
|
14
|
+
!*** ./node_modules/ws/browser.js ***!
|
|
15
|
+
\************************************/
|
|
16
|
+
/***/ ((module) => {
|
|
17
|
+
|
|
18
|
+
eval("\n\nmodule.exports = function () {\n throw new Error(\n 'ws does not work in the browser. Browser clients must use the native ' +\n 'WebSocket object'\n );\n};\n\n\n//# sourceURL=webpack://Uneeq/./node_modules/ws/browser.js?");
|
|
19
|
+
|
|
20
|
+
/***/ })
|
|
21
|
+
|
|
22
|
+
}]);
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { type Session } from "./session";
|
|
2
2
|
import { type Signaling } from "./signaling";
|
|
3
3
|
export declare class SessionTransferManager {
|
|
4
|
-
session
|
|
4
|
+
private readonly session;
|
|
5
5
|
isTransferSessionInProgress: boolean;
|
|
6
6
|
constructor(session: Session);
|
|
7
7
|
private addTransferSessionListener;
|
|
8
|
-
|
|
8
|
+
transferSession(): void;
|
|
9
9
|
switchLiveVideo(videoContainer: HTMLDivElement, newSignaling: Signaling): void;
|
|
10
10
|
}
|
package/dist/session.d.ts
CHANGED
|
@@ -1,46 +1,54 @@
|
|
|
1
1
|
import { Signaling } from "./signaling";
|
|
2
2
|
import { type UneeqConfig } from "./types/UneeqConfig";
|
|
3
3
|
import { type UneeqMessage } from "./types/UneeqMessages";
|
|
4
|
-
import {
|
|
4
|
+
import { type SpeechRecognitionInterface } from "./types/SpeechRecognitionInterface";
|
|
5
5
|
import { Subject } from "rxjs";
|
|
6
6
|
import { type DataChannelMessage } from "./webrtc-data-channel/DataChannelMessage";
|
|
7
7
|
import { type PromptMetadata } from "./types/PromptMetadata";
|
|
8
8
|
import { type CameraAnchorDistance, type CameraAnchorHorizontal } from "./webrtc-data-channel/messages/CameraAnchor";
|
|
9
|
+
import { SessionTransferManager } from "./session-transfer-manager";
|
|
10
|
+
export declare const VIDEO_REVEAL_DELAY_MS = 750;
|
|
9
11
|
export declare class Session {
|
|
10
12
|
readonly config: UneeqConfig;
|
|
11
13
|
id: string;
|
|
12
14
|
signaling?: Signaling;
|
|
13
15
|
messages: Subject<UneeqMessage>;
|
|
14
|
-
speechRec?:
|
|
16
|
+
speechRec?: SpeechRecognitionInterface;
|
|
15
17
|
promptMetadata: PromptMetadata;
|
|
18
|
+
audioStreamId?: number;
|
|
19
|
+
readonly sessionTransferManager: SessionTransferManager;
|
|
16
20
|
private jwt;
|
|
17
21
|
private welcomeRequested;
|
|
18
|
-
private
|
|
22
|
+
private resolutionLogged;
|
|
19
23
|
constructor(config: UneeqConfig);
|
|
20
24
|
updatePromptMetadata(metadata: object): void;
|
|
21
25
|
setEnableMicrophone(enable: boolean): void;
|
|
22
26
|
setCameraAnchorDistance(position: CameraAnchorDistance, duration?: number, signaling?: Signaling | undefined): void;
|
|
23
27
|
setCameraAnchorHorizontal(position: CameraAnchorHorizontal, duration?: number, signaling?: Signaling | undefined): void;
|
|
24
28
|
createSession(): void;
|
|
25
|
-
createSignaling(videoElement?: HTMLDivElement
|
|
29
|
+
createSignaling(videoElement?: HTMLDivElement): Signaling;
|
|
26
30
|
private signalingEventHandler;
|
|
27
31
|
private handleSessionReady;
|
|
28
32
|
private initSpeechRecognition;
|
|
33
|
+
private createGoogleSTT;
|
|
34
|
+
private createDeepgramSTT;
|
|
29
35
|
protected getSpeechOptions(jwt: string): {
|
|
30
36
|
apiUrl: string;
|
|
37
|
+
assetBasePath: string | undefined;
|
|
38
|
+
enableInterrupt: boolean | undefined;
|
|
39
|
+
enableVad: boolean | undefined;
|
|
31
40
|
jwtToken: string;
|
|
32
41
|
hintPhrases: string | undefined;
|
|
33
42
|
hintPhrasesBoost: number | undefined;
|
|
34
43
|
locales: string | undefined;
|
|
35
44
|
messages: Subject<UneeqMessage>;
|
|
36
|
-
sessionId: string;
|
|
37
|
-
assetBasePath: string | undefined;
|
|
38
45
|
promptMetadata: PromptMetadata;
|
|
39
|
-
|
|
40
|
-
enableInterrupt: boolean | undefined;
|
|
46
|
+
sessionId: string;
|
|
41
47
|
sendMessage: (msg: DataChannelMessage) => void;
|
|
42
48
|
};
|
|
43
49
|
private initMetadata;
|
|
44
50
|
private handleDataChannelMessage;
|
|
51
|
+
private handleChannelMessageError;
|
|
45
52
|
private sendWelcomePrompt;
|
|
53
|
+
private listenForTabClosure;
|
|
46
54
|
}
|
package/dist/signaling.d.ts
CHANGED
|
@@ -1,27 +1,31 @@
|
|
|
1
|
-
import { ReplaySubject
|
|
1
|
+
import { ReplaySubject } from 'rxjs';
|
|
2
2
|
import { PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4';
|
|
3
3
|
import { type SignalingEvent } from './types/SignalingEvents';
|
|
4
4
|
import { type DataChannelMessage } from './webrtc-data-channel/DataChannelMessage';
|
|
5
|
-
import { type
|
|
6
|
-
import { SessionEndedReason } from './types/SessionEnded';
|
|
5
|
+
import { type Session } from './session';
|
|
7
6
|
export declare class Signaling {
|
|
7
|
+
private readonly session;
|
|
8
8
|
private readonly connectionUrl;
|
|
9
9
|
videoContainerElement?: HTMLDivElement | undefined;
|
|
10
|
-
private readonly clientMessages?;
|
|
11
|
-
isStreamTransferring: boolean;
|
|
12
|
-
forceTURN: boolean;
|
|
13
10
|
stream?: PixelStreaming;
|
|
14
11
|
messages: ReplaySubject<SignalingEvent>;
|
|
15
|
-
|
|
12
|
+
videoElement?: HTMLVideoElement;
|
|
16
13
|
audioElement?: HTMLAudioElement;
|
|
17
14
|
videoInitialized: boolean;
|
|
18
15
|
dataChannelOpen: boolean;
|
|
16
|
+
sceneReady: boolean;
|
|
19
17
|
private readonly maxReconnectAttempts;
|
|
20
18
|
private reconnectionAttempts;
|
|
21
19
|
private allowReconnection;
|
|
22
20
|
private isReconnectInProgress;
|
|
23
|
-
private
|
|
24
|
-
|
|
21
|
+
private audioStatsInternal?;
|
|
22
|
+
private freezeCount;
|
|
23
|
+
private lastDecodedFrames;
|
|
24
|
+
private lastCheckTime;
|
|
25
|
+
private audioStatsAnalyzer?;
|
|
26
|
+
private readonly config;
|
|
27
|
+
private readonly clientMessages;
|
|
28
|
+
constructor(session: Session, connectionUrl: string, videoContainerElement?: HTMLDivElement | undefined);
|
|
25
29
|
init(): void;
|
|
26
30
|
private addStreamHandlers;
|
|
27
31
|
private addWebSocketHandlers;
|
|
@@ -29,19 +33,32 @@ export declare class Signaling {
|
|
|
29
33
|
private handleVideoInitialized;
|
|
30
34
|
playAudioElement(): void;
|
|
31
35
|
pauseAudioElement(): void;
|
|
36
|
+
/**
|
|
37
|
+
* Handle the case where the audio element is not found.
|
|
38
|
+
* Log an error and send a message to the client.
|
|
39
|
+
*
|
|
40
|
+
* TODO: In future make this a fatal error that will end the session, when we give users more information about errors
|
|
41
|
+
*/
|
|
42
|
+
private handleMissingAudioElement;
|
|
43
|
+
startAudioStatsMonitor(intervalMs?: number): void;
|
|
44
|
+
stopAudioStatsMonitor(): void;
|
|
32
45
|
private handleDataChannelOpen;
|
|
33
46
|
private handleWebRtcFailed;
|
|
34
47
|
private handleWebRtcDisconnected;
|
|
35
48
|
private handleWebRtcStats;
|
|
49
|
+
private checkForFreeze;
|
|
36
50
|
private reconnectIfSafe;
|
|
37
51
|
private handleDataChannelMessage;
|
|
38
52
|
private handleIndexInQueueMessage;
|
|
39
53
|
private handleRendererTakenMessage;
|
|
40
54
|
private handleRendererGoneEvent;
|
|
41
|
-
private
|
|
42
|
-
hideReconnectionUi(): void;
|
|
43
|
-
showReconnectionUi(): void;
|
|
55
|
+
private handleSwitchRendererRequestEvent;
|
|
44
56
|
socketSend(msg: any): void;
|
|
45
57
|
dataSend(data: DataChannelMessage): void;
|
|
46
|
-
|
|
58
|
+
dataSendBinary(data: Uint8Array): void;
|
|
59
|
+
private uint8ToBase64;
|
|
60
|
+
endSession(reason?: string, sendMessageToClient?: boolean): void;
|
|
61
|
+
streamDisconnect(): void;
|
|
62
|
+
startSoftSwitch(): void;
|
|
63
|
+
stopSoftSwitch(): void;
|
|
47
64
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { MicVAD } from './lib/vad/real-time-vad';
|
|
1
2
|
import { type SpeechRecognitionOptions } from './types/SpeechHandlerOptions';
|
|
2
3
|
import { type PromptMetadata } from './types/PromptMetadata';
|
|
3
4
|
export declare class SpeechRecognition {
|
|
@@ -5,17 +6,17 @@ export declare class SpeechRecognition {
|
|
|
5
6
|
private ws;
|
|
6
7
|
private mediaRecorder;
|
|
7
8
|
private readonly speechBuffer;
|
|
8
|
-
private micVad;
|
|
9
9
|
private recordingLive;
|
|
10
10
|
private headerBlob;
|
|
11
11
|
private scriptsLoadedPromise;
|
|
12
|
-
private digitalHumanSpeaking;
|
|
13
|
-
private stream?;
|
|
14
12
|
private reconnectWs;
|
|
15
13
|
private wsReconnectMessageQueue;
|
|
16
14
|
private storedTranscription?;
|
|
17
15
|
private awaitingFinalTranscript;
|
|
18
|
-
|
|
16
|
+
stream?: MediaStream;
|
|
17
|
+
micVad: MicVAD;
|
|
18
|
+
digitalHumanSpeaking: boolean;
|
|
19
|
+
interruptionBlocked: boolean;
|
|
19
20
|
constructor(options: SpeechRecognitionOptions);
|
|
20
21
|
startRecognition(): void;
|
|
21
22
|
stopRecognition(): void;
|
|
@@ -50,6 +51,7 @@ export declare class SpeechRecognition {
|
|
|
50
51
|
private handlePttTranscriptionResult;
|
|
51
52
|
private handleVadTranscriptionResult;
|
|
52
53
|
private sendChatPrompt;
|
|
54
|
+
private sendInterimTranscription;
|
|
53
55
|
private dataChannelMsgSend;
|
|
54
56
|
private clientMsgSend;
|
|
55
57
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { type Subject } from 'rxjs';
|
|
2
|
+
import { type UneeqMessage } from './types/UneeqMessages';
|
|
3
|
+
import { type DataChannelMessage } from './webrtc-data-channel/DataChannelMessage';
|
|
4
|
+
import { type PromptMetadata } from './types/PromptMetadata';
|
|
5
|
+
export interface ElevenLabsSTTOptions {
|
|
6
|
+
tokenEndpoint: string;
|
|
7
|
+
jwtToken: string;
|
|
8
|
+
sessionId: string;
|
|
9
|
+
modelId?: string;
|
|
10
|
+
languageCode?: string;
|
|
11
|
+
audioFormat?: 'pcm_8000' | 'pcm_16000' | 'pcm_22050' | 'pcm_24000' | 'pcm_44100' | 'pcm_48000' | 'ulaw_8000';
|
|
12
|
+
vadSilenceThresholdSecs?: number;
|
|
13
|
+
vadThreshold?: number;
|
|
14
|
+
minSpeechDurationMs?: number;
|
|
15
|
+
minSilenceDurationMs?: number;
|
|
16
|
+
echoCancellation?: boolean;
|
|
17
|
+
noiseSuppression?: boolean;
|
|
18
|
+
autoGainControl?: boolean;
|
|
19
|
+
telemetryEnabled?: boolean;
|
|
20
|
+
promptMetadata: PromptMetadata;
|
|
21
|
+
messages: Subject<UneeqMessage>;
|
|
22
|
+
sendMessage: (msg: DataChannelMessage) => void;
|
|
23
|
+
enableInterrupt?: boolean;
|
|
24
|
+
}
|
|
25
|
+
export declare class ElevenLabsSTT {
|
|
26
|
+
private readonly options;
|
|
27
|
+
private scribe;
|
|
28
|
+
private isConnected;
|
|
29
|
+
private isMuted;
|
|
30
|
+
private isReconnecting;
|
|
31
|
+
private shouldReconnect;
|
|
32
|
+
private digitalHumanSpeaking;
|
|
33
|
+
private interruptionBlocked;
|
|
34
|
+
private currentTranscript;
|
|
35
|
+
private isUserCurrentlySpeaking;
|
|
36
|
+
constructor(options: ElevenLabsSTTOptions);
|
|
37
|
+
startRecognition(): Promise<void>;
|
|
38
|
+
stopRecognition(): Promise<void>;
|
|
39
|
+
pause(): Promise<boolean>;
|
|
40
|
+
resume(): Promise<boolean>;
|
|
41
|
+
setChatMetadata(chatMetadata: PromptMetadata): void;
|
|
42
|
+
pauseVadIfInterruptNotAllowed(): void;
|
|
43
|
+
private connect;
|
|
44
|
+
private disconnect;
|
|
45
|
+
private getToken;
|
|
46
|
+
private setupEventHandlers;
|
|
47
|
+
private handleAppMessages;
|
|
48
|
+
private handlePartialTranscript;
|
|
49
|
+
private handleCommittedTranscript;
|
|
50
|
+
private handleConnectionOpen;
|
|
51
|
+
private handleConnectionClose;
|
|
52
|
+
private handleError;
|
|
53
|
+
private sendChatPrompt;
|
|
54
|
+
private handleSpeakingEnd;
|
|
55
|
+
private dataChannelMsgSend;
|
|
56
|
+
private clientMsgSend;
|
|
57
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export type { UneeqConfig } from './types/UneeqConfig';
|
|
2
|
+
export { SpeechRecognitionProvider } from './types/UneeqConfig';
|
|
3
|
+
export * from './types/UneeqMessages';
|
|
4
|
+
export { Uneeq } from './uneeq';
|
|
5
|
+
export type { SpeechTranscriptionResult } from './types/SpeechTranscriptionResult';
|
|
6
|
+
export { CameraAnchorDistance, CameraAnchorHorizontal } from './webrtc-data-channel/messages/CameraAnchor';
|
|
7
|
+
export { LogLevel } from './types/LogLevel';
|
|
8
|
+
export type { ElevenLabsSTTOptions } from './elevenlabs-stt';
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Session } from "../session";
|
|
2
|
+
/**
|
|
3
|
+
* Sends audio data over the session's signaling channel.
|
|
4
|
+
* @param session The session instance (must have audioStreamId and signaling)
|
|
5
|
+
* @param audio The audio data (base64 string, Uint8Array, or Blob)
|
|
6
|
+
*/
|
|
7
|
+
export declare function speakAudioSend(session: Session | undefined, audio: string | Uint8Array | Blob): Promise<void>;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type Session } from "./session";
|
|
2
|
+
import { type Signaling } from "./signaling";
|
|
3
|
+
export declare class SessionTransferManager {
|
|
4
|
+
private readonly session;
|
|
5
|
+
isTransferSessionInProgress: boolean;
|
|
6
|
+
constructor(session: Session);
|
|
7
|
+
private addTransferSessionListener;
|
|
8
|
+
transferSession(): void;
|
|
9
|
+
switchLiveVideo(videoContainer: HTMLDivElement, newSignaling: Signaling): void;
|
|
10
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { Signaling } from "./signaling";
|
|
2
|
+
import { type UneeqConfig } from "./types/UneeqConfig";
|
|
3
|
+
import { type UneeqMessage } from "./types/UneeqMessages";
|
|
4
|
+
import { SpeechRecognition } from "./speech-recognition";
|
|
5
|
+
import { ElevenLabsSTT } from "./elevenlabs-stt";
|
|
6
|
+
import { Subject } from "rxjs";
|
|
7
|
+
import { type DataChannelMessage } from "./webrtc-data-channel/DataChannelMessage";
|
|
8
|
+
import { type PromptMetadata } from "./types/PromptMetadata";
|
|
9
|
+
import { type CameraAnchorDistance, type CameraAnchorHorizontal } from "./webrtc-data-channel/messages/CameraAnchor";
|
|
10
|
+
import { SessionTransferManager } from "./session-transfer-manager";
|
|
11
|
+
export declare const VIDEO_REVEAL_DELAY_MS = 750;
|
|
12
|
+
export declare class Session {
|
|
13
|
+
readonly config: UneeqConfig;
|
|
14
|
+
id: string;
|
|
15
|
+
signaling?: Signaling;
|
|
16
|
+
messages: Subject<UneeqMessage>;
|
|
17
|
+
speechRec?: SpeechRecognition | ElevenLabsSTT;
|
|
18
|
+
promptMetadata: PromptMetadata;
|
|
19
|
+
audioStreamId?: number;
|
|
20
|
+
readonly sessionTransferManager: SessionTransferManager;
|
|
21
|
+
private jwt;
|
|
22
|
+
private welcomeRequested;
|
|
23
|
+
private resolutionLogged;
|
|
24
|
+
constructor(config: UneeqConfig);
|
|
25
|
+
updatePromptMetadata(metadata: object): void;
|
|
26
|
+
setEnableMicrophone(enable: boolean): void;
|
|
27
|
+
setCameraAnchorDistance(position: CameraAnchorDistance, duration?: number, signaling?: Signaling | undefined): void;
|
|
28
|
+
setCameraAnchorHorizontal(position: CameraAnchorHorizontal, duration?: number, signaling?: Signaling | undefined): void;
|
|
29
|
+
createSession(): void;
|
|
30
|
+
createSignaling(videoElement?: HTMLDivElement): Signaling;
|
|
31
|
+
private signalingEventHandler;
|
|
32
|
+
private handleSessionReady;
|
|
33
|
+
private initSpeechRecognition;
|
|
34
|
+
private createGoogleSTT;
|
|
35
|
+
private createElevenLabsSTT;
|
|
36
|
+
protected getSpeechOptions(jwt: string): {
|
|
37
|
+
apiUrl: string;
|
|
38
|
+
assetBasePath: string | undefined;
|
|
39
|
+
enableInterrupt: boolean | undefined;
|
|
40
|
+
enableVad: boolean | undefined;
|
|
41
|
+
jwtToken: string;
|
|
42
|
+
hintPhrases: string | undefined;
|
|
43
|
+
hintPhrasesBoost: number | undefined;
|
|
44
|
+
locales: string | undefined;
|
|
45
|
+
messages: Subject<UneeqMessage>;
|
|
46
|
+
promptMetadata: PromptMetadata;
|
|
47
|
+
sessionId: string;
|
|
48
|
+
sendMessage: (msg: DataChannelMessage) => void;
|
|
49
|
+
};
|
|
50
|
+
private initMetadata;
|
|
51
|
+
private handleDataChannelMessage;
|
|
52
|
+
private handleChannelMessageError;
|
|
53
|
+
private sendWelcomePrompt;
|
|
54
|
+
private listenForTabClosure;
|
|
55
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { ReplaySubject } from 'rxjs';
|
|
2
|
+
import { PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4';
|
|
3
|
+
import { type SignalingEvent } from './types/SignalingEvents';
|
|
4
|
+
import { type DataChannelMessage } from './webrtc-data-channel/DataChannelMessage';
|
|
5
|
+
import { type Session } from './session';
|
|
6
|
+
export declare class Signaling {
|
|
7
|
+
private readonly session;
|
|
8
|
+
private readonly connectionUrl;
|
|
9
|
+
videoContainerElement?: HTMLDivElement | undefined;
|
|
10
|
+
stream?: PixelStreaming;
|
|
11
|
+
messages: ReplaySubject<SignalingEvent>;
|
|
12
|
+
videoElement?: HTMLVideoElement;
|
|
13
|
+
audioElement?: HTMLAudioElement;
|
|
14
|
+
videoInitialized: boolean;
|
|
15
|
+
dataChannelOpen: boolean;
|
|
16
|
+
sceneReady: boolean;
|
|
17
|
+
private readonly maxReconnectAttempts;
|
|
18
|
+
private reconnectionAttempts;
|
|
19
|
+
private allowReconnection;
|
|
20
|
+
private isReconnectInProgress;
|
|
21
|
+
private audioStatsInternal?;
|
|
22
|
+
private freezeCount;
|
|
23
|
+
private lastDecodedFrames;
|
|
24
|
+
private lastCheckTime;
|
|
25
|
+
private audioStatsAnalyzer?;
|
|
26
|
+
private readonly config;
|
|
27
|
+
private readonly clientMessages;
|
|
28
|
+
constructor(session: Session, connectionUrl: string, videoContainerElement?: HTMLDivElement | undefined);
|
|
29
|
+
init(): void;
|
|
30
|
+
private addStreamHandlers;
|
|
31
|
+
private addWebSocketHandlers;
|
|
32
|
+
private handleWebSocketError;
|
|
33
|
+
private handleVideoInitialized;
|
|
34
|
+
playAudioElement(): void;
|
|
35
|
+
pauseAudioElement(): void;
|
|
36
|
+
/**
|
|
37
|
+
* Handle the case where the audio element is not found.
|
|
38
|
+
* Log an error and send a message to the client.
|
|
39
|
+
*
|
|
40
|
+
* TODO: In future make this a fatal error that will end the session, when we give users more information about errors
|
|
41
|
+
*/
|
|
42
|
+
private handleMissingAudioElement;
|
|
43
|
+
startAudioStatsMonitor(intervalMs?: number): void;
|
|
44
|
+
stopAudioStatsMonitor(): void;
|
|
45
|
+
private handleDataChannelOpen;
|
|
46
|
+
private handleWebRtcFailed;
|
|
47
|
+
private handleWebRtcDisconnected;
|
|
48
|
+
private handleWebRtcStats;
|
|
49
|
+
private checkForFreeze;
|
|
50
|
+
private reconnectIfSafe;
|
|
51
|
+
private handleDataChannelMessage;
|
|
52
|
+
private handleIndexInQueueMessage;
|
|
53
|
+
private handleRendererTakenMessage;
|
|
54
|
+
private handleRendererGoneEvent;
|
|
55
|
+
private handleSwitchRendererRequestEvent;
|
|
56
|
+
socketSend(msg: any): void;
|
|
57
|
+
dataSend(data: DataChannelMessage): void;
|
|
58
|
+
dataSendBinary(data: Uint8Array): void;
|
|
59
|
+
private uint8ToBase64;
|
|
60
|
+
endSession(reason?: string, sendMessageToClient?: boolean): void;
|
|
61
|
+
streamDisconnect(): void;
|
|
62
|
+
startSoftSwitch(): void;
|
|
63
|
+
stopSoftSwitch(): void;
|
|
64
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { type CameraAnchorDistance, type CameraAnchorHorizontal } from "../webrtc-data-channel/messages/CameraAnchor";
|
|
2
|
+
import { type LogLevel } from "./LogLevel";
|
|
3
|
+
import { type UneeqMessage } from "./UneeqMessages";
|
|
4
|
+
export declare enum SpeechRecognitionProvider {
|
|
5
|
+
Google = "google",
|
|
6
|
+
ElevenLabs = "elevenLabs"
|
|
7
|
+
}
|
|
8
|
+
export interface UneeqConfig {
|
|
9
|
+
sessionId: string;
|
|
10
|
+
sessionToken: string;
|
|
11
|
+
connectionUrl: string;
|
|
12
|
+
personaId: string;
|
|
13
|
+
videoContainerElement?: HTMLDivElement;
|
|
14
|
+
logLevel?: LogLevel | string;
|
|
15
|
+
enableMicrophone?: boolean;
|
|
16
|
+
assetBasePath?: string;
|
|
17
|
+
welcomePrompt?: string;
|
|
18
|
+
backgroundUrl?: string;
|
|
19
|
+
enableVad?: boolean;
|
|
20
|
+
enableInterruptBySpeech?: boolean;
|
|
21
|
+
cameraAnchorDistance: CameraAnchorDistance;
|
|
22
|
+
cameraAnchorHorizontal: CameraAnchorHorizontal;
|
|
23
|
+
forceTURN?: boolean;
|
|
24
|
+
/**
|
|
25
|
+
* This option specifies up to four locales that the Digital Human should understand a person speaking (speech recognition)
|
|
26
|
+
* The locales are specified in the BCP-47 format e.g "en-US".
|
|
27
|
+
* The first locale is considered the primary locale.
|
|
28
|
+
* Locales should be separated with a colon e.g "en-US:en-GB:en-AU".
|
|
29
|
+
*/
|
|
30
|
+
speechRecognitionLocales?: string;
|
|
31
|
+
/**
|
|
32
|
+
* Speech Recognition Hint Phrases
|
|
33
|
+
* A comma separated list of phrases the speech recognition system should expect.
|
|
34
|
+
*/
|
|
35
|
+
speechRecognitionHintPhrases?: string;
|
|
36
|
+
/**
|
|
37
|
+
* Speech Recognition Hint Phrases Boost
|
|
38
|
+
* An integer number between 0 and 20 that can be used to boost the hint phrases likely hood to match.
|
|
39
|
+
*/
|
|
40
|
+
speechRecognitionHintPhrasesBoost?: number;
|
|
41
|
+
/**
|
|
42
|
+
* Speech recognition provider selection.
|
|
43
|
+
* @default SpeechRecognitionProvider.Google
|
|
44
|
+
*
|
|
45
|
+
* Supported values:
|
|
46
|
+
* - 'google': Uses UneeQ's speech-recognition-service (existing implementation)
|
|
47
|
+
* - 'elevenLabs': Uses ElevenLabs Scribe 2 real-time STT
|
|
48
|
+
*
|
|
49
|
+
* Any other value will log a warning and default to 'google'.
|
|
50
|
+
*/
|
|
51
|
+
speechRecognitionProvider?: SpeechRecognitionProvider | 'google' | 'elevenLabs';
|
|
52
|
+
/**
|
|
53
|
+
* Configuration for ElevenLabs STT provider.
|
|
54
|
+
* These options only apply when using ElevenLabs - they are ignored when using Google STT provider.
|
|
55
|
+
*/
|
|
56
|
+
elevenLabsConfig?: {
|
|
57
|
+
/**
|
|
58
|
+
* Backend endpoint that provides ElevenLabs tokens.
|
|
59
|
+
* If not specified, will be auto-constructed as: ${connectionUrl}/elevenlabs/token
|
|
60
|
+
* @default `${connectionUrl}/elevenlabs/token`
|
|
61
|
+
*/
|
|
62
|
+
tokenEndpoint?: string;
|
|
63
|
+
/**
|
|
64
|
+
* ElevenLabs model ID to use for transcription.
|
|
65
|
+
* @default 'scribe_v2_realtime'
|
|
66
|
+
*/
|
|
67
|
+
modelId?: string;
|
|
68
|
+
/**
|
|
69
|
+
* Language code in ISO-639-1 or ISO-639-3 format (e.g., 'en', 'es', 'fr').
|
|
70
|
+
* If not specified, ElevenLabs will auto-detect the language.
|
|
71
|
+
* @default undefined (auto-detect)
|
|
72
|
+
*/
|
|
73
|
+
languageCode?: string;
|
|
74
|
+
/**
|
|
75
|
+
* Audio format for streaming to ElevenLabs.
|
|
76
|
+
* @default 'pcm_16000'
|
|
77
|
+
*/
|
|
78
|
+
audioFormat?: 'pcm_8000' | 'pcm_16000' | 'pcm_22050' | 'pcm_24000' | 'pcm_44100' | 'pcm_48000' | 'ulaw_8000';
|
|
79
|
+
/**
|
|
80
|
+
* VAD (Voice Activity Detection) configuration.
|
|
81
|
+
*/
|
|
82
|
+
vad?: {
|
|
83
|
+
/**
|
|
84
|
+
* Seconds of silence before VAD commits the transcript.
|
|
85
|
+
* Range: 0.3 - 3.0 seconds
|
|
86
|
+
* @default 1.5
|
|
87
|
+
*/
|
|
88
|
+
silenceThresholdSecs?: number;
|
|
89
|
+
/**
|
|
90
|
+
* VAD sensitivity threshold for speech detection.
|
|
91
|
+
* Range: 0.1 - 0.9
|
|
92
|
+
* @default 0.4
|
|
93
|
+
*/
|
|
94
|
+
threshold?: number;
|
|
95
|
+
/**
|
|
96
|
+
* Minimum speech duration in milliseconds to trigger detection.
|
|
97
|
+
* Range: 50 - 2000 ms
|
|
98
|
+
* @default 100
|
|
99
|
+
*/
|
|
100
|
+
minSpeechDurationMs?: number;
|
|
101
|
+
/**
|
|
102
|
+
* Minimum silence duration in milliseconds for segment boundary.
|
|
103
|
+
* Range: 50 - 2000 ms
|
|
104
|
+
* @default 100
|
|
105
|
+
*/
|
|
106
|
+
minSilenceDurationMs?: number;
|
|
107
|
+
};
|
|
108
|
+
/**
|
|
109
|
+
* Microphone audio processing configuration.
|
|
110
|
+
*/
|
|
111
|
+
microphone?: {
|
|
112
|
+
/**
|
|
113
|
+
* Enable echo cancellation (removes feedback from speakers).
|
|
114
|
+
* @default true
|
|
115
|
+
*/
|
|
116
|
+
echoCancellation?: boolean;
|
|
117
|
+
/**
|
|
118
|
+
* Enable noise suppression (reduces background noise).
|
|
119
|
+
* @default true
|
|
120
|
+
*/
|
|
121
|
+
noiseSuppression?: boolean;
|
|
122
|
+
/**
|
|
123
|
+
* Enable automatic gain control (normalizes volume levels).
|
|
124
|
+
* @default true
|
|
125
|
+
*/
|
|
126
|
+
autoGainControl?: boolean;
|
|
127
|
+
};
|
|
128
|
+
/**
|
|
129
|
+
* Enable telemetry data collection by ElevenLabs.
|
|
130
|
+
* @default false
|
|
131
|
+
*/
|
|
132
|
+
telemetryEnabled?: boolean;
|
|
133
|
+
};
|
|
134
|
+
customMetadata?: object;
|
|
135
|
+
messageHandler?: (msg: UneeqMessage) => void;
|
|
136
|
+
webRtcStatsEmitMessages?: boolean;
|
|
137
|
+
webRtcStatsLogMessages?: boolean;
|
|
138
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type UneeqConfig } from "./types/UneeqConfig";
|
|
2
|
+
import type { CameraAnchorDistance, CameraAnchorHorizontal } from "./webrtc-data-channel/messages/CameraAnchor";
|
|
3
|
+
export declare class Uneeq {
|
|
4
|
+
private readonly config;
|
|
5
|
+
private session?;
|
|
6
|
+
constructor(config: UneeqConfig);
|
|
7
|
+
init(): void;
|
|
8
|
+
private ensureSessionExists;
|
|
9
|
+
chatPrompt(chatPrompt: string): void;
|
|
10
|
+
speak(speech: string): void;
|
|
11
|
+
openAudioStream(): void;
|
|
12
|
+
closeAudioStream(): void;
|
|
13
|
+
interruptAudioStream(): void;
|
|
14
|
+
speakAudio(audio: string | Uint8Array | Blob): Promise<void>;
|
|
15
|
+
cameraAnchorDistance(position: CameraAnchorDistance, duration?: number): void;
|
|
16
|
+
cameraAnchorHorizontal(position: CameraAnchorHorizontal, duration?: number): void;
|
|
17
|
+
endSession(): void;
|
|
18
|
+
unmuteDigitalHuman(): void;
|
|
19
|
+
muteDigitalHuman(): void;
|
|
20
|
+
stopSpeaking(): void;
|
|
21
|
+
setCustomPromptMetadata(metadata: object): void;
|
|
22
|
+
enableMicrophone(enable?: boolean): void;
|
|
23
|
+
pauseSpeechRecognition(): Promise<boolean>;
|
|
24
|
+
resumeSpeechRecognition(): Promise<boolean>;
|
|
25
|
+
setWebRtcStatsEnabled(emitMessages: boolean, logMessages: boolean): void;
|
|
26
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/stats.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export declare class Stats {
|
|
2
|
+
private readonly stream;
|
|
3
|
+
private readonly videoElement?;
|
|
4
|
+
private readonly audioElement?;
|
|
5
|
+
constructor(stream: any, videoElement?: HTMLVideoElement | undefined, audioElement?: HTMLAudioElement | undefined);
|
|
6
|
+
analyzeAudioStats(): Promise<void>;
|
|
7
|
+
private getInboundAudioStats;
|
|
8
|
+
private getCodecFromStats;
|
|
9
|
+
private logAudioStats;
|
|
10
|
+
private logElementStatus;
|
|
11
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|