uneeq-js 3.17.1 → 3.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -59
- package/dist/esm/index.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +11 -1
package/README.md
CHANGED
|
@@ -1,97 +1,136 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://res.cloudinary.com/ddownn0ib/image/upload/v1781838531/Uneeq/uneeq_logo_yyc49n.png" alt="UneeQ" width="220">
|
|
3
|
+
</p>
|
|
3
4
|
|
|
4
|
-
|
|
5
|
-
#### Install dependencies:
|
|
6
|
-
`npm i`
|
|
5
|
+
# uneeq-js — UneeQ Digital Humans frontend SDK
|
|
7
6
|
|
|
8
|
-
|
|
9
|
-
`npm start`
|
|
7
|
+
Frontend SDK for integrating with the **UneeQ Digital Human Platform**. It manages the full lifecycle of a digital human session in the browser — WebRTC video streaming, microphone capture, speech recognition, and the data channel to the renderer.
|
|
10
8
|
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
<p align="center">
|
|
10
|
+
<img src="https://res.cloudinary.com/ddownn0ib/image/upload/v1781838526/Uneeq/uneeq_dh_laptop_promo_zncbsc.avif" alt="UneeQ digital human on the immersive platform" width="640">
|
|
11
|
+
</p>
|
|
13
12
|
|
|
14
|
-
|
|
13
|
+
- **Platform documentation:** https://docs.uneeq.io/#/
|
|
14
|
+
- **About UneeQ Digital Humans:** https://www.digitalhumans.com
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
## Install
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
```bash
|
|
19
|
+
npm install uneeq-js
|
|
20
|
+
```
|
|
19
21
|
|
|
20
|
-
|
|
22
|
+
## Quick start
|
|
21
23
|
|
|
22
|
-
|
|
23
|
-
# From uneeq-js directory:
|
|
24
|
-
npm link
|
|
25
|
-
npm start # Runs in watch mode, auto-compiles changes
|
|
24
|
+
A UneeQ session is created server-side first. Your backend calls the UneeQ session API to obtain a `sessionId` and `sessionToken`, then passes them to the SDK in the browser:
|
|
26
25
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
npm start
|
|
30
|
-
```
|
|
26
|
+
```typescript
|
|
27
|
+
import { Uneeq, CameraAnchorDistance, CameraAnchorHorizontal, type UneeqConfig } from 'uneeq-js'
|
|
31
28
|
|
|
32
|
-
|
|
29
|
+
const config: UneeqConfig = {
|
|
30
|
+
// Required — obtained from the UneeQ session API
|
|
31
|
+
sessionId: 'session-id-from-server',
|
|
32
|
+
sessionToken: 'jwt-from-server',
|
|
33
|
+
connectionUrl: 'wss://your-uneeq-host',
|
|
34
|
+
personaId: 'persona-uuid',
|
|
33
35
|
|
|
34
|
-
|
|
36
|
+
// Required — initial camera framing
|
|
37
|
+
cameraAnchorDistance: CameraAnchorDistance.MediumShot,
|
|
38
|
+
cameraAnchorHorizontal: CameraAnchorHorizontal.Center,
|
|
35
39
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
+
// Where the digital human video renders
|
|
41
|
+
videoContainerElement: document.getElementById('video-container') as HTMLDivElement,
|
|
42
|
+
|
|
43
|
+
// Optional
|
|
44
|
+
enableMicrophone: true,
|
|
45
|
+
logLevel: 'info',
|
|
46
|
+
messageHandler: (msg) => console.log(msg.uneeqMessageType),
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const uneeq = new Uneeq(config)
|
|
50
|
+
uneeq.init()
|
|
40
51
|
```
|
|
41
52
|
|
|
42
|
-
|
|
53
|
+
## Core API
|
|
54
|
+
|
|
55
|
+
```typescript
|
|
56
|
+
// Session
|
|
57
|
+
uneeq.init() // Start the session
|
|
58
|
+
uneeq.endSession() // End it gracefully
|
|
59
|
+
|
|
60
|
+
// Conversation
|
|
61
|
+
uneeq.chatPrompt('Hello!') // Send text to the NLP backend
|
|
62
|
+
uneeq.speak('Say this exactly') // Make the digital human speak text directly
|
|
63
|
+
uneeq.stopSpeaking() // Interrupt current speech
|
|
43
64
|
|
|
44
|
-
|
|
65
|
+
// Microphone & speech recognition
|
|
66
|
+
uneeq.enableMicrophone(true)
|
|
67
|
+
uneeq.pauseSpeechRecognition()
|
|
68
|
+
uneeq.resumeSpeechRecognition()
|
|
45
69
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
70
|
+
// Digital human audio & camera
|
|
71
|
+
uneeq.muteDigitalHuman()
|
|
72
|
+
uneeq.unmuteDigitalHuman()
|
|
73
|
+
uneeq.cameraAnchorDistance(position) // CameraAnchorDistance: CloseUp, MediumShot, FullShot, …
|
|
74
|
+
uneeq.cameraAnchorHorizontal(position) // CameraAnchorHorizontal: Left, Center, Right
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
## Events
|
|
78
|
+
|
|
79
|
+
All session events are delivered through the `messageHandler` callback:
|
|
80
|
+
|
|
81
|
+
```typescript
|
|
82
|
+
import { UneeqMessageType } from 'uneeq-js'
|
|
83
|
+
|
|
84
|
+
messageHandler: (msg) => {
|
|
85
|
+
switch (msg.uneeqMessageType) {
|
|
86
|
+
case UneeqMessageType.SessionLive: break // Ready to interact
|
|
87
|
+
case UneeqMessageType.AvatarStartedSpeaking: break
|
|
88
|
+
case UneeqMessageType.AvatarStoppedSpeaking: break
|
|
89
|
+
case UneeqMessageType.UserStartedSpeaking: break
|
|
90
|
+
case UneeqMessageType.SpeechTranscription: break
|
|
91
|
+
case UneeqMessageType.SessionEnded: break
|
|
92
|
+
case UneeqMessageType.SessionError: break
|
|
93
|
+
// ...see UneeqMessageType for the full set
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
```
|
|
49
97
|
|
|
50
|
-
|
|
51
|
-
A custom logger has been implemented using `pino` library. You can use it to log strings with values.
|
|
52
|
-
`Logger.info('Logging just a string')`
|
|
53
|
-
`Logger.info('Logging with value', { some: 'thing' })`
|
|
54
|
-
`Logger.info('Logging with multiple values', { some: 'thing' }, { another: 'thing' })`
|
|
98
|
+
Wait for `SessionLive` before calling conversation APIs.
|
|
55
99
|
|
|
56
|
-
|
|
57
|
-
`Logger.debug('a debug log')`
|
|
58
|
-
`Logger.info('an info log')`
|
|
59
|
-
`Logger.warn('a warning log')`
|
|
60
|
-
`Logger.error('an error log')`
|
|
100
|
+
## Configuration
|
|
61
101
|
|
|
62
|
-
|
|
102
|
+
`UneeqConfig` covers session credentials, microphone behaviour, speech-recognition provider (`google` or `deepgram`), VAD/interruption, and diagnostics. The full set of options is documented in the bundled TypeScript types (`UneeqConfig`) — your editor's autocomplete is the source of truth.
|
|
63
103
|
|
|
64
|
-
|
|
104
|
+
### Logging
|
|
65
105
|
|
|
106
|
+
Logs are prefixed with `UneeQ: ` so they're easy to filter. Set verbosity with `logLevel` (`'debug' | 'info' | 'warn' | 'error'`, default `'info'`).
|
|
66
107
|
|
|
67
108
|
## Beta features
|
|
68
109
|
|
|
69
|
-
### `audioUpstreamMode`
|
|
110
|
+
### `audioUpstreamMode`
|
|
70
111
|
|
|
71
|
-
`audioUpstreamMode` on `UneeqConfig` controls where user microphone audio is delivered. **Omit the field or leave it `undefined` for the existing GA behaviour — no existing caller needs to change anything.**
|
|
112
|
+
`audioUpstreamMode` on `UneeqConfig` controls where the user's microphone audio is delivered. **Omit the field (or leave it `undefined`) for the existing GA behaviour — no existing caller needs to change anything.**
|
|
72
113
|
|
|
73
114
|
| Value | Status | What it does |
|
|
74
115
|
| --- | --- | --- |
|
|
75
|
-
| `'speech-recognition-service'` | **GA / default** | Mic is captured with browser AEC/AGC/NS
|
|
76
|
-
| `'pixel-streaming'` | **Beta** | Mic is attached to the Pixel Streaming WebRTC peer connection as an upstream audio track
|
|
77
|
-
| `'both'` | **Beta** | Mic is delivered to BOTH the
|
|
116
|
+
| `'speech-recognition-service'` | **GA / default** | Mic is captured with browser AEC/AGC/NS and sent to UneeQ's STT WebSocket (Google or Deepgram). Transcripts flow to the renderer over the data channel. This is the behaviour when the field is omitted. |
|
|
117
|
+
| `'pixel-streaming'` | **Beta** | Mic is attached to the Pixel Streaming WebRTC peer connection as an upstream audio track. No STT client is instantiated; `speechRecognitionProvider` is ignored. Use when the renderer drives conversation in-renderer. |
|
|
118
|
+
| `'both'` | **Beta** | Mic is delivered to BOTH the peer connection AND the STT WebSocket. Two independent `getUserMedia` captures run in parallel. Use when the renderer needs raw audio AND you still want server-side transcripts (captions, compliance, transcript-driven LLM). |
|
|
78
119
|
|
|
79
|
-
**Headphone caveat:** the
|
|
120
|
+
**Headphone caveat:** the Pixel Streaming library captures the mic track without AEC/AGC/NS, so expect echo/feedback unless the user is wearing headphones. Not a concern in the default `'speech-recognition-service'` mode.
|
|
80
121
|
|
|
81
|
-
**Shape and defaults may change while the two non-default modes are beta.** The
|
|
122
|
+
**Shape and defaults may change while the two non-default modes are beta.** The SDK emits an `info`-level log line at session init whenever a beta mode is active, so the current upstream routing is visible in support captures:
|
|
82
123
|
|
|
83
124
|
```
|
|
84
125
|
audioUpstreamMode='pixel-streaming' is beta; behaviour may change.
|
|
85
126
|
```
|
|
86
127
|
|
|
87
|
-
|
|
128
|
+
Some other features, review the docs for more in-depth guidance:
|
|
88
129
|
|
|
89
|
-
- `muteUpstreamMic()` / `unmuteUpstreamMic()` — mute/unmute the
|
|
90
|
-
- `pauseSpeechRecognition()` / `resumeSpeechRecognition()` — unchanged
|
|
91
|
-
- `enableMicrophone(enabled)` —
|
|
130
|
+
- `muteUpstreamMic()` / `unmuteUpstreamMic()` — mute/unmute the Pixel Streaming upstream mic track only. No-op with a warning in `'speech-recognition-service'` mode (there is no PS track to control). In `'pixel-streaming'` / `'both'` mode, prefer these over `enableMicrophone()` when you only want to gate the PS leg.
|
|
131
|
+
- `pauseSpeechRecognition()` / `resumeSpeechRecognition()` — unchanged, but now warn and return `false` in `'pixel-streaming'` mode (there is no STT leg to pause).
|
|
132
|
+
- `enableMicrophone(enabled)` — dispatches to whichever leg(s) are active for the current mode: STT in `'speech-recognition-service'`, PS in `'pixel-streaming'`, both in `'both'`.
|
|
92
133
|
|
|
93
|
-
|
|
134
|
+
## License
|
|
94
135
|
|
|
95
|
-
|
|
96
|
-
#### onnxruntime-web
|
|
97
|
-
onnxruntime-web only works on version 1.15.1 at this time. Upgrading this package will likely break VAD funtionality.
|
|
136
|
+
ISC © UneeQ
|
package/dist/esm/index.js
CHANGED
|
@@ -78,7 +78,7 @@ m=`).map((l,c)=>(c>0?"m="+l:l).trim()+`\r
|
|
|
78
78
|
}
|
|
79
79
|
`,n=this.gl.createShader(this.gl.VERTEX_SHADER);this.gl.shaderSource(n,e),this.gl.compileShader(n);let i=this.gl.createShader(this.gl.FRAGMENT_SHADER);this.gl.shaderSource(i,t),this.gl.compileShader(i);let r=this.gl.createProgram();this.gl.attachShader(r,n),this.gl.attachShader(r,i),this.gl.linkProgram(r),this.gl.useProgram(r),this.positionLocation=this.gl.getAttribLocation(r,"a_position"),this.texcoordLocation=this.gl.getAttribLocation(r,"a_texCoord")}updateVideoTexture(){this.videoTexture||(this.videoTexture=this.gl.createTexture(),this.gl.bindTexture(this.gl.TEXTURE_2D,this.videoTexture),this.gl.texParameteri(this.gl.TEXTURE_2D,this.gl.TEXTURE_WRAP_S,this.gl.CLAMP_TO_EDGE),this.gl.texParameteri(this.gl.TEXTURE_2D,this.gl.TEXTURE_WRAP_T,this.gl.CLAMP_TO_EDGE),this.gl.texParameteri(this.gl.TEXTURE_2D,this.gl.TEXTURE_MIN_FILTER,this.gl.LINEAR),this.gl.texParameteri(this.gl.TEXTURE_2D,this.gl.TEXTURE_MAG_FILTER,this.gl.LINEAR));let e=this.webRtcController.videoPlayer.getVideoElement().videoHeight,t=this.webRtcController.videoPlayer.getVideoElement().videoWidth;this.prevVideoHeight!=e||this.prevVideoWidth!=t?this.gl.texImage2D(this.gl.TEXTURE_2D,0,this.gl.RGBA,t,e,0,this.gl.RGBA,this.gl.UNSIGNED_BYTE,this.webRtcController.videoPlayer.getVideoElement()):this.gl.texSubImage2D(this.gl.TEXTURE_2D,0,0,0,t,e,this.gl.RGBA,this.gl.UNSIGNED_BYTE,this.webRtcController.videoPlayer.getVideoElement()),this.prevVideoHeight=e,this.prevVideoWidth=t}initBuffers(){this.positionBuffer=this.gl.createBuffer(),this.gl.bindBuffer(this.gl.ARRAY_BUFFER,this.positionBuffer),this.gl.enableVertexAttribArray(this.positionLocation),this.gl.bufferData(this.gl.ARRAY_BUFFER,new Float32Array([-1,1,1,1,-1,-1,-1,-1,1,1,1,-1]),this.gl.STATIC_DRAW),this.gl.vertexAttribPointer(this.positionLocation,2,this.gl.FLOAT,!1,0,0),this.texcoordBuffer=this.gl.createBuffer(),this.gl.bindBuffer(this.gl.ARRAY_BUFFER,this.texcoordBuffer),this.gl.enableVertexAttribArray(this.texcoordLocation),this.gl.bufferData(this.gl.ARRAY_BUFFER,new Float32Array([0,0,1,0,0,1,0,1,1,0,1,1]),this.gl.STATIC_DRAW),this.gl.vertexAttribPointer(this.texcoordLocation,2,this.gl.FLOAT,!1,0,0)}onXrSessionStarted(e){d.Info("XR Session started"),this.xrSession=e,this.xrSession.addEventListener("end",()=>{this.onXrSessionEnded()}),this.initGL(),this.initShaders(),this.initBuffers(),e.requestReferenceSpace("local").then(t=>{if(this.xrRefSpace=t,this.xrSession.updateRenderState({baseLayer:new XRWebGLLayer(this.xrSession,this.gl)}),this.xrSession.supportedFrameRates)for(let n of this.xrSession.supportedFrameRates)n==90&&e.updateTargetFrameRate(90);this.xrSession.requestAnimationFrame(this.onXrFrame.bind(this))}),this.onSessionStarted.dispatchEvent(new Event("xrSessionStarted"))}areArraysEqual(e,t){return e.length===t.length&&e.every((n,i)=>Math.abs(n-t[i])<=this.EPSILON)}arePointsEqual(e,t){return Math.abs(e.x-t.x)>=this.EPSILON&&Math.abs(e.y-t.y)>=this.EPSILON&&Math.abs(e.z-t.z)>=this.EPSILON}sendXRDataToUE(){if(this.leftView==null||this.rightView==null)return;let e=this.lastSentLeftEyeProj==null||this.lastSentRightEyeProj==null||this.lastSentRelativeLeftEyePos==null||this.lastSentRelativeRightEyePos==null,t=this.leftView.transform.matrix,n=this.leftView.projectionMatrix,i=this.rightView.transform.matrix,r=this.rightView.projectionMatrix,o=this.xrViewerPose.transform.matrix;if(!e&&this.lastSentLeftEyeProj!=null&&this.lastSentRightEyeProj!=null){let c=this.areArraysEqual(n,this.lastSentLeftEyeProj),u=this.areArraysEqual(r,this.lastSentRightEyeProj);e=c==!1||u==!1}let a=new DOMPointReadOnly(this.leftView.transform.position.x-this.xrViewerPose.transform.position.x,this.leftView.transform.position.y-this.xrViewerPose.transform.position.y,this.leftView.transform.position.z-this.xrViewerPose.transform.position.z,1),l=new DOMPointReadOnly(this.leftView.transform.position.x-this.xrViewerPose.transform.position.x,this.leftView.transform.position.y-this.xrViewerPose.transform.position.y,this.leftView.transform.position.z-this.xrViewerPose.transform.position.z,1);if(!e&&this.lastSentRelativeLeftEyePos!=null&&this.lastSentRelativeRightEyePos!=null){let c=this.arePointsEqual(a,this.lastSentRelativeLeftEyePos),u=this.arePointsEqual(l,this.lastSentRelativeRightEyePos);e=c==!1||u==!1}e?(this.webRtcController.streamMessageController.toStreamerHandlers.get("XREyeViews")([t[0],t[4],t[8],t[12],t[1],t[5],t[9],t[13],t[2],t[6],t[10],t[14],t[3],t[7],t[11],t[15],n[0],n[4],n[8],n[12],n[1],n[5],n[9],n[13],n[2],n[6],n[10],n[14],n[3],n[7],n[11],n[15],i[0],i[4],i[8],i[12],i[1],i[5],i[9],i[13],i[2],i[6],i[10],i[14],i[3],i[7],i[11],i[15],r[0],r[4],r[8],r[12],r[1],r[5],r[9],r[13],r[2],r[6],r[10],r[14],r[3],r[7],r[11],r[15],o[0],o[4],o[8],o[12],o[1],o[5],o[9],o[13],o[2],o[6],o[10],o[14],o[3],o[7],o[11],o[15]]),this.lastSentLeftEyeProj=n,this.lastSentRightEyeProj=r,this.lastSentRelativeLeftEyePos=a,this.lastSentRelativeRightEyePos=l):this.webRtcController.streamMessageController.toStreamerHandlers.get("XRHMDTransform")([o[0],o[4],o[8],o[12],o[1],o[5],o[9],o[13],o[2],o[6],o[10],o[14],o[3],o[7],o[11],o[15]])}onXrFrame(e,t){if(this.xrViewerPose=t.getViewerPose(this.xrRefSpace),this.xrViewerPose){if(this.updateViews(),this.leftView==null||this.rightView==null)return;this.sendXRDataToUE(),this.updateVideoTexture(),this.render()}this.webRtcController.config.isFlagEnabled(h.XRControllerInput)&&this.xrSession.inputSources.forEach((n,i,r)=>{this.xrGamepadController.updateStatus(n,t,this.xrRefSpace)},this),this.xrSession.requestAnimationFrame((n,i)=>this.onXrFrame(n,i)),this.onFrame.dispatchEvent(new _t({time:e,frame:t}))}updateViews(){if(this.xrViewerPose)for(let e of this.xrViewerPose.views)e.eye==="left"?this.leftView=e:e.eye==="right"&&(this.rightView=e)}render(){if(!this.gl)return;let e=this.xrSession.renderState.baseLayer;this.gl.bindFramebuffer(this.gl.FRAMEBUFFER,e.framebuffer),this.gl.viewport(0,0,e.framebufferWidth,e.framebufferHeight),this.gl.drawArrays(this.gl.TRIANGLES,0,6)}static isSessionSupported(e){location.protocol!=="https:"&&d.Info("WebXR requires https, if you want WebXR use https.");try{if(navigator.xr)return navigator.xr.isSessionSupported(e)}catch{return Promise.resolve(!1)}return Promise.resolve(!1)}};var Cn=class{constructor(e){this.seq=e.Seq,this.playerSentTimestamp=Date.now(),this.requestFillerSize=e.Filler?e.Filler.length:0}update(e){this.playerReceivedTimestamp=Date.now(),this.streamerReceivedTimestamp=e.ReceivedTimestamp,this.streamerSentTimestamp=e.SentTimestamp,this.responseFillerSize=e.Filler?e.Filler.length:0}};var Mn=class{constructor(e,t){this.intervalHandle=void 0,this.sink=e,this.callback=t,this.records=new Map,this.seq=0}start(e){return this.isRunning()?!1:(this.startTime=Date.now(),this.records.clear(),this.intervalHandle=window.setInterval((()=>{Date.now()-this.startTime>=e.duration?this.stop():this.sendRequest(e.requestSize,e.responseSize)}).bind(this),Math.floor(1e3/e.rps)),!0)}stop(){this.intervalHandle&&(window.clearInterval(this.intervalHandle),this.intervalHandle=void 0,this.callback(this.produceResult()))}produceResult(){let e=new Map(this.records);return{records:e,dataChannelRtt:Math.ceil(Array.from(this.records.values()).reduce((t,n)=>t+(n.playerReceivedTimestamp-n.playerSentTimestamp),0)/this.records.size),playerToStreamerTime:Math.ceil(Array.from(this.records.values()).reduce((t,n)=>t+(n.streamerReceivedTimestamp-n.playerSentTimestamp),0)/this.records.size),streamerToPlayerTime:Math.ceil(Array.from(this.records.values()).reduce((t,n)=>t+(n.playerReceivedTimestamp-n.streamerSentTimestamp),0)/this.records.size),exportLatencyAsCSV:()=>{let t=`Timestamp;RTT;PlayerToStreamer;StreamerToPlayer;
|
|
80
80
|
`;return e.forEach(n=>{t+=n.playerSentTimestamp+";",t+=n.playerReceivedTimestamp-n.playerSentTimestamp+";",t+=n.streamerReceivedTimestamp-n.playerSentTimestamp+";",t+=n.playerReceivedTimestamp-n.streamerSentTimestamp+";",t+=`
|
|
81
|
-
`}),t}}}isRunning(){return!!this.intervalHandle}receive(e){if(!this.isRunning())return;if(!e){d.Error("Undefined response from server");return}let t=this.records.get(e.Seq);t&&t.update(e)}sendRequest(e,t){let n=this.createRequest(e,t),i=new Cn(n);this.records.set(i.seq,i),this.sink(n)}createRequest(e,t){return{Seq:this.seq++,FillResponseSize:t,Filler:e?"A".repeat(e):""}}};var xe=class{constructor(e,t){this.allowConsoleCommands=!1,this.config=e,t?.videoElementParent&&(this._videoElementParent=t.videoElementParent),this._eventEmitter=new $t,this.configureSettings(),this.setWebRtcPlayerController(new bn(this.config,this)),this._webXrController=new wn(this._webRtcController),this._setupWebRtcTCPRelayDetection=this._setupWebRtcTCPRelayDetection.bind(this),this._eventEmitter.addEventListener("webRtcConnected",n=>{this._eventEmitter.addEventListener("statsReceived",this._setupWebRtcTCPRelayDetection)})}get videoElementParent(){return this._videoElementParent||(this._videoElementParent=document.createElement("div"),this._videoElementParent.id="videoElementParent"),this._videoElementParent}configureSettings(){this.config._addOnSettingChangedListener(h.IsQualityController,e=>{e===!0&&!this._webRtcController.isQualityController&&this._webRtcController.sendRequestQualityControlOwnership()}),this.config._addOnSettingChangedListener(h.AFKDetection,e=>{this._webRtcController.setAfkEnabled(e)}),this.config._addOnSettingChangedListener(h.MatchViewportResolution,()=>{this._webRtcController.videoPlayer.updateVideoStreamSize()}),this.config._addOnSettingChangedListener(h.HoveringMouseMode,e=>{this.config.setFlagLabel(h.HoveringMouseMode,`Control Scheme: ${e?"Hovering":"Locked"} Mouse`),this._webRtcController.setMouseInputEnabled(this.config.isFlagEnabled(h.MouseInput))}),this.config._addOnSettingChangedListener(h.KeyboardInput,e=>{this._webRtcController.setKeyboardInputEnabled(e)}),this.config._addOnSettingChangedListener(h.MouseInput,e=>{this._webRtcController.setMouseInputEnabled(e)}),this.config._addOnSettingChangedListener(h.FakeMouseWithTouches,e=>{this._webRtcController.setTouchInputEnabled(this.config.isFlagEnabled(h.TouchInput))}),this.config._addOnSettingChangedListener(h.TouchInput,e=>{this._webRtcController.setTouchInputEnabled(e)}),this.config._addOnSettingChangedListener(h.GamepadInput,e=>{this._webRtcController.setGamePadInputEnabled(e)}),this.config._addOnNumericSettingChangedListener(m.MinQP,e=>{d.Info("-------- Sending MinQP --------"),this._webRtcController.sendEncoderMinQP(e),d.Info("-------------------------------------------");let t=Math.trunc(100*(1-e/51));this.config.setNumericSetting(m.CompatQualityMax,t)}),this.config._addOnNumericSettingChangedListener(m.MaxQP,e=>{d.Info("-------- Sending MaxQP --------"),this._webRtcController.sendEncoderMaxQP(e),d.Info("-------------------------------------------");let t=Math.trunc(100*(1-e/51));this.config.setNumericSetting(m.CompatQualityMin,t)}),this.config._addOnNumericSettingChangedListener(m.MinQuality,e=>{d.Info("-------- Sending MinQuality --------"),this._webRtcController.sendEncoderMinQuality(e),d.Info("-------------------------------------------"),this.config.setNumericSetting(m.CompatQualityMin,e)}),this.config._addOnNumericSettingChangedListener(m.MaxQuality,e=>{d.Info("-------- Sending MaxQuality --------"),this._webRtcController.sendEncoderMaxQuality(e),d.Info("-------------------------------------------"),this.config.setNumericSetting(m.CompatQualityMax,e)}),this.config._addOnNumericSettingChangedListener(m.CompatQualityMin,e=>{e=51-e/100*51,d.Info("-------- Sending MinQP from quality value --------"),this._webRtcController.sendEncoderMaxQP(e),d.Info("-------------------------------------------")}),this.config._addOnNumericSettingChangedListener(m.CompatQualityMax,e=>{e=51-e/100*51,d.Info("-------- Sending MaxQP from quality value --------"),this._webRtcController.sendEncoderMinQP(e),d.Info("-------------------------------------------")}),this.config._addOnNumericSettingChangedListener(m.WebRTCMinBitrate,e=>{d.Info("-------- Sending web rtc settings --------"),this._webRtcController.sendWebRTCMinBitrate(e*1e3),d.Info("-------------------------------------------")}),this.config._addOnNumericSettingChangedListener(m.WebRTCMaxBitrate,e=>{d.Info("-------- Sending web rtc settings --------"),this._webRtcController.sendWebRTCMaxBitrate(e*1e3),d.Info("-------------------------------------------")}),this.config._addOnNumericSettingChangedListener(m.WebRTCFPS,e=>{d.Info("-------- Sending web rtc settings --------"),this._webRtcController.sendWebRTCFps(e),d.Info("-------------------------------------------")}),this.config._addOnOptionSettingChangedListener(M.PreferredCodec,e=>{this._webRtcController&&this._webRtcController.setPreferredCodec(e)}),this.config._registerOnChangeEvents(this._eventEmitter)}_onInputControlOwnership(e){this._inputController=e}setWebRtcPlayerController(e){this._webRtcController=e,this._webRtcController.setPreferredCodec(this.config.getSettingOption(M.PreferredCodec).selected),this._webRtcController.resizePlayerStyle(),this.checkForAutoConnect()}connect(){this._eventEmitter.dispatchEvent(new Mt),this._webRtcController.connectToSignallingServer()}reconnect(){this._eventEmitter.dispatchEvent(new Et),this._webRtcController.tryReconnect("Reconnecting...")}disconnect(){this._eventEmitter.dispatchEvent(new Tt),this._webRtcController.close()}play(){this._onStreamLoading(),this._webRtcController.playStream()}checkForAutoConnect(){this.config.isFlagEnabled(h.AutoConnect)&&(this._onWebRtcAutoConnect(),this._webRtcController.connectToSignallingServer())}unmuteMicrophone(e=!1){if(this.config.isFlagEnabled("UseMic")){this.setMicrophoneMuted(!1);return}if(e){this.config.setFlagEnabled("UseMic",!0),this.reconnect();return}d.Warning("Trying to unmute mic, but PixelStreaming was initialized with no microphone track. Call with forceEnable == true to re-connect with a mic track.")}muteMicrophone(){if(this.config.isFlagEnabled("UseMic")){this.setMicrophoneMuted(!0);return}d.Info("Trying to mute mic, but PixelStreaming has no microphone track, so sending sound is already disabled.")}setMicrophoneMuted(e){var t,n,i,r;for(let o of(r=(i=(n=(t=this._webRtcController)===null||t===void 0?void 0:t.peerConnectionController)===null||n===void 0?void 0:n.peerConnection)===null||i===void 0?void 0:i.getTransceivers())!==null&&r!==void 0?r:[])Z.canTransceiverSendAudio(o)&&(o.sender.track.enabled=!e)}unmuteCamera(e=!1){if(this.config.isFlagEnabled("UseCamera")){this.setCameraMuted(!1);return}if(e){this.config.setFlagEnabled("UseCamera",!0),this.reconnect();return}d.Warning("Trying to unmute video, but PixelStreaming was initialized with no video track. Call with forceEnable == true to re-connect with a video track.")}muteCamera(){if(this.config.isFlagEnabled("UseCamera")){this.setCameraMuted(!0);return}d.Info("Trying to mute camera, but PixelStreaming has no video track, so sending video is already disabled.")}setCameraMuted(e){var t,n,i,r;for(let o of(r=(i=(n=(t=this._webRtcController)===null||t===void 0?void 0:t.peerConnectionController)===null||n===void 0?void 0:n.peerConnection)===null||i===void 0?void 0:i.getTransceivers())!==null&&r!==void 0?r:[])Z.canTransceiverSendVideo(o)&&(o.sender.track.enabled=!e)}_onWebRtcAutoConnect(){this._eventEmitter.dispatchEvent(new pt)}_onWebRtcSdp(){this._eventEmitter.dispatchEvent(new dt)}_onWebRtcSdpOffer(e){this._eventEmitter.dispatchEvent(new ut({sdp:e}))}_onWebRtcSdpAnswer(e){this._eventEmitter.dispatchEvent(new ht({sdp:e}))}_onLatencyCalculated(e){this._eventEmitter.dispatchEvent(new Ft({latencyInfo:e}))}_onStreamLoading(){this._eventEmitter.dispatchEvent(new Ct)}_onDisconnect(e,t){this._eventEmitter.dispatchEvent(new yt({eventString:e,allowClickToReconnect:t}))}_onWebRtcConnecting(){this._eventEmitter.dispatchEvent(new mt)}_onWebRtcConnected(){this._eventEmitter.dispatchEvent(new ft)}_onWebRtcFailed(){this._eventEmitter.dispatchEvent(new gt)}_onVideoInitialized(){this._eventEmitter.dispatchEvent(new wt),this._videoStartTime=Date.now()}_onLatencyTestResult(e){this._eventEmitter.dispatchEvent(new Ot({latencyTimings:e}))}_onDataChannelLatencyTestResponse(e){this._eventEmitter.dispatchEvent(new Nt({response:e}))}_onVideoStats(e){(!this._videoStartTime||this._videoStartTime===void 0)&&(this._videoStartTime=Date.now()),e.handleSessionStatistics(this._videoStartTime,this._inputController,this._webRtcController.videoAvgQp),this._eventEmitter.dispatchEvent(new It({aggregatedStats:e}))}_onVideoEncoderAvgQP(e){this._eventEmitter.dispatchEvent(new ct({avgQP:e}))}_onInitialSettings(e){var t;this._eventEmitter.dispatchEvent(new Vt({settings:e})),e.PixelStreamingSettings&&(this.allowConsoleCommands=(t=e.PixelStreamingSettings.AllowPixelStreamingCommands)!==null&&t!==void 0?t:!1,this.allowConsoleCommands===!1&&d.Info("-AllowPixelStreamingCommands=false, sending arbitrary console commands from browser to UE is disabled."));let n=this.config.useUrlParams,i=new ue(window.location.search);d.Info(`using URL parameters ${n}`),e.EncoderSettings&&(e.EncoderSettings.MinQP&&(this.config.setNumericSetting(m.MinQP,n&&i.has(m.MinQP)?Number.parseFloat(i.get(m.MinQP)):e.EncoderSettings.MinQP||0),this.config.setNumericSetting(m.MaxQP,n&&i.has(m.MaxQP)?Number.parseFloat(i.get(m.MaxQP)):e.EncoderSettings.MaxQP||51)),e.EncoderSettings.MinQuality&&(this.config.setNumericSetting(m.MinQuality,n&&i.has(m.MinQuality)?Number.parseFloat(i.get(m.MinQuality)):e.EncoderSettings.MinQuality||0),this.config.setNumericSetting(m.MaxQuality,n&&i.has(m.MaxQuality)?Number.parseFloat(i.get(m.MaxQuality)):e.EncoderSettings.MaxQuality||100)),n&&(i.has(m.CompatQualityMin)&&this.config.setNumericSetting(m.CompatQualityMin,Number.parseFloat(i.get(m.CompatQualityMin))),i.has(m.CompatQualityMax)&&this.config.setNumericSetting(m.CompatQualityMax,Number.parseFloat(i.get(m.CompatQualityMax))))),e.WebRTCSettings&&(this.config.setNumericSetting(m.WebRTCMinBitrate,n&&i.has(m.WebRTCMinBitrate)?Number.parseFloat(i.get(m.WebRTCMinBitrate)):e.WebRTCSettings.MinBitrate/1e3),this.config.setNumericSetting(m.WebRTCMaxBitrate,n&&i.has(m.WebRTCMaxBitrate)?Number.parseFloat(i.get(m.WebRTCMaxBitrate)):e.WebRTCSettings.MaxBitrate/1e3),this.config.setNumericSetting(m.WebRTCFPS,n&&i.has(m.WebRTCFPS)?Number.parseFloat(i.get(m.WebRTCFPS)):e.WebRTCSettings.FPS))}_onQualityControlOwnership(e){this.config.setFlagEnabled(h.IsQualityController,e)}_onPlayerCount(e){this._eventEmitter.dispatchEvent(new Wt({count:e}))}_onSubscribeFailed(e){this._eventEmitter.dispatchEvent(new Bt({message:e}))}_setupWebRtcTCPRelayDetection(e){let t=e.data.aggregatedStats.getActiveCandidatePair();if(t!=null){let n=e.data.aggregatedStats.localCandidates.find(i=>i.id==t.localCandidateId,null);n!=null&&n.candidateType=="relay"&&n.relayProtocol=="tcp"&&this._eventEmitter.dispatchEvent(new Ht),this._eventEmitter.removeEventListener("statsReceived",this._setupWebRtcTCPRelayDetection)}}requestLatencyTest(){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.sendLatencyTest(),!0):!1}requestDataChannelLatencyTest(e){return this._webRtcController.videoPlayer.isVideoReady()?(this._dataChannelLatencyTestController||(this._dataChannelLatencyTestController=new Mn(this._webRtcController.sendDataChannelLatencyTest.bind(this._webRtcController),t=>{this._eventEmitter.dispatchEvent(new Ut({result:t}))}),this.addEventListener("dataChannelLatencyTestResponse",({data:{response:t}})=>{this._dataChannelLatencyTestController.receive(t)})),this._dataChannelLatencyTestController.start(e)):!1}requestShowFps(){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.sendShowFps(),!0):!1}requestIframe(){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.sendIframeRequest(),!0):!1}emitUIInteraction(e){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.emitUIInteraction(e),!0):!1}emitCommand(e){return!this._webRtcController.videoPlayer.isVideoReady()||!this.allowConsoleCommands&&"ConsoleCommand"in e?!1:(this._webRtcController.emitCommand(e),!0)}emitConsoleCommand(e){return!this.allowConsoleCommands||!this._webRtcController.videoPlayer.isVideoReady()?!1:(this._webRtcController.emitConsoleCommand(e),!0)}sendTextboxEntry(e){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.sendTextboxEntry(e),!0):!1}addResponseEventListener(e,t){this._webRtcController.responseController.addResponseEventListener(e,t)}removeResponseEventListener(e){this._webRtcController.responseController.removeResponseEventListener(e)}dispatchEvent(e){return this._eventEmitter.dispatchEvent(e)}addEventListener(e,t){this._eventEmitter.addEventListener(e,t)}removeEventListener(e,t){this._eventEmitter.removeEventListener(e,t)}toggleXR(){this.webXrController.xrClicked()}setSignallingUrlBuilder(e){this._webRtcController.signallingUrlBuilder=e}get webRtcController(){return this._webRtcController}get signallingProtocol(){return this._webRtcController.protocol}get webXrController(){return this._webXrController}registerMessageHandler(e,t,n){if(t===S.FromStreamer&&typeof n>"u"){d.Warning(`Unable to register an undefined handler for ${e}`);return}t===S.ToStreamer&&typeof n>"u"?this._webRtcController.streamMessageController.registerMessageHandler(t,e,i=>this._webRtcController.sendMessageController.sendMessageToStreamer(e,i)):this._webRtcController.streamMessageController.registerMessageHandler(t,e,i=>n(i))}get toStreamerHandlers(){return this._webRtcController.streamMessageController.toStreamerHandlers}isReconnecting(){return this._webRtcController.isReconnecting}};var Tn=class{type=0},En=class{type=1},Rn=class{constructor(e){this.msg=e}msg;type=2};var kn=class{constructor(e){this.stats=e}stats;type=5};var Ps="3.17.1";var ur="sessionEndedDueToInactivity",pr="sessionEndedByUser",Is="sessionEndedDueToConnectionLoss",Ls="sessionEndedDueToVideoPlaybackError",mr="sessionEndedDueToPermissionsError";var Ds="sessionEndedDueToRendererError";var fr=(n=>(n.Good="Good",n.Fair="Fair",n.Poor="Poor",n))(fr||{}),gr=(n=>(n.Always="always",n.WhenDegraded="when-degraded",n.Hidden="hidden",n))(gr||{}),yr=(i=>(i.TopLeft="top-left",i.TopRight="top-right",i.BottomLeft="bottom-left",i.BottomRight="bottom-right",i))(yr||{}),Sr=(n=>(n.Click="click",n.Hover="hover",n.None="none",n))(Sr||{});function Pe(s,e,t){return Math.max(e,Math.min(s,t))}function br(s){let e=s.packetLoss??0,t=s.bufferDelay??50,n=s.roundTripTime??50,i=s.fec??!0,r=s.dtx??!1,o=s.bitrate,a=20+t+n/2,l=100,c=r?8:o?Pe(55-4.6*Math.log(o),0,30):6,u=i?20:10,w=c+(100-c)*(e/(e+u)),v=a*.03+(a>150?.1*(a-150):0),C=Pe(l-w-v,0,100),k=1+.035*C+C*(C-60)*(100-C)*7/1e6;return Pe(Math.round(k*100)/100,1,5)}function vr(s){let e=s.bitrate??0,t=s.roundTripTime??50,n=s.bufferDelay??0,i=s.width||640,r=s.height||480,o=s.frameRate??0,a=s.expectedFrameRate??30,l=s.codec?.toLowerCase();if(o===0)return 1;let c=i*r,u=l==="vp9"?1.2:1,w=n+t/2,v=u*e/c/o,k=Pe(.56*Math.log(v)+5.36,1,5)-1.9*Math.log(a/o)-w*.002;return Pe(Math.round(k*100)/100,1,5)}function xn(s){let e=s?.srcObject;return!!e&&typeof e.getAudioTracks=="function"&&e.getAudioTracks().length>0}var to=3.5,no=2.5,pe=class{constructor(e,t,n){this.stream=e;this.videoElement=t;this.audioElement=n}stream;videoElement;audioElement;previousStats;getPeerConnection(){return this.stream?.webRtcController?.peerConnectionController?.peerConnection}async analyzeAudioStats(){let e=this.getPeerConnection();if(!e){p.error("Stream appears to be down, RTCPeerConnection is not defined");return}let t=await e.getStats(null),n=this.getInboundAudioStats(t);if(!n){p.warn("WebRTC Connection - No inbound audio RTP stats found.");return}let i=n.codecId,r=i?this.getCodecFromStats(t,i):void 0;this.logAudioStats(n,r),this.logElementStatus(r)}async collectNetworkQuality(e){let t=this.getPeerConnection();if(!t)return p.warn("Cannot collect network quality: RTCPeerConnection is not defined"),null;let n=await t.getStats(null),i=this.extractRawStats(n);if(!this.previousStats)return this.previousStats=i,null;let r=this.computeMetrics(i,this.previousStats,e);return this.previousStats=i,r}resetStats(){this.previousStats=void 0}extractRawStats(e){let t={timestamp:Date.now(),audio:{packetsReceived:0,packetsLost:0,bytesReceived:0,jitter:0},video:{packetsReceived:0,packetsLost:0,bytesReceived:0,framesDecoded:0,framesDropped:0,frameWidth:0,frameHeight:0,framesPerSecond:0,jitter:0},connection:{currentRoundTripTime:0}},n,i;if(e.forEach(r=>{r.type==="inbound-rtp"&&r.kind==="audio"?(t.audio.packetsReceived=r.packetsReceived??0,t.audio.packetsLost=r.packetsLost??0,t.audio.bytesReceived=r.bytesReceived??0,t.audio.jitter=r.jitter??0,n=r.codecId):r.type==="inbound-rtp"&&r.kind==="video"?(t.video.packetsReceived=r.packetsReceived??0,t.video.packetsLost=r.packetsLost??0,t.video.bytesReceived=r.bytesReceived??0,t.video.framesDecoded=r.framesDecoded??0,t.video.framesDropped=r.framesDropped??0,t.video.frameWidth=r.frameWidth??0,t.video.frameHeight=r.frameHeight??0,t.video.framesPerSecond=r.framesPerSecond??0,t.video.jitter=r.jitter??0,i=r.codecId):r.type==="candidate-pair"&&r.nominated&&(t.connection.currentRoundTripTime=r.currentRoundTripTime??0)}),n){let r=e.get(n);r&&(t.audio.codecMimeType=r.mimeType)}if(i){let r=e.get(i);r&&(t.video.codecMimeType=r.mimeType)}return t}computeMetrics(e,t,n){let i=(e.timestamp-t.timestamp)/1e3,r=e.audio.packetsReceived-t.audio.packetsReceived,o=e.audio.packetsLost-t.audio.packetsLost,a=r+o,l=a>0?o/a*100:0,c=i>0?(e.audio.bytesReceived-t.audio.bytesReceived)*8/i/1e3:0,u=e.video.packetsReceived-t.video.packetsReceived,w=e.video.packetsLost-t.video.packetsLost,v=u+w,C=v>0?w/v*100:0,k=i>0?(e.video.bytesReceived-t.video.bytesReceived)*8/i/1e3:0,U=e.connection.currentRoundTripTime*1e3,H=br({packetLoss:l,bitrate:c*1e3,roundTripTime:U,bufferDelay:e.audio.jitter*1e3}),A=n.expectedFrameRate??30,Er=e.video.codecMimeType?.toLowerCase()??"",Rr=[["vp9","vp9"],["vp8","vp8"],["h264","h264"],["avc","h264"],["h265","h265"],["hevc","h265"],["av1","av1"]].find(([kr])=>Er.includes(kr))?.[1],As=vr({bitrate:k*1e3,roundTripTime:U,width:e.video.frameWidth,height:e.video.frameHeight,frameRate:e.video.framesPerSecond,expectedFrameRate:A,codec:Rr}),Ns=Math.min(H,As),De;return Ns>=to?De="Good":Ns>=no?De="Fair":De="Poor",{audioPacketLossPercent:Math.round(l*100)/100,videoPacketLossPercent:Math.round(C*100)/100,audioBitrateKbps:Math.round(c),videoBitrateKbps:Math.round(k),frameRate:e.video.framesPerSecond,audioJitterMs:Math.round(e.audio.jitter*1e3*100)/100,videoJitterMs:Math.round(e.video.jitter*1e3*100)/100,roundTripTimeMs:Math.round(U*100)/100,audioMos:H,videoMos:As,qualityLevel:De}}getInboundAudioStats(e){let t=null;return e.forEach(n=>{n.type==="inbound-rtp"&&n.kind==="audio"&&(t=n)}),t}getCodecFromStats(e,t){let n;return e.forEach(i=>{i.type==="codec"&&i.id===t&&(n=i)}),n}logAudioStats(e,t){let{packetsReceived:n,packetsLost:i,bytesReceived:r,jitter:o,audioLevel:a}=e;p.info(`WebRTC Connection - Audio -
|
|
81
|
+
`}),t}}}isRunning(){return!!this.intervalHandle}receive(e){if(!this.isRunning())return;if(!e){d.Error("Undefined response from server");return}let t=this.records.get(e.Seq);t&&t.update(e)}sendRequest(e,t){let n=this.createRequest(e,t),i=new Cn(n);this.records.set(i.seq,i),this.sink(n)}createRequest(e,t){return{Seq:this.seq++,FillResponseSize:t,Filler:e?"A".repeat(e):""}}};var xe=class{constructor(e,t){this.allowConsoleCommands=!1,this.config=e,t?.videoElementParent&&(this._videoElementParent=t.videoElementParent),this._eventEmitter=new $t,this.configureSettings(),this.setWebRtcPlayerController(new bn(this.config,this)),this._webXrController=new wn(this._webRtcController),this._setupWebRtcTCPRelayDetection=this._setupWebRtcTCPRelayDetection.bind(this),this._eventEmitter.addEventListener("webRtcConnected",n=>{this._eventEmitter.addEventListener("statsReceived",this._setupWebRtcTCPRelayDetection)})}get videoElementParent(){return this._videoElementParent||(this._videoElementParent=document.createElement("div"),this._videoElementParent.id="videoElementParent"),this._videoElementParent}configureSettings(){this.config._addOnSettingChangedListener(h.IsQualityController,e=>{e===!0&&!this._webRtcController.isQualityController&&this._webRtcController.sendRequestQualityControlOwnership()}),this.config._addOnSettingChangedListener(h.AFKDetection,e=>{this._webRtcController.setAfkEnabled(e)}),this.config._addOnSettingChangedListener(h.MatchViewportResolution,()=>{this._webRtcController.videoPlayer.updateVideoStreamSize()}),this.config._addOnSettingChangedListener(h.HoveringMouseMode,e=>{this.config.setFlagLabel(h.HoveringMouseMode,`Control Scheme: ${e?"Hovering":"Locked"} Mouse`),this._webRtcController.setMouseInputEnabled(this.config.isFlagEnabled(h.MouseInput))}),this.config._addOnSettingChangedListener(h.KeyboardInput,e=>{this._webRtcController.setKeyboardInputEnabled(e)}),this.config._addOnSettingChangedListener(h.MouseInput,e=>{this._webRtcController.setMouseInputEnabled(e)}),this.config._addOnSettingChangedListener(h.FakeMouseWithTouches,e=>{this._webRtcController.setTouchInputEnabled(this.config.isFlagEnabled(h.TouchInput))}),this.config._addOnSettingChangedListener(h.TouchInput,e=>{this._webRtcController.setTouchInputEnabled(e)}),this.config._addOnSettingChangedListener(h.GamepadInput,e=>{this._webRtcController.setGamePadInputEnabled(e)}),this.config._addOnNumericSettingChangedListener(m.MinQP,e=>{d.Info("-------- Sending MinQP --------"),this._webRtcController.sendEncoderMinQP(e),d.Info("-------------------------------------------");let t=Math.trunc(100*(1-e/51));this.config.setNumericSetting(m.CompatQualityMax,t)}),this.config._addOnNumericSettingChangedListener(m.MaxQP,e=>{d.Info("-------- Sending MaxQP --------"),this._webRtcController.sendEncoderMaxQP(e),d.Info("-------------------------------------------");let t=Math.trunc(100*(1-e/51));this.config.setNumericSetting(m.CompatQualityMin,t)}),this.config._addOnNumericSettingChangedListener(m.MinQuality,e=>{d.Info("-------- Sending MinQuality --------"),this._webRtcController.sendEncoderMinQuality(e),d.Info("-------------------------------------------"),this.config.setNumericSetting(m.CompatQualityMin,e)}),this.config._addOnNumericSettingChangedListener(m.MaxQuality,e=>{d.Info("-------- Sending MaxQuality --------"),this._webRtcController.sendEncoderMaxQuality(e),d.Info("-------------------------------------------"),this.config.setNumericSetting(m.CompatQualityMax,e)}),this.config._addOnNumericSettingChangedListener(m.CompatQualityMin,e=>{e=51-e/100*51,d.Info("-------- Sending MinQP from quality value --------"),this._webRtcController.sendEncoderMaxQP(e),d.Info("-------------------------------------------")}),this.config._addOnNumericSettingChangedListener(m.CompatQualityMax,e=>{e=51-e/100*51,d.Info("-------- Sending MaxQP from quality value --------"),this._webRtcController.sendEncoderMinQP(e),d.Info("-------------------------------------------")}),this.config._addOnNumericSettingChangedListener(m.WebRTCMinBitrate,e=>{d.Info("-------- Sending web rtc settings --------"),this._webRtcController.sendWebRTCMinBitrate(e*1e3),d.Info("-------------------------------------------")}),this.config._addOnNumericSettingChangedListener(m.WebRTCMaxBitrate,e=>{d.Info("-------- Sending web rtc settings --------"),this._webRtcController.sendWebRTCMaxBitrate(e*1e3),d.Info("-------------------------------------------")}),this.config._addOnNumericSettingChangedListener(m.WebRTCFPS,e=>{d.Info("-------- Sending web rtc settings --------"),this._webRtcController.sendWebRTCFps(e),d.Info("-------------------------------------------")}),this.config._addOnOptionSettingChangedListener(M.PreferredCodec,e=>{this._webRtcController&&this._webRtcController.setPreferredCodec(e)}),this.config._registerOnChangeEvents(this._eventEmitter)}_onInputControlOwnership(e){this._inputController=e}setWebRtcPlayerController(e){this._webRtcController=e,this._webRtcController.setPreferredCodec(this.config.getSettingOption(M.PreferredCodec).selected),this._webRtcController.resizePlayerStyle(),this.checkForAutoConnect()}connect(){this._eventEmitter.dispatchEvent(new Mt),this._webRtcController.connectToSignallingServer()}reconnect(){this._eventEmitter.dispatchEvent(new Et),this._webRtcController.tryReconnect("Reconnecting...")}disconnect(){this._eventEmitter.dispatchEvent(new Tt),this._webRtcController.close()}play(){this._onStreamLoading(),this._webRtcController.playStream()}checkForAutoConnect(){this.config.isFlagEnabled(h.AutoConnect)&&(this._onWebRtcAutoConnect(),this._webRtcController.connectToSignallingServer())}unmuteMicrophone(e=!1){if(this.config.isFlagEnabled("UseMic")){this.setMicrophoneMuted(!1);return}if(e){this.config.setFlagEnabled("UseMic",!0),this.reconnect();return}d.Warning("Trying to unmute mic, but PixelStreaming was initialized with no microphone track. Call with forceEnable == true to re-connect with a mic track.")}muteMicrophone(){if(this.config.isFlagEnabled("UseMic")){this.setMicrophoneMuted(!0);return}d.Info("Trying to mute mic, but PixelStreaming has no microphone track, so sending sound is already disabled.")}setMicrophoneMuted(e){var t,n,i,r;for(let o of(r=(i=(n=(t=this._webRtcController)===null||t===void 0?void 0:t.peerConnectionController)===null||n===void 0?void 0:n.peerConnection)===null||i===void 0?void 0:i.getTransceivers())!==null&&r!==void 0?r:[])Z.canTransceiverSendAudio(o)&&(o.sender.track.enabled=!e)}unmuteCamera(e=!1){if(this.config.isFlagEnabled("UseCamera")){this.setCameraMuted(!1);return}if(e){this.config.setFlagEnabled("UseCamera",!0),this.reconnect();return}d.Warning("Trying to unmute video, but PixelStreaming was initialized with no video track. Call with forceEnable == true to re-connect with a video track.")}muteCamera(){if(this.config.isFlagEnabled("UseCamera")){this.setCameraMuted(!0);return}d.Info("Trying to mute camera, but PixelStreaming has no video track, so sending video is already disabled.")}setCameraMuted(e){var t,n,i,r;for(let o of(r=(i=(n=(t=this._webRtcController)===null||t===void 0?void 0:t.peerConnectionController)===null||n===void 0?void 0:n.peerConnection)===null||i===void 0?void 0:i.getTransceivers())!==null&&r!==void 0?r:[])Z.canTransceiverSendVideo(o)&&(o.sender.track.enabled=!e)}_onWebRtcAutoConnect(){this._eventEmitter.dispatchEvent(new pt)}_onWebRtcSdp(){this._eventEmitter.dispatchEvent(new dt)}_onWebRtcSdpOffer(e){this._eventEmitter.dispatchEvent(new ut({sdp:e}))}_onWebRtcSdpAnswer(e){this._eventEmitter.dispatchEvent(new ht({sdp:e}))}_onLatencyCalculated(e){this._eventEmitter.dispatchEvent(new Ft({latencyInfo:e}))}_onStreamLoading(){this._eventEmitter.dispatchEvent(new Ct)}_onDisconnect(e,t){this._eventEmitter.dispatchEvent(new yt({eventString:e,allowClickToReconnect:t}))}_onWebRtcConnecting(){this._eventEmitter.dispatchEvent(new mt)}_onWebRtcConnected(){this._eventEmitter.dispatchEvent(new ft)}_onWebRtcFailed(){this._eventEmitter.dispatchEvent(new gt)}_onVideoInitialized(){this._eventEmitter.dispatchEvent(new wt),this._videoStartTime=Date.now()}_onLatencyTestResult(e){this._eventEmitter.dispatchEvent(new Ot({latencyTimings:e}))}_onDataChannelLatencyTestResponse(e){this._eventEmitter.dispatchEvent(new Nt({response:e}))}_onVideoStats(e){(!this._videoStartTime||this._videoStartTime===void 0)&&(this._videoStartTime=Date.now()),e.handleSessionStatistics(this._videoStartTime,this._inputController,this._webRtcController.videoAvgQp),this._eventEmitter.dispatchEvent(new It({aggregatedStats:e}))}_onVideoEncoderAvgQP(e){this._eventEmitter.dispatchEvent(new ct({avgQP:e}))}_onInitialSettings(e){var t;this._eventEmitter.dispatchEvent(new Vt({settings:e})),e.PixelStreamingSettings&&(this.allowConsoleCommands=(t=e.PixelStreamingSettings.AllowPixelStreamingCommands)!==null&&t!==void 0?t:!1,this.allowConsoleCommands===!1&&d.Info("-AllowPixelStreamingCommands=false, sending arbitrary console commands from browser to UE is disabled."));let n=this.config.useUrlParams,i=new ue(window.location.search);d.Info(`using URL parameters ${n}`),e.EncoderSettings&&(e.EncoderSettings.MinQP&&(this.config.setNumericSetting(m.MinQP,n&&i.has(m.MinQP)?Number.parseFloat(i.get(m.MinQP)):e.EncoderSettings.MinQP||0),this.config.setNumericSetting(m.MaxQP,n&&i.has(m.MaxQP)?Number.parseFloat(i.get(m.MaxQP)):e.EncoderSettings.MaxQP||51)),e.EncoderSettings.MinQuality&&(this.config.setNumericSetting(m.MinQuality,n&&i.has(m.MinQuality)?Number.parseFloat(i.get(m.MinQuality)):e.EncoderSettings.MinQuality||0),this.config.setNumericSetting(m.MaxQuality,n&&i.has(m.MaxQuality)?Number.parseFloat(i.get(m.MaxQuality)):e.EncoderSettings.MaxQuality||100)),n&&(i.has(m.CompatQualityMin)&&this.config.setNumericSetting(m.CompatQualityMin,Number.parseFloat(i.get(m.CompatQualityMin))),i.has(m.CompatQualityMax)&&this.config.setNumericSetting(m.CompatQualityMax,Number.parseFloat(i.get(m.CompatQualityMax))))),e.WebRTCSettings&&(this.config.setNumericSetting(m.WebRTCMinBitrate,n&&i.has(m.WebRTCMinBitrate)?Number.parseFloat(i.get(m.WebRTCMinBitrate)):e.WebRTCSettings.MinBitrate/1e3),this.config.setNumericSetting(m.WebRTCMaxBitrate,n&&i.has(m.WebRTCMaxBitrate)?Number.parseFloat(i.get(m.WebRTCMaxBitrate)):e.WebRTCSettings.MaxBitrate/1e3),this.config.setNumericSetting(m.WebRTCFPS,n&&i.has(m.WebRTCFPS)?Number.parseFloat(i.get(m.WebRTCFPS)):e.WebRTCSettings.FPS))}_onQualityControlOwnership(e){this.config.setFlagEnabled(h.IsQualityController,e)}_onPlayerCount(e){this._eventEmitter.dispatchEvent(new Wt({count:e}))}_onSubscribeFailed(e){this._eventEmitter.dispatchEvent(new Bt({message:e}))}_setupWebRtcTCPRelayDetection(e){let t=e.data.aggregatedStats.getActiveCandidatePair();if(t!=null){let n=e.data.aggregatedStats.localCandidates.find(i=>i.id==t.localCandidateId,null);n!=null&&n.candidateType=="relay"&&n.relayProtocol=="tcp"&&this._eventEmitter.dispatchEvent(new Ht),this._eventEmitter.removeEventListener("statsReceived",this._setupWebRtcTCPRelayDetection)}}requestLatencyTest(){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.sendLatencyTest(),!0):!1}requestDataChannelLatencyTest(e){return this._webRtcController.videoPlayer.isVideoReady()?(this._dataChannelLatencyTestController||(this._dataChannelLatencyTestController=new Mn(this._webRtcController.sendDataChannelLatencyTest.bind(this._webRtcController),t=>{this._eventEmitter.dispatchEvent(new Ut({result:t}))}),this.addEventListener("dataChannelLatencyTestResponse",({data:{response:t}})=>{this._dataChannelLatencyTestController.receive(t)})),this._dataChannelLatencyTestController.start(e)):!1}requestShowFps(){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.sendShowFps(),!0):!1}requestIframe(){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.sendIframeRequest(),!0):!1}emitUIInteraction(e){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.emitUIInteraction(e),!0):!1}emitCommand(e){return!this._webRtcController.videoPlayer.isVideoReady()||!this.allowConsoleCommands&&"ConsoleCommand"in e?!1:(this._webRtcController.emitCommand(e),!0)}emitConsoleCommand(e){return!this.allowConsoleCommands||!this._webRtcController.videoPlayer.isVideoReady()?!1:(this._webRtcController.emitConsoleCommand(e),!0)}sendTextboxEntry(e){return this._webRtcController.videoPlayer.isVideoReady()?(this._webRtcController.sendTextboxEntry(e),!0):!1}addResponseEventListener(e,t){this._webRtcController.responseController.addResponseEventListener(e,t)}removeResponseEventListener(e){this._webRtcController.responseController.removeResponseEventListener(e)}dispatchEvent(e){return this._eventEmitter.dispatchEvent(e)}addEventListener(e,t){this._eventEmitter.addEventListener(e,t)}removeEventListener(e,t){this._eventEmitter.removeEventListener(e,t)}toggleXR(){this.webXrController.xrClicked()}setSignallingUrlBuilder(e){this._webRtcController.signallingUrlBuilder=e}get webRtcController(){return this._webRtcController}get signallingProtocol(){return this._webRtcController.protocol}get webXrController(){return this._webXrController}registerMessageHandler(e,t,n){if(t===S.FromStreamer&&typeof n>"u"){d.Warning(`Unable to register an undefined handler for ${e}`);return}t===S.ToStreamer&&typeof n>"u"?this._webRtcController.streamMessageController.registerMessageHandler(t,e,i=>this._webRtcController.sendMessageController.sendMessageToStreamer(e,i)):this._webRtcController.streamMessageController.registerMessageHandler(t,e,i=>n(i))}get toStreamerHandlers(){return this._webRtcController.streamMessageController.toStreamerHandlers}isReconnecting(){return this._webRtcController.isReconnecting}};var Tn=class{type=0},En=class{type=1},Rn=class{constructor(e){this.msg=e}msg;type=2};var kn=class{constructor(e){this.stats=e}stats;type=5};var Ps="3.18.0";var ur="sessionEndedDueToInactivity",pr="sessionEndedByUser",Is="sessionEndedDueToConnectionLoss",Ls="sessionEndedDueToVideoPlaybackError",mr="sessionEndedDueToPermissionsError";var Ds="sessionEndedDueToRendererError";var fr=(n=>(n.Good="Good",n.Fair="Fair",n.Poor="Poor",n))(fr||{}),gr=(n=>(n.Always="always",n.WhenDegraded="when-degraded",n.Hidden="hidden",n))(gr||{}),yr=(i=>(i.TopLeft="top-left",i.TopRight="top-right",i.BottomLeft="bottom-left",i.BottomRight="bottom-right",i))(yr||{}),Sr=(n=>(n.Click="click",n.Hover="hover",n.None="none",n))(Sr||{});function Pe(s,e,t){return Math.max(e,Math.min(s,t))}function br(s){let e=s.packetLoss??0,t=s.bufferDelay??50,n=s.roundTripTime??50,i=s.fec??!0,r=s.dtx??!1,o=s.bitrate,a=20+t+n/2,l=100,c=r?8:o?Pe(55-4.6*Math.log(o),0,30):6,u=i?20:10,w=c+(100-c)*(e/(e+u)),v=a*.03+(a>150?.1*(a-150):0),C=Pe(l-w-v,0,100),k=1+.035*C+C*(C-60)*(100-C)*7/1e6;return Pe(Math.round(k*100)/100,1,5)}function vr(s){let e=s.bitrate??0,t=s.roundTripTime??50,n=s.bufferDelay??0,i=s.width||640,r=s.height||480,o=s.frameRate??0,a=s.expectedFrameRate??30,l=s.codec?.toLowerCase();if(o===0)return 1;let c=i*r,u=l==="vp9"?1.2:1,w=n+t/2,v=u*e/c/o,k=Pe(.56*Math.log(v)+5.36,1,5)-1.9*Math.log(a/o)-w*.002;return Pe(Math.round(k*100)/100,1,5)}function xn(s){let e=s?.srcObject;return!!e&&typeof e.getAudioTracks=="function"&&e.getAudioTracks().length>0}var to=3.5,no=2.5,pe=class{constructor(e,t,n){this.stream=e;this.videoElement=t;this.audioElement=n}stream;videoElement;audioElement;previousStats;getPeerConnection(){return this.stream?.webRtcController?.peerConnectionController?.peerConnection}async analyzeAudioStats(){let e=this.getPeerConnection();if(!e){p.error("Stream appears to be down, RTCPeerConnection is not defined");return}let t=await e.getStats(null),n=this.getInboundAudioStats(t);if(!n){p.warn("WebRTC Connection - No inbound audio RTP stats found.");return}let i=n.codecId,r=i?this.getCodecFromStats(t,i):void 0;this.logAudioStats(n,r),this.logElementStatus(r)}async collectNetworkQuality(e){let t=this.getPeerConnection();if(!t)return p.warn("Cannot collect network quality: RTCPeerConnection is not defined"),null;let n=await t.getStats(null),i=this.extractRawStats(n);if(!this.previousStats)return this.previousStats=i,null;let r=this.computeMetrics(i,this.previousStats,e);return this.previousStats=i,r}resetStats(){this.previousStats=void 0}extractRawStats(e){let t={timestamp:Date.now(),audio:{packetsReceived:0,packetsLost:0,bytesReceived:0,jitter:0},video:{packetsReceived:0,packetsLost:0,bytesReceived:0,framesDecoded:0,framesDropped:0,frameWidth:0,frameHeight:0,framesPerSecond:0,jitter:0},connection:{currentRoundTripTime:0}},n,i;if(e.forEach(r=>{r.type==="inbound-rtp"&&r.kind==="audio"?(t.audio.packetsReceived=r.packetsReceived??0,t.audio.packetsLost=r.packetsLost??0,t.audio.bytesReceived=r.bytesReceived??0,t.audio.jitter=r.jitter??0,n=r.codecId):r.type==="inbound-rtp"&&r.kind==="video"?(t.video.packetsReceived=r.packetsReceived??0,t.video.packetsLost=r.packetsLost??0,t.video.bytesReceived=r.bytesReceived??0,t.video.framesDecoded=r.framesDecoded??0,t.video.framesDropped=r.framesDropped??0,t.video.frameWidth=r.frameWidth??0,t.video.frameHeight=r.frameHeight??0,t.video.framesPerSecond=r.framesPerSecond??0,t.video.jitter=r.jitter??0,i=r.codecId):r.type==="candidate-pair"&&r.nominated&&(t.connection.currentRoundTripTime=r.currentRoundTripTime??0)}),n){let r=e.get(n);r&&(t.audio.codecMimeType=r.mimeType)}if(i){let r=e.get(i);r&&(t.video.codecMimeType=r.mimeType)}return t}computeMetrics(e,t,n){let i=(e.timestamp-t.timestamp)/1e3,r=e.audio.packetsReceived-t.audio.packetsReceived,o=e.audio.packetsLost-t.audio.packetsLost,a=r+o,l=a>0?o/a*100:0,c=i>0?(e.audio.bytesReceived-t.audio.bytesReceived)*8/i/1e3:0,u=e.video.packetsReceived-t.video.packetsReceived,w=e.video.packetsLost-t.video.packetsLost,v=u+w,C=v>0?w/v*100:0,k=i>0?(e.video.bytesReceived-t.video.bytesReceived)*8/i/1e3:0,U=e.connection.currentRoundTripTime*1e3,H=br({packetLoss:l,bitrate:c*1e3,roundTripTime:U,bufferDelay:e.audio.jitter*1e3}),A=n.expectedFrameRate??30,Er=e.video.codecMimeType?.toLowerCase()??"",Rr=[["vp9","vp9"],["vp8","vp8"],["h264","h264"],["avc","h264"],["h265","h265"],["hevc","h265"],["av1","av1"]].find(([kr])=>Er.includes(kr))?.[1],As=vr({bitrate:k*1e3,roundTripTime:U,width:e.video.frameWidth,height:e.video.frameHeight,frameRate:e.video.framesPerSecond,expectedFrameRate:A,codec:Rr}),Ns=Math.min(H,As),De;return Ns>=to?De="Good":Ns>=no?De="Fair":De="Poor",{audioPacketLossPercent:Math.round(l*100)/100,videoPacketLossPercent:Math.round(C*100)/100,audioBitrateKbps:Math.round(c),videoBitrateKbps:Math.round(k),frameRate:e.video.framesPerSecond,audioJitterMs:Math.round(e.audio.jitter*1e3*100)/100,videoJitterMs:Math.round(e.video.jitter*1e3*100)/100,roundTripTimeMs:Math.round(U*100)/100,audioMos:H,videoMos:As,qualityLevel:De}}getInboundAudioStats(e){let t=null;return e.forEach(n=>{n.type==="inbound-rtp"&&n.kind==="audio"&&(t=n)}),t}getCodecFromStats(e,t){let n;return e.forEach(i=>{i.type==="codec"&&i.id===t&&(n=i)}),n}logAudioStats(e,t){let{packetsReceived:n,packetsLost:i,bytesReceived:r,jitter:o,audioLevel:a}=e;p.info(`WebRTC Connection - Audio -
|
|
82
82
|
Codec: ${t?t.mimeType+" ("+t.payloadType+")":"Unknown"},
|
|
83
83
|
Packets received: ${n},
|
|
84
84
|
Packets lost: ${i},
|