teleportxr 1.0.83 → 1.0.84

package/connections/webrtcconnection.js

@@ -67,6 +67,10 @@ class WebRtcConnection extends EventEmitter
 		{
 			console.log("ICE candidate error: "+event.errorCode+" "+event.errorText+" "+event.port+" "+event.url);
 		};
+		this._onTrack = this._handleTrack.bind(this);
+		this._audioSource = null;
+		this._localAudioTrack = null;
+		this._audioSink = null;
 
 
 		Object.defineProperties(this, {
@@ -115,8 +119,10 @@ class WebRtcConnection extends EventEmitter
 			this.peerConnection.removeEventListener('icegatheringstatechange', this._onIceGatheringStateChange);
 			this.peerConnection.removeEventListener("icecandidateerror", this._onIceCandidateError);
 			this.peerConnection.removeEventListener("connectionstatechange", this._onConnectionStateChange);
+			this.peerConnection.removeEventListener('track', this._onTrack);
 			if (this.onIceCandidate)
 				this.peerConnection.removeEventListener('icecandidate', this.onIceCandidate);
+			this._teardownAudioMediaTrack();
 			try { this.peerConnection.close(); } catch (e) {}
 			this.peerConnection = null;
 		}
@@ -157,6 +163,7 @@ class WebRtcConnection extends EventEmitter
 		this.peerConnection.addEventListener('icegatheringstatechange', this._onIceGatheringStateChange);
 		this.peerConnection.addEventListener("icecandidateerror", this._onIceCandidateError);
 		this.peerConnection.addEventListener("connectionstatechange", this._onConnectionStateChange);
+		this.peerConnection.addEventListener('track', this._onTrack);
 		// Attach the icecandidate listener here, before doOffer can call
 		// setLocalDescription(). libwebrtc starts the ICE agent inside
 		// setLocalDescription() and emits host candidates synchronously on
@@ -329,11 +336,13 @@ class WebRtcConnection extends EventEmitter
 			this.peerConnection.removeEventListener('icegatheringstatechange', this._onIceGatheringStateChange);
 			this.peerConnection.removeEventListener("icecandidateerror", this._onIceCandidateError);
 			this.peerConnection.removeEventListener("connectionstatechange", this._onConnectionStateChange);
+			this.peerConnection.removeEventListener('track', this._onTrack);
 			if (this.onIceCandidate)
 			{
 				this.peerConnection.removeEventListener('icecandidate', this.onIceCandidate);
 			}
 		}
+		this._teardownAudioMediaTrack();
 		if (this.connectionTimer)
 		{
 			this.options.clearTimeout(this.connectionTimer);
@@ -415,20 +424,22 @@ class WebRtcConnection extends EventEmitter
 		}
 	}
     beforeOffer() {
-          
+
         this.videoDataChannel = this.createDataChannel("video",20);
         this.tagDataChannel = this.createDataChannel("video_tags",40);
         this.audioToClientDataChannel = this.createDataChannel("audio_server_to_client",60);
         this.geometryDataChannel = this.createDataChannel("geometry_unframed",80);
         this.reliableDataChannel = this.createDataChannel("reliable",100);
         this.unreliableDataChannel = this.createDataChannel("unreliable",120,false);
-      
+
+        this._setupAudioMediaTrack();
+
         function onMessage({ data }) {
           if (data === 'ping') {
             //dataChannel.send('pong');
           }
         }
-      
+
         // NOTE(mroberts): This is a hack so that we can get a callback when the
         // RTCPeerConnection is closed. In the future, we can subscribe to
         // "connectionstatechange" events.
@@ -439,6 +450,52 @@ class WebRtcConnection extends EventEmitter
           return close.apply(this, arguments);
         };
       }
+    // Adds one sendrecv audio transceiver per the protocol (see docs/protocol/audio.rst).
+    // Our outbound track is an RTCAudioSource so a future SFU can push selected peer
+    // PCM into it; incoming RTP is unwrapped via RTCAudioSink and surfaced as the
+    // 'micFrame' event. If options.audioEchoTest is set, received frames are looped
+    // straight back to the source, which lets a single browser tab self-test the path.
+    _setupAudioMediaTrack() {
+        try {
+            this._audioSource = new wrtc.nonstandard.RTCAudioSource();
+            this._localAudioTrack = this._audioSource.createTrack();
+            this.peerConnection.addTransceiver(this._localAudioTrack, { direction: 'sendrecv' });
+        } catch (e) {
+            console.error('_setupAudioMediaTrack failed: '+e.message);
+            this._teardownAudioMediaTrack();
+        }
+    }
+    _handleTrack(event) {
+        if (!event || !event.track || event.track.kind !== 'audio')
+            return;
+        if (this._audioSink) {
+            try { this._audioSink.stop(); } catch (e) {}
+            this._audioSink = null;
+        }
+        try {
+            this._audioSink = new wrtc.nonstandard.RTCAudioSink(event.track);
+        } catch (e) {
+            console.error('RTCAudioSink construction failed: '+e.message);
+            return;
+        }
+        this._audioSink.ondata = (data) => {
+            this.emit('micFrame', this.id, data);
+            if (this.options && this.options.audioEchoTest && this._audioSource) {
+                try { this._audioSource.onData(data); } catch (e) {}
+            }
+        };
+    }
+    _teardownAudioMediaTrack() {
+        if (this._audioSink) {
+            try { this._audioSink.stop(); } catch (e) {}
+            this._audioSink = null;
+        }
+        if (this._localAudioTrack) {
+            try { this._localAudioTrack.stop(); } catch (e) {}
+            this._localAudioTrack = null;
+        }
+        this._audioSource = null;
+    }
       
 	receiveMessage(id,event)
 	{

package/connections/webrtcconnectionmanager.js

@@ -91,6 +91,10 @@ class WebRtcConnectionManager
 		if (policy === 'all' || policy === 'relay')
 			this.options.iceTransportPolicy=policy;
 	}
+	SetAudioEchoTest(enabled)
+	{
+		this.options.audioEchoTest = !!enabled;
+	}
 }
 
 WebRtcConnectionManager.create = function create (options)

package/index.js

@@ -29,6 +29,8 @@ function initServer(signaling_port, options) {
 		webRtcConnectionManager.SetIceServers(options.iceServers);
 	if (options && options.iceTransportPolicy)
 		webRtcConnectionManager.SetIceTransportPolicy(options.iceTransportPolicy);
+	if (options && typeof options.audioEchoTest !== 'undefined')
+		webRtcConnectionManager.SetAudioEchoTest(options.audioEchoTest);
 	return signaling.init(serverID, webRtcConnectionManager,cm.newClient.bind(cm),cm.disconnectClient.bind(cm),signaling_port);
   }
   

package/package.json

@@ -16,7 +16,7 @@
   },
   "name": "teleportxr",
   "description": "Teleport Spatial Server on node.js",
-  "version": "1.0.83",
+  "version": "1.0.84",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/simul/teleport-nodejs.git"

package/utils/cubemap-reproject.js

@@ -0,0 +1,129 @@
+const sharp = require('sharp');
+const { parseKtx, buildKtx } = require('ktx-parse'); // install ktx-parse
+const os = require('os');
+
+const FACE_NAMES = ['+X','-X','+Y','-Y','+Z','-Z'];
+const CONVENTIONS = {
+  Engineering: { upAxis: 'Z', handed: 'R' },
+  OpenGL:      { upAxis: 'Y', handed: 'R' },
+  Unreal:      { upAxis: 'Z', handed: 'L' },
+  Unity:       { upAxis: 'Y', handed: 'L' }
+};
+
+// ... (include basisForConvention, mat3MulVec, mat3Transpose, dirToFaceUV, etc.)
+// Use same helper implementations from previous file (omitted here for brevity).
+// Paste the functions: basisForConvention, mat3MulVec, mat3Transpose, dirToFaceUV,
+// mat3Transpose, convertVectorBetweenConventions, sampleFaceNearest, readFaces, FACE_NAMES, CONVENTIONS, etc.
+
+// ---------- KTX I/O helpers ----------
+
+// parseKtx(buffer) returns an object describing levels/images; we expect cubemap with 6 faces.
+// This helper extracts the first mip level and returns an object:
+// { faces: Map faceName-> { w,h,data }, faceOrder: array of face names in input file order }
+async function readKtx(buffer) {
+  const k = parseKtx(buffer); // throws on invalid
+  // ktx-parse exposes ktx.header, ktx.images: array of levels; for cubemap, images[0] is array of 6 faces
+  if (!k.header || !k.images || k.images.length === 0) throw new Error('Invalid KTX file');
+  const level0 = k.images[0];
+  // level0 should be an array of 6 face objects (each with pixelData Uint8Array)
+  if (!Array.isArray(level0) || level0.length < 6) throw new Error('Not a cubemap or missing faces');
+  // Determine width/height from header
+  const w = k.header.pixelWidth;
+  const h = k.header.pixelHeight;
+  if (w !== h) throw new Error('Non-square faces not supported');
+  // ktx-parse gives faces in the order stored in file. We'll assume canonical order +X,-X,+Y,-Y,+Z,-Z
+  // If the file stores in a different order, user must provide ordering; many KTX cubemaps follow canonical.
+  const faces = {};
+  for (let i = 0; i < 6; i++) {
+    const img = level0[i];
+    // img.pixelData is Uint8Array in the file format's pixel encoding; parseKtx should give raw RGBA for uncompressed
+    let data = img.pixelData;
+    // If data is not RGBA raw, use sharp to decode buffer (e.g., PNG inside KTX) — handle common cases:
+    if (!(data instanceof Uint8Array) || data.length !== w * h * 4) {
+      // try to decode with sharp (supports PNG/JPEG inside)
+      const decoded = await sharp(Buffer.from(data)).ensureAlpha().raw().toBuffer();
+      data = decoded;
+    } else {
+      data = Buffer.from(data);
+    }
+    faces[FACE_NAMES[i]] = { w, h, data, channels: 4 };
+  }
+  return { faces, faceOrder: FACE_NAMES.slice() };
+}
+
+// Build a KTX2 buffer from 6 RGBA face Buffers using ktx-parse builder.
+// options: {isKTX2: true/false, format: 'RGBA8' }
+function writeKtx(faceBuffersMap, faceOrder = FACE_NAMES, options = { isKTX2: true }) {
+  // build images array: single level with 6 faces in the specified order
+  const facesArray = faceOrder.map(fn => {
+    const f = faceBuffersMap[fn];
+    return { pixelData: f.data, width: f.w, height: f.h };
+  });
+  const ktx = buildKtx({
+    pixelWidth: facesArray[0].width,
+    pixelHeight: facesArray[0].height,
+    images: [facesArray],
+    isKTX2: options.isKTX2,
+    // choose uncompressed RGBA8 format
+    vkFormat: 'VK_FORMAT_R8G8B8A8_UNORM'
+  });
+  return Buffer.from(ktx);
+}
+
+// ---------- Core conversion using raw face maps ----------
+
+async function convertCubemapFromFaceMap(faceMap, srcConvention, dstConvention) {
+  // faceMap: faceName-> {w,h,data}
+  const srcBasis = basisForConvention(CONVENTIONS[srcConvention]);
+  const dstBasis = basisForConvention(CONVENTIONS[dstConvention]);
+  const faceSize = faceMap[FACE_NAMES[0]].w;
+  const dstRawFaces = {};
+  for (const faceName of FACE_NAMES) {
+    const w = faceSize, h = faceSize;
+    const out = Buffer.alloc(w*h*4);
+    for (let y=0;y<h;y++) {
+      for (let x=0;x<w;x++) {
+        const u = (2*(x + 0.5)/w - 1);
+        const v = (2*(y + 0.5)/h - 1);
+        let dirLocal;
+        switch(faceName) {
+          case '+X': dirLocal = [1, -v, -u]; break;
+          case '-X': dirLocal = [-1, -v, u]; break;
+          case '+Y': dirLocal = [u, 1, v]; break;
+          case '-Y': dirLocal = [u, -1, -v]; break;
+          case '+Z': dirLocal = [u, -v, 1]; break;
+          case '-Z': dirLocal = [-u, -v, -1]; break;
+        }
+        const L = Math.hypot(dirLocal[0],dirLocal[1],dirLocal[2]);
+        dirLocal = dirLocal.map(c => c / L);
+        const srcDirLocal = convertVectorBetweenConventions(dirLocal, dstBasis, srcBasis);
+        const { face: srcFace, u: su, v: sv } = dirToFaceUV(srcDirLocal[0], srcDirLocal[1], srcDirLocal[2]);
+        const rgba = sampleFaceNearest(faceMap[srcFace], su, sv);
+        const idx = (y*w + x)*4;
+        out[idx]=rgba[0]; out[idx+1]=rgba[1]; out[idx+2]=rgba[2]; out[idx+3]=rgba[3];
+      }
+    }
+    dstRawFaces[faceName] = { w: faceSize, h: faceSize, data: out };
+  }
+  return dstRawFaces;
+}
+
+// ---------- Public wrapper: accepts KTX/KTX2 Buffer in, returns KTX/KTX2 Buffer out ----------
+async function convertCubemapKtx(inputKtxBuffer, srcConvention, dstConvention, opts = { isKTX2: true }) {
+  // read KTX into face map
+  const { faces: srcFaces, faceOrder } = await readKtx(inputKtxBuffer);
+  // convert
+  const dstFaces = await convertCubemapFromFaceMap(srcFaces, srcConvention, dstConvention);
+  // write KTX (preserve KTX2 if requested)
+  const outBuf = writeKtx(dstFaces, FACE_NAMES, opts);
+  return outBuf;
+}
+
+module.exports = {
+  convertCubemapKtx,
+  convertCubemapFromFaceMap,
+  writeKtx,
+  readKtx,
+  FACE_NAMES,
+  CONVENTIONS
+};