teleportxr 1.0.93 → 1.0.94

package/client/client.js

@@ -67,6 +67,12 @@ class Client {
 		this.webRtcConnection=null;
 		this.currentOriginState=new OriginState();
 		this.currentLightingState=new LightingState();
+		// Latest head pose reported by the client (node.Pose), or null before the first
+		// NodePosesMessage. Used server-side for proximity-based audio selection.
+		this.currentHeadPose=null;
+		// When true, the SetupCommand tells the client to send its microphone track
+		// (needed for the audio SFU). Set by the host before Start(); default off.
+		this.acceptMicrophone=false;
 		this.next_ack_id=BigInt(1);
 		this.clientStartMs=Date.now();
 		this.webRtcConnectedAtMs=0;
@@ -211,6 +217,9 @@ class Client {
 		this.clientStartMs=Date.now();
 		console.log("[T+0ms] Client.Start() — sending SetupCommand for client "+this.clientID);
         this.setupCommand=new command.SetupCommand();
+        // Accept the client's microphone media track when the host has enabled it
+        // (required for the audio SFU to receive and forward this client's voice).
+        this.setupCommand.uint8_audio_input_enabled = this.acceptMicrophone ? 1 : 0;
         this.clientDynamicLighting=new core.ClientDynamicLighting();
 		// Session is (re)starting; the client has zero state, so retract any
 		// outstanding ack tracking from a previous session and force a resend.
@@ -385,6 +394,15 @@ class Client {
 	ProcessNodePoses(headPose,numPoses,nodePoses)
 	{
 		//console.log("Client: ProcessNodePoses ", numPoses, " poses.");
+		// Retain the head pose as this client's current world position, for
+		// proximity-based audio source selection (see src/mic-router.js).
+		if (headPose)
+			this.currentHeadPose = headPose;
+	}
+	// This client's current world position {x,y,z}, or null if not yet known.
+	GetHeadPosition()
+	{
+		return (this.currentHeadPose && this.currentHeadPose.position) ? this.currentHeadPose.position : null;
 	}
 	UpdateStreaming()
 	{

package/connections/webrtcconnection.js

@@ -228,7 +228,12 @@ class WebRtcConnection extends EventEmitter
 			{
 				const offer = await pc.createOffer();
 				if (this.peerConnection !== pc) return;
-				await pc.setLocalDescription(offer);
+				// Rename each node-audio m-line's mid to the emitting node's uid
+				// (see docs/protocol/audio.rst) before it is committed, so the binding
+				// is intrinsic to the track. Munging pre-setLocalDescription is accepted
+				// by wrtc and reflected in pc.localDescription.
+				const mungedSdp = this._mungeAudioMids(offer.sdp);
+				await pc.setLocalDescription({ type: 'offer', sdp: mungedSdp });
 				if (this.peerConnection !== pc) return;
 				// Use pc.localDescription.sdp rather than the createOffer() result: the
 				// ICE ufrag/pwd in the SDP returned by createOffer() are provisional and
@@ -499,6 +504,98 @@ class WebRtcConnection extends EventEmitter
             this._localAudioTrack = null;
         }
         this._audioSource = null;
+        if (this._nodeAudioSources) {
+            for (const v of this._nodeAudioSources.values()) {
+                try { v.track.stop(); } catch (e) {}
+            }
+            this._nodeAudioSources.clear();
+        }
+    }
+    // ── Per-node forwarded audio tracks ────────────────────────────────────────
+    // A node may emit audio (a participant's microphone, but equally an object,
+    // ambient source or announcer). Each such source is its own sendonly
+    // transceiver whose SDP mid is set to the emitting node's uid (see
+    // docs/protocol/audio.rst), so the client binds and spatialises it by node.
+    // The caller pushes that node's PCM into the returned RTCAudioSource. Call
+    // renegotiate() after adding/removing to re-offer.
+    // sourceNodeUid: BigInt|number|string. Returns the RTCAudioSource (or null).
+    addNodeAudioSource(sourceNodeUid) {
+        const uid = BigInt(sourceNodeUid);
+        if (!uid) return null;
+        if (!this._nodeAudioSources) this._nodeAudioSources = new Map();
+        const existing = this._nodeAudioSources.get(uid);
+        if (existing) return existing.audioSource;
+        if (!this.peerConnection) return null;
+        try {
+            const audioSource = new wrtc.nonstandard.RTCAudioSource();
+            const track = audioSource.createTrack();
+            const transceiver = this.peerConnection.addTransceiver(track, { direction: 'sendonly' });
+            this._nodeAudioSources.set(uid, { audioSource, track, transceiver, trackId: track.id });
+            return audioSource;
+        } catch (e) {
+            console.error('addNodeAudioSource failed for node ' + uid + ': ' + e.message);
+            return null;
+        }
+    }
+    removeNodeAudioSource(sourceNodeUid) {
+        const uid = BigInt(sourceNodeUid);
+        if (!this._nodeAudioSources) return;
+        const v = this._nodeAudioSources.get(uid);
+        if (!v) return;
+        try { v.track.stop(); } catch (e) {}
+        // The m-line cannot be removed, but going inactive/silent retires the source.
+        try { if (v.transceiver) v.transceiver.direction = 'inactive'; } catch (e) {}
+        this._nodeAudioSources.delete(uid);
+    }
+    // The node uids of the node-audio sources currently attached to this listener.
+    getNodeAudioSourceUids() {
+        if (!this._nodeAudioSources) return [];
+        return [...this._nodeAudioSources.keys()];
+    }
+    // Re-offer to apply added/removed node-audio tracks. No-op unless the connection
+    // is in a stable signalling state (a pending offer will be picked up next tick).
+    async renegotiate() {
+        const pc = this.peerConnection;
+        if (!pc || typeof pc.signalingState === 'string' && pc.signalingState !== 'stable') return;
+        await this.doOffer();
+    }
+    // Rewrite each node-audio m-line's mid to its emitting node uid, correlating
+    // m-line -> node by the track id carried in the m-line's a=msid, and repair the
+    // BUNDLE group. Idempotent: an m-line already carrying its node-uid mid is skipped.
+    _mungeAudioMids(sdp) {
+        if (!this._nodeAudioSources || this._nodeAudioSources.size === 0) return sdp;
+        const byTrack = new Map();
+        for (const [uid, v] of this._nodeAudioSources) byTrack.set(v.trackId, uid.toString());
+        const lines = sdp.split(/\r\n|\n/);
+        const renames = new Map(); // oldMid -> newMid
+        let i = 0;
+        while (i < lines.length) {
+            if (!lines[i].startsWith('m=')) { i++; continue; }
+            let j = i + 1, midIdx = -1, oldMid = null, trackId = null;
+            while (j < lines.length && !lines[j].startsWith('m=')) {
+                const l = lines[j];
+                if (l.startsWith('a=mid:')) { midIdx = j; oldMid = l.slice(6).trim(); }
+                else if (l.startsWith('a=msid:')) {
+                    const parts = l.slice(7).trim().split(/\s+/); // "- <trackId>"
+                    if (parts.length >= 2) trackId = parts[1];
+                }
+                j++;
+            }
+            if (midIdx >= 0 && trackId && byTrack.has(trackId)) {
+                const newMid = byTrack.get(trackId);
+                if (newMid !== oldMid) { lines[midIdx] = 'a=mid:' + newMid; renames.set(oldMid, newMid); }
+            }
+            i = j;
+        }
+        if (renames.size === 0) return sdp;
+        for (let k = 0; k < lines.length; k++) {
+            if (!lines[k].startsWith('a=group:BUNDLE')) continue;
+            const toks = lines[k].split(/\s+/);
+            for (let t = 1; t < toks.length; t++)
+                if (renames.has(toks[t])) toks[t] = renames.get(toks[t]);
+            lines[k] = toks.join(' ');
+        }
+        return lines.join('\r\n');
     }
     // Start streaming all SoundComponents in the given scene as a single mixed
     // 48 kHz / mono / int16 PCM track via the outbound audio media track. Idempotent

package/core/core.js

@@ -32,6 +32,10 @@ function SizeOfType(member) {
 			return [16, "struct"];
 		case "uid":
 			return [8, "uint64"];
+		case "int16":
+			return [2, "int16"];
+		case "uint16":
+			return [2, "uint16"];
 		case "int32":
 			return [4, "int32"];
 		case "uint32":
@@ -42,6 +46,8 @@ function SizeOfType(member) {
 			return [8, "int64"];
 		case "VideoConfig":
 			return [89, "struct"];
+		case "AudioConfig":
+			return [17, "struct"];
 		case "VideoCodec":
 		case "AxesStandard":
 		case "GeometryPayloadType":
@@ -67,6 +73,12 @@ function encodeIntoDataView(obj, dataView, byteOffset) {
 		else if (tp == "int8") {
 			dataView.setInt8(byteOffset, value, endian);
 		}
+		else if (tp == "uint16") {
+			dataView.setUint16(byteOffset, value, endian);
+		}
+		else if (tp == "int16") {
+			dataView.setInt16(byteOffset, value, endian);
+		}
 		else if (tp == "uint32") {
 			dataView.setUint32(byteOffset, value, endian);
 		}
@@ -110,6 +122,14 @@ function decodeFromDataView(obj, dataView, byteOffset) {
 			var value = dataView.getInt8(byteOffset);
 			obj[key] = value;
 		}
+		else if (tp == "uint16") {
+			var value = dataView.getUint16(byteOffset, endian);
+			obj[key] = value;
+		}
+		else if (tp == "int16") {
+			var value = dataView.getInt16(byteOffset, endian);
+			obj[key] = value;
+		}
 		else if (tp == "uint32") {
 			var value = dataView.getUint32(byteOffset, endian);
 			obj[key] = value;
@@ -395,6 +415,24 @@ class VideoConfig {
 	}
 };	// 89 bytes
 
+//! Audio configuration carried inside SetupCommand (17 bytes).
+//! See docs/protocol/audio.rst §AudioConfig for the full specification.
+class AudioConfig {
+	constructor() {
+		this.uint8_codec             = 1;    //!< 0=disabled; 1=Opus
+		this.uint8_rtpPayloadType    = 111;
+		this.uint32_sampleRateHz     = 48000;
+		this.uint8_channelCount      = 1;
+		this.uint8_frameDurationMs   = 20;
+		this.uint8_flags             = 3;    //!< bit0=FEC, bit1=DTX
+		this.uint8_maxInboundStreams = 0;
+		this.uint8_selectionPolicy   = 0;    //!< 0=All
+		this.float32_proximityRadiusMetres = 0.0;
+		this.uint16_evictionGraceMs  = 0;
+	}
+	static sizeof() { return 17; }
+};
+
 //! Setup for dynamically-lit objects on the client.
 class ClientDynamicLighting {
 	constructor() {
@@ -568,7 +606,7 @@ module.exports = {
 	, vec4, BackgroundMode, AxesStandard, AxesStandardToCubemapSuffix
 	, ConvertPosition, ConvertRotation, ConvertScale, ConvertPose
 	, GeometryPayloadType, DisplayInfo, RenderingFeatures, LightingMode, VideoCodec
-	, VideoConfig, ClientDynamicLighting, encodeToUint8Array, decodeFromUint8Array
+	, VideoConfig, AudioConfig, ClientDynamicLighting, encodeToUint8Array, decodeFromUint8Array
 	, generateUid, getStartTimeUnixUs, getTimestampUs,
 	unixTimeToUTCString, put_float32, put_uint16, put_int32, put_uint32
 	, put_uint64, put_uint8, put_vec2, put_vec3, put_vec4, put_string

package/package.json

@@ -16,7 +16,7 @@
   },
   "name": "teleportxr",
   "description": "Teleport Spatial Server on node.js",
-  "version": "1.0.93",
+  "version": "1.0.94",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/simul/teleport-nodejs.git"

package/protocol/command.js

@@ -21,7 +21,9 @@ const CommandPayloadType =
 	AssignNodePosePath:14,				
 	SetupInputs:15,
 	PingForLatency:16,
-	SetOriginNode:128		
+	AudioSourceMapping:17,
+	AudioParticipantStateChange:18,
+	SetOriginNode:128
 };
 class Command{
 	constructor(){
@@ -44,19 +46,20 @@ class SetupCommand extends Command
         this.int32_requiredLatencyMs = 0;							    //!< 9+4=13
         this.uint32_idle_connection_timeout = 5000.0;				    //!< 13+4=17
         this.uint64_session_id = BigInt.asUintN(64, BigInt(0));							    //!< 17+8=25	The server's session id changes when the server session changes.	37 bytes
-        this.VideoConfig_video_config=new core.VideoConfig();			    //!< 25+89=114	Video setup structure. 41+89=130 bytes
-        this.float32_draw_distance = 0.0;								    //!< 114+4=118	Maximum distance in metres to render locally. 134
-        this.AxesStandard_axesStandard = core.AxesStandard.NotInitialized;	//!< 118+1=119	The axis standard that the server uses, may be different from the client's. 147
-        this.uint8_audio_input_enabled = 0;							    //!< 119+1=120	Server accepts audio stream from client.
-        this.bool_using_ssl = true;									    //!< 120+1=121	Not in use, for later.
-        this.int64_startTimestamp_utc_unix_us = BigInt.asUintN(64,  BigInt(0));			    //!< 121+8=129	UTC Unix Timestamp in microseconds when the server session began.
-        // TODO: replace this with a background Material, which MAY contain video, te			xture and/or plain colours.
-        this.BackgroundMode_backgroundMode=core.BackgroundMode.COLOUR;   	//!< 129+1=130	Whether the server supplies a background, and of which type.
-        this.vec4_backgroundColour=new core.vec4();						    //!< 130+16=146 If the background is of the COLOUR type, which colour to use.
-        this.uid_backgroundTexture=BigInt(0);
+        this.VideoConfig_video_config=new core.VideoConfig();			    //!< 25+89=114	Video setup structure.
+        this.AudioConfig_audio_config=new core.AudioConfig();			    //!< 114+17=131	Audio media-track config.
+        this.float32_draw_distance = 0.0;								    //!< 131+4=135	Maximum distance in metres to render locally.
+        this.AxesStandard_axesStandard = core.AxesStandard.NotInitialized;	//!< 135+1=136	The axis standard that the server uses.
+        this.uint8_audio_input_enabled = 0;							    //!< 136+1=137	Server accepts a microphone media track from the client.
+        this.bool_using_ssl = true;									    //!< 137+1=138	Not in use, for later.
+        this.int64_startTimestamp_utc_unix_us = BigInt.asUintN(64,  BigInt(0));			    //!< 138+8=146	UTC Unix Timestamp in microseconds when the server session began.
+        // TODO: replace this with a background Material, which MAY contain video, texture and/or plain colours.
+        this.BackgroundMode_backgroundMode=core.BackgroundMode.COLOUR;   	//!< 146+1=147	Whether the server supplies a background, and of which type.
+        this.vec4_backgroundColour=new core.vec4();						    //!< 147+16=163	If the background is of the COLOUR type, which colour to use.
+        this.uid_backgroundTexture=BigInt(0);                               //!< 163+8=171
     }
     static sizeof(){
-        return 154;
+        return 171;
     }
     size(){
         return SetupCommand.sizeof();
@@ -143,4 +146,33 @@ class SetLightingCommand extends AckedCommand
 }
 
 
-module.exports= {Command,CommandPayloadType,SetupCommand,AcknowledgeHandshakeCommand,SetOriginNodeCommand,SetLightingCommand};
+//! Sent from server to client when the set of audio tracks delivered to a client changes.
+//! The fixed-size header is followed on the wire by addedCount AddedEntry records
+//! (uint8 midLen + midLen UTF-8 bytes + uint64 sourceClientUid) then removedCount
+//! RemovedEntry records (uint8 midLen + midLen UTF-8 bytes).
+class AudioSourceMappingCommand extends Command
+{
+	constructor(){
+		super();
+		this.CommandPayloadType_commandPayloadType = CommandPayloadType.AudioSourceMapping;
+		this.uint16_addedCount   = 0;
+		this.uint16_removedCount = 0;
+	}
+	static sizeof() { return 5; } // 1 tag + 2 addedCount + 2 removedCount
+	size() { return AudioSourceMappingCommand.sizeof(); }
+}
+
+//! Sent from server to client to report user-visible audio state changes for participants.
+//! Followed by updateCount × 10-byte Update records (uint64 sourceClientUid + uint8 state + uint8 reason).
+class AudioParticipantStateChangeCommand extends Command
+{
+	constructor(){
+		super();
+		this.CommandPayloadType_commandPayloadType = CommandPayloadType.AudioParticipantStateChange;
+		this.uint16_updateCount = 0;
+	}
+	static sizeof() { return 3; } // 1 tag + 2 updateCount
+	size() { return AudioParticipantStateChangeCommand.sizeof(); }
+}
+
+module.exports= {Command,CommandPayloadType,SetupCommand,AcknowledgeHandshakeCommand,SetOriginNodeCommand,SetLightingCommand,AudioSourceMappingCommand,AudioParticipantStateChangeCommand};

package/scene/node.js

@@ -12,7 +12,8 @@ const NodeDataType =
 	SubScene:5,
 	Skeleton:6,
 	Link:7,
-	Script:8
+	Script:8,
+	AudioEmitter:9		// Reserved. Audio is bound via the track SDP mid = emitting node uid; not carried in the node payload.
 };
 
 class Pose