teleportxr 1.0.85 → 1.0.87

package/client/client.js

@@ -99,10 +99,21 @@ class Client {
 			// channels are in 'open' state yet; sends issued now will throw
 			// InvalidStateError. Instead, UpdateStreaming is triggered from the
 			// dataChannelsOpen callback registered in StartStreaming().
+			// Begin streaming scene SoundComponents (server-side audio) over the
+			// outbound media track. Safe even if no sound components are present.
+			if (this.webRtcConnection && this.scene)
+			{
+				try { this.webRtcConnection.startSceneAudio(this.scene); }
+				catch (e) { console.error("startSceneAudio threw: "+e.message); }
+			}
 		}
 		else
 		{
 			this.webRtcConnected=false;
+			if (this.webRtcConnection)
+			{
+				try { this.webRtcConnection.stopSceneAudio(); } catch (e) {}
+			}
 		}
     }
 	// Invoked once per WebRTC connection, the moment both reliable and geometry

package/connections/webrtcconnection.js

@@ -1,8 +1,10 @@
 'use strict';
 
 const EventEmitter = require('events');
+const path = require('path');
 const wrtc =require('@roamhq/wrtc');
 const DefaultRTCPeerConnection = require('@roamhq/wrtc').RTCPeerConnection;
+const sound = require('../scene/sound.js');
 
 const TIME_TO_CONNECTED = 30000;
 const TIME_TO_HOST_CANDIDATES = 3000;  // NOTE: Too long.
@@ -71,6 +73,8 @@ class WebRtcConnection extends EventEmitter
 		this._audioSource = null;
 		this._localAudioTrack = null;
 		this._audioSink = null;
+		this._sceneAudioStreamer = null;
+		this._sceneAudioInterval = null;
 
 
 		Object.defineProperties(this, {
@@ -486,6 +490,7 @@ class WebRtcConnection extends EventEmitter
         };
     }
     _teardownAudioMediaTrack() {
+        this.stopSceneAudio();
         if (this._audioSink) {
             try { this._audioSink.stop(); } catch (e) {}
             this._audioSink = null;
@@ -496,6 +501,85 @@ class WebRtcConnection extends EventEmitter
         }
         this._audioSource = null;
     }
+    // Start streaming all SoundComponents in the given scene as a single mixed
+    // 48 kHz / mono / int16 PCM track via the outbound audio media track. Idempotent
+    // for a given scene; calling again replaces the active source set.
+    startSceneAudio(scene) {
+        if (!this._audioSource) {
+            console.warn('startSceneAudio: no audio source on this connection');
+            return;
+        }
+        if (!scene || typeof scene.GetSoundComponents !== 'function') {
+            return;
+        }
+        const components = scene.GetSoundComponents();
+        if (!components || components.length === 0) {
+            return;
+        }
+        if (!this._sceneAudioStreamer) {
+            this._sceneAudioStreamer = new sound.SceneAudioStreamer(this._audioSource);
+        } else {
+            this._sceneAudioStreamer.clear();
+        }
+        const publicPath = (typeof scene.GetPublicPath === 'function') ? scene.GetPublicPath() : 'http_resources';
+        let loaded = 0;
+        for (const c of components) {
+            if (!c.url) continue;
+            const rel = c.url.startsWith('/') ? c.url.slice(1) : c.url;
+            const filePath = path.join(publicPath, rel);
+            try {
+                const src = new sound.WavSource(filePath);
+                this._sceneAudioStreamer.addSource(c.url, src);
+                loaded++;
+            } catch (e) {
+                console.warn('startSceneAudio: failed to load '+filePath+': '+e.message);
+            }
+        }
+        if (loaded === 0) {
+            return;
+        }
+        if (this._sceneAudioInterval) {
+            clearInterval(this._sceneAudioInterval);
+            this._sceneAudioInterval = null;
+        }
+        // Drive a 10 ms tick. setInterval is sufficient: RTCAudioSource buffers
+        // internally, and a few ms of jitter is absorbed by the WebRTC jitter buffer.
+        this._sceneAudioTickCount = 0;
+        this._sceneAudioInterval = setInterval(() => {
+            if (!this._sceneAudioStreamer) return;
+            this._sceneAudioStreamer.tick();
+            this._sceneAudioTickCount++;
+            // Every ~5 s, log tick count + outbound RTP stats so we can confirm
+            // libwebrtc is actually transmitting audio packets.
+            if ((this._sceneAudioTickCount % 500) === 0) {
+                const n = this._sceneAudioTickCount;
+                if (typeof this.peerConnection.getStats === 'function') {
+                    Promise.resolve(this.peerConnection.getStats()).then((report) => {
+                        let summary = 'no outbound-rtp/audio stats';
+                        try {
+                            const entries = (typeof report.values === 'function') ? Array.from(report.values()) : Object.values(report);
+                            const outRtp = entries.find((s) => s && s.type === 'outbound-rtp' && (s.kind === 'audio' || s.mediaType === 'audio'));
+                            if (outRtp) summary = 'packetsSent='+outRtp.packetsSent+' bytesSent='+outRtp.bytesSent;
+                        } catch (e) { summary = 'stats parse error: '+e.message; }
+                        console.log('SceneAudioStreamer tick #'+n+' — '+summary);
+                    }).catch((e) => { console.log('SceneAudioStreamer tick #'+n+' — getStats failed: '+e.message); });
+                } else {
+                    console.log('SceneAudioStreamer tick #'+n+' — getStats unavailable');
+                }
+            }
+        }, sound.FRAME_MS);
+        console.log('startSceneAudio: streaming '+loaded+' source(s) at '+sound.SAMPLE_RATE+' Hz mono');
+    }
+    stopSceneAudio() {
+        if (this._sceneAudioInterval) {
+            clearInterval(this._sceneAudioInterval);
+            this._sceneAudioInterval = null;
+        }
+        if (this._sceneAudioStreamer) {
+            this._sceneAudioStreamer.clear();
+            this._sceneAudioStreamer = null;
+        }
+    }
       
 	receiveMessage(id,event)
 	{

package/package.json

@@ -16,7 +16,7 @@
   },
   "name": "teleportxr",
   "description": "Teleport Spatial Server on node.js",
-  "version": "1.0.85",
+  "version": "1.0.87",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/simul/teleport-nodejs.git"

package/scene/node.js

@@ -227,6 +227,19 @@ class SkeletonComponent extends Component
 	}
 };
 
+// Server-side only. Sound components reference an audio asset on the server
+// and are streamed via the WebRTC audio media track; they are never encoded
+// into the scene-graph payload sent to clients.
+class SoundComponent
+{
+	constructor(url = "")
+	{
+		this.url = url;
+		this.loop = true;
+		this.gain = 1.0;
+	}
+};
+
 class Node {
 	constructor( name = "") {
 		this.uid = core.generateUid();
@@ -240,6 +253,9 @@ class Node {
 		this.priority = 0;
 
 		this.components = [];
+		// Server-side only audio sources attached to this node. Not encoded
+		// into the wire payload; consumed by SceneAudioStreamer.
+		this.soundComponents = [];
 	}
 	static sizeof() {
 		return 8 + 24 + Pose.size + 8;
@@ -267,6 +283,12 @@ class Node {
 		);
 		this.components.push(m);
 	}
+	setSoundComponent(url) {
+		for (const c of this.soundComponents) {
+			if (c.url === url) return;
+		}
+		this.soundComponents.push(new SoundComponent(url));
+	}
 	setCanvasComponent(canvas_path) {
 		this.components.forEach((component) => {
 			if (component.getType() == NodeDataType.TextCanvas) {
@@ -356,4 +378,4 @@ class Node {
 	}
 };
 
-module.exports = {NodeDataType,Pose,PoseDynamic,NodePoseDynamic, Node };
+module.exports = {NodeDataType,Pose,PoseDynamic,NodePoseDynamic, Node, SoundComponent };

package/scene/scene.js

@@ -52,6 +52,21 @@ class Scene {
 		let node_uids = Array.from(this.nodes.keys());
 		return node_uids;
 	}
+	// Returns [{ nodeUid, url, loop, gain }] for every SoundComponent in the scene.
+	// Sound components are server-side only and never appear in the encoded node payload.
+	GetSoundComponents() {
+		const out = [];
+		for (const [uid, node] of this.nodes) {
+			if (!node.soundComponents || node.soundComponents.length === 0) continue;
+			for (const sc of node.soundComponents) {
+				out.push({ nodeUid: uid, url: sc.url, loop: sc.loop, gain: sc.gain });
+			}
+		}
+		return out;
+	}
+	GetPublicPath() {
+		return this.publicPath;
+	}
 	LoadFontAtlas(res, filename) {
 		const data = fs.readFileSync(filename, "utf8");
 		const jfa = JSON.parse(data);
@@ -252,6 +267,10 @@ class Scene {
 							var canvas=c.url;
 							n.setCanvasComponent(canvas);
 						}
+						if (c["type"] == "sound")
+						{
+							n.setSoundComponent(c["url"]);
+						}
 					}
 				}
 			}

package/scene/sound.js

@@ -0,0 +1,136 @@
+// SPDX-FileCopyrightText: 2025 Teleport XR Ltd <contact@teleportxr.io>
+//
+// SPDX-License-Identifier: MIT
+//
+// Server-side audio streaming for scene "sound" components.
+//
+// Sound components are kept server-side: their PCM is mixed and pushed into
+// an @roamhq/wrtc RTCAudioSource as 10 ms / 48 kHz / mono / int16 frames,
+// which libwebrtc encodes to Opus on the outbound audio media track.
+'use strict';
+const fs = require('fs');
+
+// Required output format for RTCAudioSource.onData.
+const SAMPLE_RATE = 48000;
+const CHANNEL_COUNT = 1;
+const FRAME_MS = 10;
+const FRAME_SAMPLES = (SAMPLE_RATE * FRAME_MS) / 1000; // 480
+
+// Parse a minimal RIFF/WAVE file. Returns { samples: Int16Array, sampleRate, channelCount }
+// or throws on any unsupported variant (we only accept mono / 16-bit / 48 kHz).
+function parseWav(buffer) {
+	if (buffer.length < 44) throw new Error('WAV: file too short');
+	if (buffer.toString('ascii', 0, 4) !== 'RIFF') throw new Error('WAV: missing RIFF header');
+	if (buffer.toString('ascii', 8, 12) !== 'WAVE') throw new Error('WAV: missing WAVE marker');
+	let pos = 12;
+	let fmt = null;
+	let dataOffset = -1;
+	let dataSize = 0;
+	while (pos + 8 <= buffer.length) {
+		const id = buffer.toString('ascii', pos, pos + 4);
+		const size = buffer.readUInt32LE(pos + 4);
+		const next = pos + 8 + size + (size & 1); // chunks are word-aligned
+		if (id === 'fmt ') {
+			fmt = {
+				audioFormat: buffer.readUInt16LE(pos + 8),
+				channels: buffer.readUInt16LE(pos + 10),
+				sampleRate: buffer.readUInt32LE(pos + 12),
+				bitsPerSample: buffer.readUInt16LE(pos + 22)
+			};
+		} else if (id === 'data') {
+			dataOffset = pos + 8;
+			dataSize = size;
+			break;
+		}
+		pos = next;
+	}
+	if (!fmt) throw new Error('WAV: no fmt chunk');
+	if (dataOffset < 0) throw new Error('WAV: no data chunk');
+	if (fmt.audioFormat !== 1) throw new Error('WAV: only PCM (audioFormat=1) is supported, got '+fmt.audioFormat);
+	if (fmt.channels !== CHANNEL_COUNT) throw new Error('WAV: only mono is supported, got channels='+fmt.channels);
+	if (fmt.sampleRate !== SAMPLE_RATE) throw new Error('WAV: only '+SAMPLE_RATE+' Hz is supported, got '+fmt.sampleRate);
+	if (fmt.bitsPerSample !== 16) throw new Error('WAV: only 16-bit PCM is supported, got bitsPerSample='+fmt.bitsPerSample);
+	const numSamples = Math.floor(dataSize / 2);
+	const samples = new Int16Array(numSamples);
+	for (let i = 0; i < numSamples; i++) {
+		samples[i] = buffer.readInt16LE(dataOffset + i * 2);
+	}
+	return { samples, sampleRate: fmt.sampleRate, channelCount: fmt.channels };
+}
+
+// One looping mono int16 PCM source. pull() writes FRAME_SAMPLES samples into
+// the supplied Int32 accumulator (caller mixes; we just add).
+class WavSource {
+	constructor(filePath) {
+		const buf = fs.readFileSync(filePath);
+		const wav = parseWav(buf);
+		this.samples = wav.samples;
+		this.position = 0;
+		this.filePath = filePath;
+	}
+	pull(accumulator) {
+		const len = this.samples.length;
+		if (len === 0) return;
+		let pos = this.position;
+		for (let i = 0; i < FRAME_SAMPLES; i++) {
+			accumulator[i] += this.samples[pos];
+			pos++;
+			if (pos >= len) pos = 0;
+		}
+		this.position = pos;
+	}
+}
+
+// SceneAudioStreamer mixes all active sources at 48 kHz mono int16 and
+// pushes one 10 ms frame per tick into the supplied RTCAudioSource.
+class SceneAudioStreamer {
+	constructor(audioSource) {
+		this.audioSource = audioSource;
+		this.sources = []; // [{ url, source }]
+		this.mixBuffer = new Int32Array(FRAME_SAMPLES);
+		this.outputBuffer = new Int16Array(FRAME_SAMPLES);
+		this.frame = {
+			samples: this.outputBuffer,
+			sampleRate: SAMPLE_RATE,
+			bitsPerSample: 16,
+			channelCount: CHANNEL_COUNT,
+			numberOfFrames: FRAME_SAMPLES
+		};
+	}
+	addSource(url, source) {
+		this.sources.push({ url, source });
+	}
+	clear() {
+		this.sources = [];
+	}
+	tick() {
+		const mix = this.mixBuffer;
+		mix.fill(0);
+		for (const s of this.sources) {
+			s.source.pull(mix);
+		}
+		// Clip to int16 range.
+		const out = this.outputBuffer;
+		for (let i = 0; i < FRAME_SAMPLES; i++) {
+			let v = mix[i];
+			if (v > 32767) v = 32767;
+			else if (v < -32768) v = -32768;
+			out[i] = v;
+		}
+		try {
+			this.audioSource.onData(this.frame);
+		} catch (e) {
+			console.error('SceneAudioStreamer: onData failed: '+e.message);
+		}
+	}
+}
+
+module.exports = {
+	SAMPLE_RATE,
+	CHANNEL_COUNT,
+	FRAME_MS,
+	FRAME_SAMPLES,
+	parseWav,
+	WavSource,
+	SceneAudioStreamer
+};