xrblocks 0.3.1 → 0.5.0

package/build/xrblocks.js

@@ -14,9 +14,9 @@
  * limitations under the License.
  *
  * @file xrblocks.js
- * @version v0.3.1
- * @commitid 8945d46
- * @builddate 2025-11-04T06:32:35.799Z
+ * @version v0.5.0
+ * @commitid c2f4b09
+ * @builddate 2025-12-04T15:14:30.184Z
  * @description XR Blocks SDK, built from source with the above commit ID.
  * @agent When using with Gemini to create XR apps, use **Gemini Canvas** mode,
  * and follow rules below:
@@ -2408,6 +2408,8 @@ class DepthOptions {
         // Occlusion pass.
         this.occlusion = { enabled: false };
         this.useFloat32 = true;
+        this.depthTypeRequest = ['raw'];
+        this.matchDepthView = true;
         deepMerge(this, options);
     }
 }
@@ -3208,9 +3210,9 @@ class Depth {
     update(frame) {
         if (!this.options.enabled)
             return;
-        if (!frame)
-            return;
-        this.updateLocalDepth(frame);
+        if (frame) {
+            this.updateLocalDepth(frame);
+        }
         if (this.options.occlusion.enabled) {
             this.renderOcclusionPass();
         }
@@ -4284,6 +4286,7 @@ class WebXRSessionManager extends THREE.EventDispatcher {
         this.sessionInit = sessionInit;
         this.mode = mode;
         this.onSessionEndedBound = this.onSessionEndedInternal.bind(this);
+        this.waitingForXRSession = false;
     }
     /**
      * Checks for WebXR support and availability of the requested session mode.
@@ -4350,8 +4353,15 @@ class WebXRSessionManager extends THREE.EventDispatcher {
         else if (this.currentSession) {
             throw new Error('Session already started');
         }
+        else if (this.waitingForXRSession) {
+            throw new Error('Waiting for session to start');
+        }
+        this.waitingForXRSession = true;
         navigator
             .xr.requestSession(this.mode, this.sessionOptions)
+            .finally(() => {
+            this.waitingForXRSession = false;
+        })
             .then(this.onSessionStartedInternal.bind(this));
     }
     /**
@@ -4394,19 +4404,25 @@ class WebXRSessionManager extends THREE.EventDispatcher {
 const XRBUTTON_WRAPPER_ID = 'XRButtonWrapper';
 const XRBUTTON_CLASS = 'XRButton';
 class XRButton {
-    constructor(sessionManager, startText = 'ENTER XR', endText = 'END XR', invalidText = 'XR NOT SUPPORTED', startSimulatorText = 'START SIMULATOR', enableSimulator = false, startSimulator = () => { }) {
+    constructor(sessionManager, permissionsManager, startText = 'ENTER XR', endText = 'END XR', invalidText = 'XR NOT SUPPORTED', startSimulatorText = 'START SIMULATOR', showEnterSimulatorButton = false, startSimulator = () => { }, permissions = {
+        geolocation: false,
+        camera: false,
+        microphone: false,
+    }) {
         this.sessionManager = sessionManager;
+        this.permissionsManager = permissionsManager;
         this.startText = startText;
         this.endText = endText;
         this.invalidText = invalidText;
         this.startSimulatorText = startSimulatorText;
         this.startSimulator = startSimulator;
+        this.permissions = permissions;
         this.domElement = document.createElement('div');
         this.simulatorButtonElement = document.createElement('button');
         this.xrButtonElement = document.createElement('button');
         this.domElement.id = XRBUTTON_WRAPPER_ID;
         this.createXRButtonElement();
-        if (enableSimulator) {
+        if (showEnterSimulatorButton) {
             this.createSimulatorButton();
         }
         this.sessionManager.addEventListener(WebXRSessionEventType.UNSUPPORTED, this.showXRNotSupported.bind(this));
@@ -4435,7 +4451,17 @@ class XRButton {
         button.innerHTML = this.startText;
         button.disabled = false;
         button.onclick = () => {
-            this.sessionManager.startSession();
+            this.permissionsManager
+                .checkAndRequestPermissions(this.permissions)
+                .then((result) => {
+                if (result.granted) {
+                    this.sessionManager.startSession();
+                }
+                else {
+                    this.xrButtonElement.textContent =
+                        'Error:' + result.error + '\nPlease try again.';
+                }
+            });
         };
     }
     showXRNotSupported() {
@@ -7164,6 +7190,7 @@ class SimulatorOptions {
     constructor(options) {
         this.initialCameraPosition = { x: 0, y: 1.5, z: 0 };
         this.scenePath = XR_BLOCKS_ASSETS_PATH + 'simulator/scenes/XREmulatorsceneV5_livingRoom.glb';
+        this.videoPath = undefined;
         this.initialScenePosition = { x: -1.6, y: 0.3, z: 0 };
         this.defaultMode = SimulatorMode.USER;
         this.defaultHand = Handedness.LEFT;
@@ -7180,7 +7207,10 @@ class SimulatorOptions {
             enabled: true,
             element: 'xrblocks-simulator-hand-pose-panel',
         };
-        this.geminilive = false;
+        this.geminiLivePanel = {
+            enabled: false,
+            element: 'xrblocks-simulator-geminilive',
+        };
         this.stereo = {
             enabled: false,
         };
@@ -7414,6 +7444,7 @@ class Options {
          * Whether to use post-processing effects.
          */
         this.usePostprocessing = false;
+        this.enableSimulator = true;
         /**
          * Configuration for the XR session button.
          */
@@ -7423,10 +7454,18 @@ class Options {
             endText: 'Exit XR',
             invalidText: 'XR Not Supported',
             startSimulatorText: 'Enter Simulator',
-            enableSimulator: true,
+            showEnterSimulatorButton: false,
             // Whether to autostart the simulator even if WebXR is available.
             alwaysAutostartSimulator: false,
         };
+        /**
+         * Which permissions to request before entering the XR session.
+         */
+        this.permissions = {
+            geolocation: false,
+            camera: false,
+            microphone: false,
+        };
         deepMerge(this, options);
     }
     /**
@@ -7467,6 +7506,7 @@ class Options {
      * @returns The instance for chaining.
      */
     enableObjectDetection() {
+        this.permissions.camera = true;
         this.world.enableObjectDetection();
         return this;
     }
@@ -7477,6 +7517,7 @@ class Options {
      * @returns The instance for chaining.
      */
     enableCamera(facingMode = 'environment') {
+        this.permissions.camera = true;
         this.deviceCamera = new DeviceCameraOptions(facingMode === 'environment'
             ? xrDeviceCameraEnvironmentOptions
             : xrDeviceCameraUserOptions);
@@ -7507,14 +7548,6 @@ class Options {
         this.controllers.visualizeRays = true;
         return this;
     }
-    /**
-     * Enables the Gemini Live feature.
-     * @returns The instance for chaining.
-     */
-    enableGeminiLive() {
-        this.simulator.geminilive = true;
-        return this;
-    }
     /**
      * Enables a standard set of AI features, including Gemini Live.
      * @returns The instance for chaining.
@@ -9776,8 +9809,8 @@ class SimulatorInterface {
         }
     }
     showGeminiLivePanel(simulatorOptions) {
-        if (simulatorOptions.geminilive) {
-            const element = document.createElement('xrblocks-simulator-geminilive');
+        if (simulatorOptions.geminiLivePanel.enabled) {
+            const element = document.createElement(simulatorOptions.geminiLivePanel.element);
             document.body.appendChild(element);
             this.elements.push(element);
         }
@@ -9821,6 +9854,9 @@ class SimulatorScene extends THREE.Scene {
     }
     async init(simulatorOptions) {
         this.addLights();
+        if (simulatorOptions.videoPath) {
+            return;
+        }
         if (simulatorOptions.scenePath) {
             await this.loadGLTF(simulatorOptions.scenePath, new THREE.Vector3(simulatorOptions.initialScenePosition.x, simulatorOptions.initialScenePosition.y, simulatorOptions.initialScenePosition.z));
         }
@@ -9958,6 +9994,21 @@ class Simulator extends Script {
         if (this.options.stereo.enabled) {
             this.setupStereoCameras(camera);
         }
+        if (this.options.videoPath) {
+            this.videoElement = document.createElement('video');
+            this.videoElement.src = this.options.videoPath;
+            this.videoElement.loop = true;
+            this.videoElement.muted = true;
+            this.videoElement.play().catch((e) => {
+                console.error(`Simulator: Failed to play video at ${this.options.videoPath}`, e);
+            });
+            this.videoElement.addEventListener('error', () => {
+                console.error(`Simulator: Error loading video at ${this.options.videoPath}`, this.videoElement?.error);
+            });
+            const videoTexture = new THREE.VideoTexture(this.videoElement);
+            videoTexture.colorSpace = THREE.SRGBColorSpace;
+            this.backgroundVideoQuad = new FullScreenQuad(new THREE.MeshBasicMaterial({ map: videoTexture }));
+        }
         this.virtualSceneRenderTarget = new THREE.WebGLRenderTarget(renderer.domElement.width, renderer.domElement.height, { stencilBuffer: options.stencil });
         const virtualSceneMaterial = new THREE.MeshBasicMaterial({
             map: this.virtualSceneRenderTarget.texture,
@@ -10067,6 +10118,9 @@ class Simulator extends Script {
             this.sparkRenderer.defaultView.encodeLinear = false;
         }
         this.renderer.setRenderTarget(null);
+        if (this.backgroundVideoQuad) {
+            this.backgroundVideoQuad.render(this.renderer);
+        }
         this.renderer.render(this.simulatorScene, camera);
         this.renderer.clearDepth();
     }
@@ -10259,6 +10313,7 @@ class AudioListener extends Script {
     }
 }
 
+const DEFAULT_SCHEDULE_AHEAD_TIME = 1.0;
 class AudioPlayer extends Script {
     constructor(options = {}) {
         super();
@@ -10267,6 +10322,7 @@ class AudioPlayer extends Script {
         this.nextStartTime = 0;
         this.volume = 1.0;
         this.category = 'speech';
+        this.scheduleAheadTime = DEFAULT_SCHEDULE_AHEAD_TIME;
         this.options = { sampleRate: 24000, channelCount: 1, ...options };
         if (options.category) {
             this.category = options.category;
@@ -10331,9 +10387,9 @@ class AudioPlayer extends Script {
         this.scheduleAudioBuffers();
     }
     scheduleAudioBuffers() {
-        const SCHEDULE_AHEAD_TIME = 0.2;
         while (this.audioQueue.length > 0 &&
-            this.nextStartTime <= this.audioContext.currentTime + SCHEDULE_AHEAD_TIME) {
+            this.nextStartTime <=
+                this.audioContext.currentTime + this.scheduleAheadTime) {
             const audioBuffer = this.audioQueue.shift();
             const currentTime = this.audioContext.currentTime;
             const startTime = Math.max(this.nextStartTime, currentTime);
@@ -14386,6 +14442,216 @@ class XRTransition extends MeshScript {
     }
 }
 
+/**
+ * A utility class to manage and request browser permissions for
+ * Location, Camera, and Microphone.
+ */
+class PermissionsManager {
+    /**
+     * Requests permission to access the user's geolocation.
+     * Note: This actually attempts to fetch the position to trigger the prompt.
+     */
+    async requestLocationPermission() {
+        if (!('geolocation' in navigator)) {
+            return {
+                granted: false,
+                status: 'error',
+                error: 'Geolocation is not supported by this browser.',
+            };
+        }
+        return new Promise((resolve) => {
+            navigator.geolocation.getCurrentPosition(() => {
+                resolve({ granted: true, status: 'granted' });
+            }, (error) => {
+                let errorMsg = 'Unknown error';
+                switch (error.code) {
+                    case error.PERMISSION_DENIED:
+                        errorMsg = 'User denied the request.';
+                        break;
+                    case error.POSITION_UNAVAILABLE:
+                        errorMsg = 'Location information is unavailable.';
+                        break;
+                    case error.TIMEOUT:
+                        errorMsg = 'The request to get user location timed out.';
+                        break;
+                }
+                resolve({ granted: false, status: 'denied', error: errorMsg });
+            }, { timeout: 10000 } // 10 second timeout
+            );
+        });
+    }
+    /**
+     * Requests permission to access the microphone.
+     * Opens a stream to trigger the prompt, then immediately closes it.
+     */
+    async requestMicrophonePermission() {
+        return this.requestMediaPermission({ audio: true });
+    }
+    /**
+     * Requests permission to access the camera.
+     * Opens a stream to trigger the prompt, then immediately closes it.
+     */
+    async requestCameraPermission() {
+        return this.requestMediaPermission({ video: true });
+    }
+    /**
+     * Requests permission for both camera and microphone simultaneously.
+     */
+    async requestAVPermission() {
+        return this.requestMediaPermission({ video: true, audio: true });
+    }
+    /**
+     * Internal helper to handle getUserMedia requests.
+     * Crucially, this stops the tracks immediately after permission is granted
+     * so the hardware doesn't remain active.
+     */
+    async requestMediaPermission(constraints) {
+        if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+            return {
+                granted: false,
+                status: 'error',
+                error: 'Media Devices API is not supported by this browser.',
+            };
+        }
+        try {
+            const stream = await navigator.mediaDevices.getUserMedia(constraints);
+            // Permission granted. Now stop the stream to release hardware.
+            stream.getTracks().forEach((track) => track.stop());
+            return { granted: true, status: 'granted' };
+        }
+        catch (err) {
+            // Handle common getUserMedia errors
+            const status = 'denied';
+            let errorMessage = 'Permission denied';
+            if (err instanceof Error) {
+                if (err.name === 'NotFoundError' ||
+                    err.name === 'DevicesNotFoundError') {
+                    return {
+                        granted: false,
+                        status: 'error',
+                        error: 'Hardware not found.',
+                    };
+                }
+                errorMessage = err.message || errorMessage;
+            }
+            return { granted: false, status: status, error: errorMessage };
+        }
+    }
+    /**
+     * Requests multiple permissions sequentially.
+     * Returns a single result: granted is true only if ALL requested permissions are granted.
+     */
+    async checkAndRequestPermissions({ geolocation = false, camera = false, microphone = false, }) {
+        const results = [];
+        // 1. Handle Location
+        if (geolocation) {
+            const status = await this.checkPermissionStatus('geolocation');
+            if (status === 'granted') {
+                results.push({ granted: true, status: 'granted' });
+            }
+            else {
+                results.push(await this.requestLocationPermission());
+            }
+        }
+        // 2. Handle Media (Camera & Mic)
+        // We group these because requestAVPermission can ask for both in one prompt
+        if (camera && microphone) {
+            const camStatus = await this.checkPermissionStatus('camera');
+            const micStatus = await this.checkPermissionStatus('microphone');
+            if (camStatus === 'granted' && micStatus === 'granted') {
+                results.push({ granted: true, status: 'granted' });
+            }
+            else if (camStatus === 'granted') {
+                // Only need mic
+                results.push(await this.requestMicrophonePermission());
+            }
+            else if (micStatus === 'granted') {
+                // Only need camera
+                results.push(await this.requestCameraPermission());
+            }
+            else {
+                // Need both
+                results.push(await this.requestAVPermission());
+            }
+        }
+        else if (camera) {
+            const status = await this.checkPermissionStatus('camera');
+            if (status === 'granted') {
+                results.push({ granted: true, status: 'granted' });
+            }
+            else {
+                results.push(await this.requestCameraPermission());
+            }
+        }
+        else if (microphone) {
+            const status = await this.checkPermissionStatus('microphone');
+            if (status === 'granted') {
+                results.push({ granted: true, status: 'granted' });
+            }
+            else {
+                results.push(await this.requestMicrophonePermission());
+            }
+        }
+        // 3. Aggregate results
+        if (results.length === 0) {
+            return { granted: true, status: 'granted' };
+        }
+        const allGranted = results.every((r) => r.granted);
+        const anyDenied = results.find((r) => r.status === 'denied');
+        const anyError = results.find((r) => r.status === 'error');
+        // Aggregate errors
+        const errors = results
+            .filter((r) => r.error)
+            .map((r) => r.error)
+            .join(' | ');
+        let finalStatus = 'granted';
+        if (anyError)
+            finalStatus = 'error';
+        else if (anyDenied)
+            finalStatus = 'denied';
+        return {
+            granted: allGranted,
+            status: finalStatus,
+            error: errors || undefined,
+        };
+    }
+    /**
+     * Checks the current status of a permission without triggering a prompt.
+     * Useful for UI state (e.g., disabling buttons if already denied).
+     * * @param permissionName - 'geolocation', 'camera', or 'microphone'
+     */
+    async checkPermissionStatus(permissionName) {
+        if (!navigator.permissions || !navigator.permissions.query) {
+            return 'unknown';
+        }
+        try {
+            let queryName;
+            // Map friendly names to API PermissionName types
+            // Note: 'camera' and 'microphone' are part of the newer spec,
+            // but strictly Typed TypeScript might expect specific descriptor objects.
+            if (permissionName === 'geolocation') {
+                queryName = 'geolocation';
+            }
+            else if (permissionName === 'camera' ||
+                permissionName === 'microphone') {
+                const descriptor = { name: permissionName };
+                const result = await navigator.permissions.query(descriptor);
+                return result.state;
+            }
+            else {
+                return 'unknown';
+            }
+            const result = await navigator.permissions.query({ name: queryName });
+            return result.state;
+        }
+        catch (error) {
+            // Firefox and Safari have incomplete Permissions API support
+            console.warn(`Error checking permission status for ${permissionName}`, error);
+            return 'unknown';
+        }
+    }
+}
+
 /**
  * Core is the central engine of the XR Blocks framework, acting as a
  * singleton manager for all XR subsystems. Its primary goal is to abstract
@@ -14445,6 +14711,7 @@ class Core {
                 await script.initPhysics(this.physics);
             }
         });
+        this.permissionsManager = new PermissionsManager();
         if (Core.instance) {
             return Core.instance;
         }
@@ -14553,6 +14820,8 @@ class Core {
                 dataFormatPreference: [
                     this.options.depth.useFloat32 ? 'float32' : 'luminance-alpha',
                 ],
+                depthTypeRequest: options.depth.depthTypeRequest,
+                matchDepthView: options.depth.matchDepthView,
             };
             this.depth.init(this.camera, options.depth, this.renderer, this.registry, this.scene);
         }
@@ -14589,11 +14858,11 @@ class Core {
         // Sets up xrButton.
         let shouldAutostartSimulator = this.options.xrButton.alwaysAutostartSimulator;
         if (!shouldAutostartSimulator && options.xrButton.enabled) {
-            this.xrButton = new XRButton(this.webXRSessionManager, options.xrButton?.startText, options.xrButton?.endText, options.xrButton?.invalidText, options.xrButton?.startSimulatorText, options.xrButton?.enableSimulator, this.startSimulator.bind(this));
+            this.xrButton = new XRButton(this.webXRSessionManager, this.permissionsManager, options.xrButton?.startText, options.xrButton?.endText, options.xrButton?.invalidText, options.xrButton?.startSimulatorText, options.xrButton?.showEnterSimulatorButton, this.startSimulator.bind(this), options.permissions);
             document.body.appendChild(this.xrButton.domElement);
         }
         this.webXRSessionManager.addEventListener(WebXRSessionEventType.UNSUPPORTED, () => {
-            if (this.options.xrButton.enableSimulator) {
+            if (this.options.enableSimulator) {
                 this.xrButton?.domElement.remove();
                 shouldAutostartSimulator = true;
             }
@@ -14716,7 +14985,6 @@ class Core {
      * scripts.
      */
     onXRSessionEnded() {
-        this.startSimulator();
         this.scriptsManager.onXRSessionEnded();
     }
     /**