xrblocks 0.1.0 → 0.2.0

package/build/xrblocks.js

@@ -14,9 +14,9 @@
  * limitations under the License.
  *
  * @file xrblocks.js
- * @version v0.1.0
- * @commitid 924884f
- * @builddate 2025-09-29T18:42:43.977Z
+ * @version v0.2.0
+ * @commitid c24c0e2
+ * @builddate 2025-10-20T22:06:39.480Z
  * @description XR Blocks SDK, built from source with the above commit ID.
  * @agent When using with Gemini to create XR apps, use **Gemini Canvas** mode,
  * and follow rules below:
@@ -120,11 +120,13 @@ class Memory {
  */
 class Agent {
     static { this.dependencies = {}; }
-    constructor(ai, tools = [], instruction = '') {
+    constructor(ai, tools = [], instruction = '', callbacks) {
+        this.isSessionActive = false;
         this.ai = ai;
         this.tools = tools;
         this.memory = new Memory();
         this.contextBuilder = new Context(instruction);
+        this.lifecycleCallbacks = callbacks;
     }
     /**
      * Starts the agent's reasoning loop with an initial prompt.
@@ -176,6 +178,17 @@ class Agent {
     findTool(name) {
         return this.tools.find(tool => tool.name === name);
     }
+    /**
+     * Get the current session state.
+     * @returns Object containing session information
+     */
+    getSessionState() {
+        return {
+            isActive: this.isSessionActive,
+            toolCount: this.tools.length,
+            memorySize: this.memory.getShortTerm?.()?.length || 0,
+        };
+    }
 }
 
 /**
@@ -192,15 +205,29 @@ class Tool {
         this.onTriggered = options.onTriggered;
     }
     /**
-     * Executes the tool's action.
+     * Executes the tool's action with standardized error handling.
      * @param args - The arguments for the tool.
-     * @returns The result of the tool's action.
+     * @returns A promise that resolves with a ToolResult containing success/error information.
      */
-    execute(args) {
-        if (this.onTriggered) {
-            return this.onTriggered(args);
+    async execute(args) {
+        try {
+            if (this.onTriggered) {
+                const result = await Promise.resolve(this.onTriggered(args));
+                return {
+                    success: true,
+                    data: result,
+                    metadata: { executedAt: Date.now(), toolName: this.name }
+                };
+            }
+            throw new Error('The execute method must be implemented by a subclass or onTriggered must be provided.');
+        }
+        catch (error) {
+            return {
+                success: false,
+                error: error instanceof Error ? error.message : String(error),
+                metadata: { executedAt: Date.now(), toolName: this.name }
+            };
         }
-        throw new Error('The execute method must be implemented by a subclass or onTriggered must be provided.');
     }
     /**
      * Returns a JSON representation of the tool.
@@ -211,7 +238,7 @@ class Tool {
         if (this.description) {
             result.description = this.description;
         }
-        if (this.parameters && this.parameters.required) {
+        if (this.parameters) {
             result.parameters = this.parameters;
         }
         return result;
@@ -244,7 +271,7 @@ class GenerateSkyboxTool extends Tool {
     /**
      * Executes the tool's action.
      * @param args - The prompt to use to generate the skybox.
-     * @returns A promise that resolves with the result of the skybox generation.
+     * @returns A promise that resolves with a ToolResult containing success/error information.
      */
     async execute(args) {
         try {
@@ -254,30 +281,105 @@ class GenerateSkyboxTool extends Tool {
                 console.log('Applying texture...');
                 this.scene.background = new THREE.TextureLoader().load(image);
                 this.scene.background.mapping = THREE.EquirectangularReflectionMapping;
-                return 'Skybox generated successfully.';
+                return {
+                    success: true,
+                    data: 'Skybox generated successfully.',
+                    metadata: { prompt: args.prompt, timestamp: Date.now() }
+                };
             }
             else {
-                return 'Sorry, I had trouble creating that skybox.';
+                return {
+                    success: false,
+                    error: 'Failed to generate skybox image',
+                    metadata: { prompt: args.prompt, timestamp: Date.now() }
+                };
             }
         }
         catch (e) {
             console.error('error:', e);
-            return 'Sorry, I encountered an error while creating the skybox.';
+            return {
+                success: false,
+                error: e instanceof Error ? e.message :
+                    'Unknown error while creating skybox',
+                metadata: { prompt: args.prompt, timestamp: Date.now() }
+            };
         }
     }
 }
 
+/**
+ * Skybox Agent for generating 360-degree equirectangular backgrounds through conversation.
+ *
+ * @example Basic usage
+ * ```typescript
+ * // 1. Enable audio (required for live sessions)
+ * await xb.core.sound.enableAudio();
+ *
+ * // 2. Create agent
+ * const agent = new xb.SkyboxAgent(xb.core.ai, xb.core.sound, xb.core.scene);
+ *
+ * // 3. Start session
+ * await agent.startLiveSession({
+ *   onopen: () => console.log('Session ready'),
+ *   onmessage: (msg) => handleMessage(msg),
+ *   onclose: () => console.log('Session closed')
+ * });
+ *
+ * // 4. Clean up when done
+ * await agent.stopLiveSession();
+ * xb.core.sound.disableAudio();
+ * ```
+ *
+ * @example With lifecycle callbacks
+ * ```typescript
+ * const agent = new xb.SkyboxAgent(
+ *   xb.core.ai,
+ *   xb.core.sound,
+ *   xb.core.scene,
+ *   {
+ *     onSessionStart: () => updateUI('active'),
+ *     onSessionEnd: () => updateUI('inactive'),
+ *     onError: (error) => showError(error)
+ *   }
+ * );
+ * ```
+ *
+ * @remarks
+ * - Audio must be enabled BEFORE starting live session using `xb.core.sound.enableAudio()`
+ * - Users are responsible for managing audio lifecycle
+ * - Always call `stopLiveSession()` before disabling audio
+ * - Session state can be checked using `getSessionState()` and `getLiveSessionState()`
+ */
 class SkyboxAgent extends Agent {
-    constructor(ai, sound, scene) {
+    constructor(ai, sound, scene, callbacks) {
         super(ai, [new GenerateSkyboxTool(ai, scene)], `You are a friendly and helpful skybox designer. The response should be short. Your only capability
          is to generate a 360-degree equirectangular skybox image based on
          a user's description. You will generate a default skybox if the user
          does not provide any description. You will use the tool 'generateSkybox'
-         with the summarized description as the 'prompt' argument to create the skybox.`);
+         with the summarized description as the 'prompt' argument to create the skybox.`, callbacks);
         this.sound = sound;
+        this.sessionState = {
+            isActive: false,
+            messageCount: 0,
+            toolCallCount: 0
+        };
     }
+    /**
+     * Starts a live AI session for real-time conversation.
+     *
+     * @param callbacks - Optional callbacks for session events. Can also be set using ai.setLiveCallbacks()
+     * @throws If AI model is not initialized or live session is not available
+     *
+     * @remarks
+     * Audio must be enabled separately using `xb.core.sound.enableAudio()` before starting the session.
+     * This gives users control over when microphone permissions are requested.
+     */
     async startLiveSession(callbacks) {
-        this.ai.setLiveCallbacks(callbacks);
+        // Wrap callbacks to track session state
+        const wrappedCallbacks = this.wrapCallbacks(callbacks);
+        if (callbacks) {
+            this.ai.setLiveCallbacks(wrappedCallbacks);
+        }
         const functionDeclarations = this.tools.map(tool => tool.toJSON());
         const systemInstruction = {
             parts: [{ text: this.contextBuilder.instruction }]
@@ -286,16 +388,118 @@ class SkyboxAgent extends Agent {
             tools: functionDeclarations,
             systemInstruction: systemInstruction,
         });
-        this.sound.enableAudio();
+        this.sessionState.isActive = true;
+        this.sessionState.startTime = Date.now();
+        this.isSessionActive = true;
+        await this.lifecycleCallbacks?.onSessionStart?.();
     }
+    /**
+     * Stops the live AI session.
+     *
+     * @remarks
+     * Audio must be disabled separately using `xb.core.sound.disableAudio()` after stopping the session.
+     */
     async stopLiveSession() {
         await this.ai.stopLiveSession();
-        this.sound.disableAudio();
+        this.sessionState.isActive = false;
+        this.sessionState.endTime = Date.now();
+        this.isSessionActive = false;
+        await this.lifecycleCallbacks?.onSessionEnd?.();
     }
+    /**
+     * Wraps user callbacks to track session state and trigger lifecycle events.
+     * @param callbacks - The callbacks to wrap.
+     * @returns The wrapped callbacks.
+     */
+    wrapCallbacks(callbacks) {
+        return {
+            onopen: () => {
+                callbacks?.onopen?.();
+            },
+            onmessage: (message) => {
+                this.sessionState.messageCount++;
+                callbacks?.onmessage?.(message);
+            },
+            onerror: (error) => {
+                this.sessionState.lastError = error.message;
+                this.lifecycleCallbacks?.onError?.(new Error(error.message));
+                callbacks?.onerror?.(error);
+            },
+            onclose: (event) => {
+                this.sessionState.isActive = false;
+                this.sessionState.endTime = Date.now();
+                this.isSessionActive = false;
+                callbacks?.onclose?.(event);
+            }
+        };
+    }
+    /**
+     * Sends tool execution results back to the AI.
+     *
+     * @param response - The tool response containing function results
+     */
     async sendToolResponse(response) {
+        if (!this.validateToolResponse(response)) {
+            console.error('Invalid tool response format:', response);
+            return;
+        }
+        // Handle both single response and array of responses
+        const responses = Array.isArray(response.functionResponses) ?
+            response.functionResponses :
+            [response.functionResponses];
+        this.sessionState.toolCallCount += responses.length;
         console.log('Sending tool response:', response);
         this.ai.sendToolResponse(response);
     }
+    /**
+     * Validates that a tool response has the correct format.
+     * @param response - The tool response to validate.
+     * @returns True if the response is valid, false otherwise.
+     */
+    validateToolResponse(response) {
+        if (!response.functionResponses) {
+            return false;
+        }
+        // Handle both single response and array of responses
+        const responses = Array.isArray(response.functionResponses) ?
+            response.functionResponses :
+            [response.functionResponses];
+        return responses.every(fr => fr.id && fr.name && fr.response !== undefined);
+    }
+    /**
+     * Helper to create a properly formatted tool response from a ToolResult.
+     *
+     * @param id - The function call ID
+     * @param name - The function name
+     * @param result - The ToolResult from tool execution
+     * @returns A properly formatted FunctionResponse
+     */
+    static createToolResponse(id, name, result) {
+        return {
+            id,
+            name,
+            response: result.success ? { result: result.data } : { error: result.error }
+        };
+    }
+    /**
+     * Gets the current live session state.
+     *
+     * @returns Read-only session state information
+     */
+    getLiveSessionState() {
+        return { ...this.sessionState };
+    }
+    /**
+     * Gets the duration of the session in milliseconds.
+     *
+     * @returns Duration in ms, or null if session hasn't started
+     */
+    getSessionDuration() {
+        if (!this.sessionState.startTime)
+            return null;
+        const endTime = this.sessionState.endTime || Date.now();
+        return endTime - this.sessionState.startTime;
+    }
 }
 
 /**
@@ -325,7 +529,7 @@ class GetWeatherTool extends Tool {
     /**
      * Executes the tool's action.
      * @param args - The arguments for the tool.
-     * @returns A promise that resolves with the weather information.
+     * @returns A promise that resolves with a ToolResult containing weather information.
      */
     async execute(args) {
         if (!args.latitude || !args.longitude) {
@@ -338,20 +542,33 @@ class GetWeatherTool extends Tool {
             const data = await response.json();
             if (response.ok) {
                 return {
-                    temperature: data.current.temperature_2m,
-                    weathercode: data.current.weather_code,
+                    success: true,
+                    data: {
+                        temperature: data.current.temperature_2m,
+                        weathercode: data.current.weather_code,
+                    },
+                    metadata: {
+                        latitude: args.latitude,
+                        longitude: args.longitude,
+                        timestamp: Date.now()
+                    }
                 };
             }
             else {
                 return {
-                    error: 'Could not retrieve weather for the specified location.'
+                    success: false,
+                    error: 'Could not retrieve weather for the specified location.',
+                    metadata: { latitude: args.latitude, longitude: args.longitude }
                 };
             }
         }
         catch (error) {
             console.error('Error fetching weather:', error);
             return {
-                error: 'There was an error fetching the weather.'
+                success: false,
+                error: error instanceof Error ? error.message :
+                    'There was an error fetching the weather.',
+                metadata: { latitude: args.latitude, longitude: args.longitude }
             };
         }
     }
@@ -1061,7 +1278,7 @@ class Gemini extends BaseAIModel {
     }
     sendToolResponse(response) {
         if (this.liveSession) {
-            console.log('Sending tool response from gemini:', response);
+            console.debug('Sending tool response to gemini:', response);
             this.liveSession.sendToolResponse(response);
         }
     }
@@ -1652,6 +1869,7 @@ void main() {
 
 uniform vec3 uColor;
 uniform sampler2D uDepthTexture;
+uniform sampler2DArray uDepthTextureArray;
 uniform vec3 uLightDirection;
 uniform vec2 uResolution;
 uniform float uRawValueToMeters;
@@ -1667,6 +1885,7 @@ uniform float uMaxDepth;
 uniform float uDebug;
 uniform float uOpacity;
 uniform bool uUsingFloatDepth;
+uniform bool uIsTextureArray;
 
 float saturate(in float x) {
   return clamp(x, 0.0, 1.0);
@@ -1701,6 +1920,10 @@ float DepthGetMeters(in sampler2D depth_texture, in vec2 depth_uv) {
   return dot(packedDepthAndVisibility, vec2(255.0, 256.0 * 255.0)) * uRawValueToMeters;
 }
 
+float DepthArrayGetMeters(in sampler2DArray depth_texture, in vec2 depth_uv) {
+  return uRawValueToMeters * texture(uDepthTextureArray, vec3 (depth_uv.x, depth_uv.y, 0)).r;
+}
+
 vec3 DepthGetColorVisualization(in float x) {
   return step(kInvalidDepthThreshold, x) * TurboColormap(x);
 }
@@ -1736,7 +1959,7 @@ void main() {
   vec2 depth_uv = uv;
   depth_uv.y = 1.0 - depth_uv.y;
 
-  float depth = DepthGetMeters(uDepthTexture, depth_uv) * 8.0;
+  float depth = (uIsTextureArray ? DepthArrayGetMeters(uDepthTextureArray, depth_uv) : DepthGetMeters(uDepthTexture, depth_uv)) * 8.0;
   float normalized_depth =
     saturate((depth - uMinDepth) / (uMaxDepth - uMinDepth));
   gl_FragColor = uOpacity * vec4(TurboColormap(normalized_depth), 1.0);
@@ -1758,6 +1981,8 @@ class DepthMesh extends MeshScript {
         if (options.useDepthTexture || options.showDebugTexture) {
             uniforms = {
                 uDepthTexture: { value: null },
+                uDepthTextureArray: { value: null },
+                uIsTextureArray: { value: 0.0 },
                 uColor: { value: new THREE.Color(0xaaaaaa) },
                 uResolution: { value: new THREE.Vector2(width, height) },
                 uRawValueToMeters: { value: 1.0 },
@@ -1844,18 +2069,97 @@ class DepthMesh extends MeshScript {
         this.geometry.attributes.position.needsUpdate = true;
         const depthTextureLeft = this.depthTextures?.get(0);
         if (depthTextureLeft && this.depthTextureMaterialUniforms) {
-            this.depthTextureMaterialUniforms.uDepthTexture.value = depthTextureLeft;
+            const isTextureArray = depthTextureLeft instanceof THREE.ExternalTexture;
+            this.depthTextureMaterialUniforms.uIsTextureArray.value =
+                isTextureArray ? 1.0 : 0;
+            if (isTextureArray)
+                this.depthTextureMaterialUniforms.uDepthTextureArray.value =
+                    depthTextureLeft;
+            else
+                this.depthTextureMaterialUniforms.uDepthTexture.value =
+                    depthTextureLeft;
             this.depthTextureMaterialUniforms.uMinDepth.value = this.minDepth;
             this.depthTextureMaterialUniforms.uMaxDepth.value = this.maxDepth;
             this.depthTextureMaterialUniforms.uRawValueToMeters.value =
-                this.depthTextures.depthData[0].rawValueToMeters;
-            this.material.needsUpdate = true;
+                this.depthTextures.depthData.length ?
+                    this.depthTextures.depthData[0].rawValueToMeters :
+                    1.0;
         }
         if (this.options.updateVertexNormals) {
             this.geometry.computeVertexNormals();
         }
         this.updateColliderIfNeeded();
     }
+    updateGPUDepth(depthData) {
+        this.updateDepth(this.convertGPUToGPU(depthData));
+    }
+    convertGPUToGPU(depthData) {
+        if (!this.depthTarget) {
+            this.depthTarget =
+                new THREE.WebGLRenderTarget(depthData.width, depthData.height, {
+                    format: THREE.RedFormat,
+                    type: THREE.FloatType,
+                    internalFormat: 'R32F',
+                    minFilter: THREE.NearestFilter,
+                    magFilter: THREE.NearestFilter,
+                    depthBuffer: false
+                });
+            this.depthTexture = new THREE.ExternalTexture(depthData.texture);
+            const textureProperties = this.renderer.properties.get(this.depthTexture);
+            textureProperties.__webglTexture = depthData.texture;
+            this.gpuPixels = new Float32Array(depthData.width * depthData.height);
+            const depthShader = new THREE.ShaderMaterial({
+                vertexShader: `
+                varying vec2 vUv;
+                void main() {
+                    vUv = uv;
+                    vUv.y = 1.0-vUv.y;
+                    gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
+                }
+            `,
+                fragmentShader: `
+                precision highp float;
+                precision highp sampler2DArray;
+
+                uniform sampler2DArray uTexture;
+                uniform float uCameraNear;
+                varying vec2 vUv;
+
+                void main() {
+                  float z = texture(uTexture, vec3(vUv, 0)).r;
+                  z = uCameraNear / (1.0 - z);
+                  z = clamp(z, 0.0, 20.0);
+                  gl_FragColor = vec4(z, 0, 0, 1.0);
+                }
+            `,
+                uniforms: {
+                    uTexture: { value: this.depthTexture },
+                    uCameraNear: { value: depthData.depthNear }
+                },
+                blending: THREE.NoBlending,
+                depthTest: false,
+                depthWrite: false,
+                side: THREE.DoubleSide
+            });
+            const depthMesh = new THREE.Mesh(new THREE.PlaneGeometry(2, 2), depthShader);
+            this.depthScene = new THREE.Scene();
+            this.depthScene.add(depthMesh);
+            this.depthCamera = new THREE.OrthographicCamera(-1, 1, 1, -1, 0, 1);
+        }
+        const originalRenderTarget = this.renderer.getRenderTarget();
+        this.renderer.xr.enabled = false;
+        this.renderer.setRenderTarget(this.depthTarget);
+        this.renderer.render(this.depthScene, this.depthCamera);
+        this.renderer.readRenderTargetPixels(this.depthTarget, 0, 0, depthData.width, depthData.height, this.gpuPixels, 0);
+        this.renderer.xr.enabled = true;
+        this.renderer.setRenderTarget(originalRenderTarget);
+        return {
+            width: depthData.width,
+            height: depthData.height,
+            data: this.gpuPixels.buffer,
+            rawValueToMeters: depthData.rawValueToMeters
+        };
+    }
     /**
      * Method to manually update the full resolution geometry.
      * Only needed if options.updateFullResolutionGeometry is false.
@@ -2127,33 +2431,34 @@ class DepthTextures {
         this.options = options;
         this.uint16Arrays = [];
         this.uint8Arrays = [];
-        this.textures = [];
+        this.dataTextures = [];
+        this.nativeTextures = [];
         this.depthData = [];
     }
-    createDepthTextures(depthData, view_id) {
-        if (this.textures[view_id]) {
-            this.textures[view_id].dispose();
+    createDataDepthTextures(depthData, view_id) {
+        if (this.dataTextures[view_id]) {
+            this.dataTextures[view_id].dispose();
         }
         if (this.options.useFloat32) {
             const typedArray = new Uint16Array(depthData.width * depthData.height);
             const format = THREE.RedFormat;
             const type = THREE.HalfFloatType;
             this.uint16Arrays[view_id] = typedArray;
-            this.textures[view_id] = new THREE.DataTexture(typedArray, depthData.width, depthData.height, format, type);
+            this.dataTextures[view_id] = new THREE.DataTexture(typedArray, depthData.width, depthData.height, format, type);
         }
         else {
             const typedArray = new Uint8Array(depthData.width * depthData.height * 2);
             const format = THREE.RGFormat;
             const type = THREE.UnsignedByteType;
             this.uint8Arrays[view_id] = typedArray;
-            this.textures[view_id] = new THREE.DataTexture(typedArray, depthData.width, depthData.height, format, type);
+            this.dataTextures[view_id] = new THREE.DataTexture(typedArray, depthData.width, depthData.height, format, type);
         }
     }
-    update(depthData, view_id) {
-        if (this.textures.length < view_id + 1 ||
-            this.textures[view_id].image.width !== depthData.width ||
-            this.textures[view_id].image.height !== depthData.height) {
-            this.createDepthTextures(depthData, view_id);
+    updateData(depthData, view_id) {
+        if (this.dataTextures.length < view_id + 1 ||
+            this.dataTextures[view_id].image.width !== depthData.width ||
+            this.dataTextures[view_id].image.height !== depthData.height) {
+            this.createDataDepthTextures(depthData, view_id);
         }
         if (this.options.useFloat32) {
             const float32Data = new Float32Array(depthData.data);
@@ -2166,11 +2471,26 @@ class DepthTextures {
         else {
             this.uint8Arrays[view_id].set(new Uint8Array(depthData.data));
         }
-        this.textures[view_id].needsUpdate = true;
+        this.dataTextures[view_id].needsUpdate = true;
         this.depthData[view_id] = depthData;
     }
+    updateNativeTexture(depthData, renderer, view_id) {
+        if (this.dataTextures.length < view_id + 1) {
+            this.nativeTextures[view_id] = new THREE.ExternalTexture(depthData.texture);
+        }
+        else {
+            this.nativeTextures[view_id].sourceTexture = depthData.texture;
+        }
+        // fixed in newer revision of three
+        const textureProperties = renderer.properties.get(this.nativeTextures[view_id]);
+        textureProperties.__webglTexture = depthData.texture;
+        textureProperties.__version = 1;
+    }
     get(view_id) {
-        return this.textures[view_id];
+        if (this.dataTextures.length > 0) {
+            return this.dataTextures[view_id];
+        }
+        return this.nativeTextures[view_id];
     }
 }
 
@@ -2370,10 +2690,15 @@ class OcclusionMapMeshMaterial extends THREE.MeshBasicMaterial {
         super();
         this.uniforms = {
             uDepthTexture: { value: null },
+            uDepthTextureArray: { value: null },
+            uViewId: { value: 0.0 },
+            uIsTextureArray: { value: 0.0 },
             uRawValueToMeters: { value: 8.0 / 65536.0 },
             cameraFar: { value: camera.far },
             cameraNear: { value: camera.near },
             uFloatDepth: { value: useFloatDepth },
+            // Used for interpreting Quest 3 depth.
+            uDepthNear: { value: 0 }
         };
         this.onBeforeCompile = (shader) => {
             Object.assign(shader.uniforms, this.uniforms);
@@ -2395,10 +2720,13 @@ class OcclusionMapMeshMaterial extends THREE.MeshBasicMaterial {
                 shader.fragmentShader
                     .replace('uniform vec3 diffuse;', [
                     'uniform vec3 diffuse;', 'uniform sampler2D uDepthTexture;',
+                    'uniform sampler2DArray uDepthTextureArray;',
                     'uniform float uRawValueToMeters;',
                     'uniform float cameraNear;', 'uniform float cameraFar;',
-                    'uniform bool uFloatDepth;', 'varying vec2 vTexCoord;',
-                    'varying float vVirtualDepth;'
+                    'uniform bool uFloatDepth;',
+                    'uniform bool uIsTextureArray;',
+                    'uniform float uDepthNear;', 'uniform int uViewId;',
+                    'varying vec2 vTexCoord;', 'varying float vVirtualDepth;'
                 ].join('\n'))
                     .replace('#include <clipping_planes_pars_fragment>', [
                     '#include <clipping_planes_pars_fragment>', `
@@ -2412,13 +2740,17 @@ class OcclusionMapMeshMaterial extends THREE.MeshBasicMaterial {
     }
     return dot(packedDepthAndVisibility, vec2(255.0, 256.0 * 255.0)) * uRawValueToMeters;
   }
+  float DepthArrayGetMeters(in sampler2DArray depth_texture, in vec2 depth_uv) {
+    float textureValue = texture(depth_texture, vec3(depth_uv.x, depth_uv.y, uViewId)).r;
+    return uRawValueToMeters * uDepthNear / (1.0 - textureValue);
+  }
 `
                 ].join('\n'))
                     .replace('#include <dithering_fragment>', [
                     '#include <dithering_fragment>',
                     'vec4 texCoord = vec4(vTexCoord, 0, 1);',
-                    'vec2 uv = vec2(texCoord.x, 1.0 - texCoord.y);',
-                    'highp float real_depth = DepthGetMeters(uDepthTexture, uv);',
+                    'vec2 uv = vec2(texCoord.x, uIsTextureArray?texCoord.y:(1.0 - texCoord.y));',
+                    'highp float real_depth = uIsTextureArray ? DepthArrayGetMeters(uDepthTextureArray, uv) : DepthGetMeters(uDepthTexture, uv);',
                     'gl_FragColor = vec4(step(vVirtualDepth, real_depth), 1.0, 0.0, 1.0);'
                 ].join('\n'));
         };
@@ -2450,10 +2782,14 @@ class OcclusionPass extends Pass {
         this.renderToScreen = renderToScreen;
         this.occludableItemsLayer = occludableItemsLayer;
         this.depthTextures = [];
+        this.depthNear = [];
         this.occlusionMeshMaterial =
             new OcclusionMapMeshMaterial(camera, useFloatDepth);
         this.occlusionMapUniforms = {
             uDepthTexture: { value: null },
+            uDepthTextureArray: { value: null },
+            uViewId: { value: 0.0 },
+            uIsTextureArray: { value: 0.0 },
             uUvTransform: { value: new THREE.Matrix4() },
             uRawValueToMeters: { value: 8.0 / 65536.0 },
             uAlpha: { value: 0.75 },
@@ -2510,14 +2846,12 @@ class OcclusionPass extends Pass {
         });
         return new FullScreenQuad(kawase1Material);
     }
-    setDepthTexture(depthTexture, rawValueToMeters, view_id) {
-        if (view_id > 1) {
-            return;
-        }
-        this.depthTextures[view_id] = depthTexture;
+    setDepthTexture(depthTexture, rawValueToMeters, viewId, depthNear) {
+        this.depthTextures[viewId] = depthTexture;
         this.occlusionMapUniforms.uRawValueToMeters.value = rawValueToMeters;
         this.occlusionMeshMaterial.uniforms.uRawValueToMeters.value =
             rawValueToMeters;
+        this.depthNear[viewId] = depthNear;
         depthTexture.needsUpdate = true;
     }
     /**
@@ -2525,16 +2859,16 @@ class OcclusionPass extends Pass {
      * @param renderer - The three.js renderer.
      * @param writeBuffer - The buffer to write the final result.
      * @param readBuffer - The buffer for the current of virtual depth.
-     * @param view_id - The view to render.
+     * @param viewId - The view to render.
      */
-    render(renderer, writeBuffer, readBuffer, view_id = 0) {
+    render(renderer, writeBuffer, readBuffer, viewId = 0) {
         const originalRenderTarget = renderer.getRenderTarget();
         const dimensions = new THREE.Vector2();
         if (readBuffer == null) {
-            this.renderOcclusionMapFromScene(renderer, dimensions, view_id);
+            this.renderOcclusionMapFromScene(renderer, dimensions, viewId);
         }
         else {
-            this.renderOcclusionMapFromReadBuffer(renderer, readBuffer, dimensions, view_id);
+            this.renderOcclusionMapFromReadBuffer(renderer, readBuffer, dimensions, viewId);
         }
         // Blur the occlusion map
         this.blurOcclusionMap(renderer, dimensions);
@@ -2542,16 +2876,27 @@ class OcclusionPass extends Pass {
         this.applyOcclusionMapToRenderedImage(renderer, readBuffer, writeBuffer);
         renderer.setRenderTarget(originalRenderTarget);
     }
-    renderOcclusionMapFromScene(renderer, dimensions, view_id) {
+    renderOcclusionMapFromScene(renderer, dimensions, viewId) {
         // Compute our own read buffer.
-        this.occlusionMeshMaterial.uniforms.uDepthTexture.value =
-            this.depthTextures[view_id];
+        const texture = this.depthTextures[viewId];
+        const isTextureArray = texture instanceof THREE.ExternalTexture;
+        this.occlusionMeshMaterial.uniforms.uIsTextureArray.value =
+            isTextureArray ? 1.0 : 0;
+        this.occlusionMeshMaterial.uniforms.uViewId.value = viewId;
+        if (isTextureArray) {
+            this.occlusionMeshMaterial.uniforms.uDepthTextureArray.value = texture;
+            this.occlusionMeshMaterial.uniforms.uDepthNear.value =
+                this.depthNear[viewId];
+        }
+        else {
+            this.occlusionMeshMaterial.uniforms.uDepthTexture.value = texture;
+        }
         this.scene.overrideMaterial = this.occlusionMeshMaterial;
         renderer.getDrawingBufferSize(dimensions);
         this.occlusionMapTexture.setSize(dimensions.x, dimensions.y);
         const renderTarget = this.occlusionMapTexture;
         renderer.setRenderTarget(renderTarget);
-        const camera = renderer.xr.getCamera().cameras[view_id] || this.camera;
+        const camera = renderer.xr.getCamera().cameras[viewId] || this.camera;
         const originalCameraLayers = Array.from(Array(32).keys())
             .filter(element => camera.layers.isEnabled(element));
         camera.layers.set(this.occludableItemsLayer);
@@ -2562,12 +2907,24 @@ class OcclusionPass extends Pass {
         });
         this.scene.overrideMaterial = null;
     }
-    renderOcclusionMapFromReadBuffer(renderer, readBuffer, dimensions, view_id) {
+    renderOcclusionMapFromReadBuffer(renderer, readBuffer, dimensions, viewId) {
         // Convert the readBuffer into an occlusion map.
         // Render depth into texture
         this.occlusionMapUniforms.tDiffuse.value = readBuffer.texture;
         this.occlusionMapUniforms.tDepth.value = readBuffer.depthTexture;
-        this.occlusionMapUniforms.uDepthTexture.value = this.depthTextures[view_id];
+        const texture = this.depthTextures[viewId];
+        const isTextureArray = texture instanceof THREE.ExternalTexture;
+        this.occlusionMeshMaterial.uniforms.uIsTextureArray.value =
+            isTextureArray ? 1.0 : 0;
+        this.occlusionMeshMaterial.uniforms.uViewId.value = viewId;
+        if (isTextureArray) {
+            this.occlusionMeshMaterial.uniforms.uDepthTextureArray.value = texture;
+            this.occlusionMeshMaterial.uniforms.uDepthNear.value =
+                this.depthNear[viewId];
+        }
+        else {
+            this.occlusionMeshMaterial.uniforms.uDepthTexture.value = texture;
+        }
         // First render the occlusion map to an intermediate buffer.
         renderer.getDrawingBufferSize(dimensions);
         this.occlusionMapTexture.setSize(dimensions.x, dimensions.y);
@@ -2630,7 +2987,8 @@ class Depth {
     constructor() {
         this.projectionMatrixInverse = new THREE.Matrix4();
         this.view = [];
-        this.depthData = [];
+        this.cpuDepthData = [];
+        this.gpuDepthData = [];
         this.depthArray = [];
         this.options = new DepthOptions();
         this.width = DEFAULT_DEPTH_WIDTH;
@@ -2722,8 +3080,8 @@ class Depth {
         vertexPosition.multiplyScalar(-depth / vertexPosition.z);
         return vertexPosition;
     }
-    updateDepthData(depthData, view_id = 0) {
-        this.depthData[view_id] = depthData;
+    updateCPUDepthData(depthData, view_id = 0) {
+        this.cpuDepthData[view_id] = depthData;
         // Workaround for b/382679381.
         this.rawValueToMeters = depthData.rawValueToMeters;
         if (this.options.useFloat32) {
@@ -2744,12 +3102,52 @@ class Depth {
         }
         // Updates Depth Texture.
         if (this.options.depthTexture.enabled && this.depthTextures) {
-            this.depthTextures.update(depthData, view_id);
+            this.depthTextures.updateData(depthData, view_id);
         }
         if (this.options.depthMesh.enabled && this.depthMesh && view_id == 0) {
             this.depthMesh.updateDepth(depthData);
         }
     }
+    updateGPUDepthData(depthData, view_id = 0) {
+        this.gpuDepthData[view_id] = depthData;
+        // Workaround for b/382679381.
+        this.rawValueToMeters = depthData.rawValueToMeters;
+        if (this.options.useFloat32) {
+            this.rawValueToMeters = 1.0;
+        }
+        // For now, assume that we need cpu depth only if depth mesh is enabled.
+        // In the future, add a separate option.
+        const needCpuDepth = this.options.depthMesh.enabled;
+        const cpuDepth = needCpuDepth && this.depthMesh ?
+            this.depthMesh.convertGPUToGPU(depthData) :
+            null;
+        if (cpuDepth) {
+            if (this.depthArray[view_id] == null) {
+                this.depthArray[view_id] = this.options.useFloat32 ?
+                    new Float32Array(cpuDepth.data) :
+                    new Uint16Array(cpuDepth.data);
+                this.width = cpuDepth.width;
+                this.height = cpuDepth.height;
+            }
+            else {
+                // Copies the data from an ArrayBuffer to the existing TypedArray.
+                this.depthArray[view_id].set(this.options.useFloat32 ? new Float32Array(cpuDepth.data) :
+                    new Uint16Array(cpuDepth.data));
+            }
+        }
+        // Updates Depth Texture.
+        if (this.options.depthTexture.enabled && this.depthTextures) {
+            this.depthTextures.updateNativeTexture(depthData, this.renderer, view_id);
+        }
+        if (this.options.depthMesh.enabled && this.depthMesh && view_id == 0) {
+            if (cpuDepth) {
+                this.depthMesh.updateDepth(cpuDepth);
+            }
+            else {
+                this.depthMesh.updateGPUDepth(depthData);
+            }
+        }
+    }
     getTexture(view_id) {
         if (!this.options.depthTexture.enabled)
             return undefined;
@@ -2775,6 +3173,7 @@ class Depth {
         if (!frame)
             return;
         const session = frame.session;
+        const binding = this.renderer.xr.getBinding();
         // Enable or disable depth based on the number of clients.
         const pausingDepthSupported = session.depthActive !== undefined;
         if (pausingDepthSupported && this.depthClientsInitialized) {
@@ -2803,11 +3202,20 @@ class Depth {
                 for (let view_id = 0; view_id < pose.views.length; ++view_id) {
                     const view = pose.views[view_id];
                     this.view[view_id] = view;
-                    const depthData = frame.getDepthInformation(view);
-                    if (!depthData) {
-                        return;
+                    if (session.depthUsage === 'gpu-optimized') {
+                        const depthData = binding.getDepthInformation(view);
+                        if (!depthData) {
+                            return;
+                        }
+                        this.updateGPUDepthData(depthData, view_id);
+                    }
+                    else {
+                        const depthData = frame.getDepthInformation(view);
+                        if (!depthData) {
+                            return;
+                        }
+                        this.updateCPUDepthData(depthData, view_id);
                     }
-                    this.updateDepthData(depthData, view_id);
                 }
             }
             else {
@@ -2818,11 +3226,13 @@ class Depth {
     renderOcclusionPass() {
         const leftDepthTexture = this.getTexture(0);
         if (leftDepthTexture) {
-            this.occlusionPass.setDepthTexture(leftDepthTexture, this.rawValueToMeters, 0);
+            this.occlusionPass.setDepthTexture(leftDepthTexture, this.rawValueToMeters, 0, this.gpuDepthData[0]
+                ?.depthNear);
         }
         const rightDepthTexture = this.getTexture(1);
         if (rightDepthTexture) {
-            this.occlusionPass.setDepthTexture(rightDepthTexture, this.rawValueToMeters, 1);
+            this.occlusionPass.setDepthTexture(rightDepthTexture, this.rawValueToMeters, 1, this.gpuDepthData[1]
+                ?.depthNear);
         }
         const xrIsPresenting = this.renderer.xr.isPresenting;
         this.renderer.xr.isPresenting = false;
@@ -2833,7 +3243,7 @@ class Depth {
         }
     }
     debugLog() {
-        const arrayBuffer = this.depthData[0].data;
+        const arrayBuffer = this.cpuDepthData[0].data;
         const uint8Array = new Uint8Array(arrayBuffer);
         // Convert Uint8Array to a string where each character represents a byte
         const binaryString = Array.from(uint8Array, byte => String.fromCharCode(byte)).join('');
@@ -3450,6 +3860,9 @@ class Registry {
     }
 }
 
+// Use a small canvas since a full size canvas can consume a lot of memory and
+// cause toDataUrl to be slow.
+const DEFAULT_CANVAS_WIDTH = 640;
 function flipBufferVertically(buffer, width, height) {
     const bytesPerRow = width * 4;
     const tempRow = new Uint8Array(bytesPerRow);
@@ -3475,8 +3888,9 @@ class ScreenshotSynthesizer {
         this.pendingScreenshotRequests = [];
         this.virtualBuffer = new Uint8Array();
         this.virtualRealBuffer = new Uint8Array();
+        this.renderTargetWidth = DEFAULT_CANVAS_WIDTH;
     }
-    async onAfterRender(renderer, deviceCamera) {
+    async onAfterRender(renderer, renderSceneFn, deviceCamera) {
         if (this.pendingScreenshotRequests.length == 0) {
             return;
         }
@@ -3486,13 +3900,14 @@ class ScreenshotSynthesizer {
         }
         const haveVirtualOnlyRequests = this.pendingScreenshotRequests.every((request) => !request.overlayOnCamera);
         if (haveVirtualOnlyRequests) {
-            this.createVirtualImageDataURL(renderer).then((virtualImageDataUrl) => {
+            this.createVirtualImageDataURL(renderer, renderSceneFn)
+                .then((virtualImageDataUrl) => {
                 this.resolveVirtualOnlyRequests(virtualImageDataUrl);
             });
         }
         const haveVirtualAndRealReqeusts = this.pendingScreenshotRequests.some((request) => request.overlayOnCamera);
         if (haveVirtualAndRealReqeusts && deviceCamera) {
-            this.createVirtualRealImageDataURL(renderer, deviceCamera)
+            this.createVirtualRealImageDataURL(renderer, renderSceneFn, deviceCamera)
                 .then((virtualRealImageDataUrl) => {
                 if (virtualRealImageDataUrl) {
                     this.resolveVirtualRealRequests(virtualRealImageDataUrl);
@@ -3503,25 +3918,42 @@ class ScreenshotSynthesizer {
             throw new Error('No device camera provided');
         }
     }
-    async createVirtualImageDataURL(renderer) {
-        const renderTarget = renderer.getRenderTarget();
-        if (this.virtualBuffer.length !=
-            renderTarget.width * renderTarget.height * 4) {
-            this.virtualBuffer =
-                new Uint8Array(renderTarget.width * renderTarget.height * 4);
+    async createVirtualImageDataURL(renderer, renderSceneFn) {
+        const mainRenderTarget = renderer.getRenderTarget();
+        const isRenderingStereo = renderer.xr.isPresenting && renderer.xr.getCamera().cameras.length == 2;
+        const mainRenderTargetSingleViewWidth = isRenderingStereo ? mainRenderTarget.width / 2 : mainRenderTarget.width;
+        const scaledHeight = Math.round(mainRenderTarget.height *
+            (this.renderTargetWidth / mainRenderTargetSingleViewWidth));
+        if (!this.virtualRenderTarget ||
+            this.virtualRenderTarget.width != this.renderTargetWidth) {
+            this.virtualRenderTarget?.dispose();
+            this.virtualRenderTarget = new THREE.WebGLRenderTarget(this.renderTargetWidth, scaledHeight, { colorSpace: THREE.SRGBColorSpace });
+        }
+        const xrIsPresenting = renderer.xr.isPresenting;
+        renderer.xr.isPresenting = false;
+        const virtualRenderTarget = this.virtualRenderTarget;
+        renderer.setRenderTarget(virtualRenderTarget);
+        renderer.clearColor();
+        renderer.clearDepth();
+        renderSceneFn();
+        renderer.setRenderTarget(mainRenderTarget);
+        renderer.xr.isPresenting = xrIsPresenting;
+        const expectedBufferLength = virtualRenderTarget.width * virtualRenderTarget.height * 4;
+        if (this.virtualBuffer.length != expectedBufferLength) {
+            this.virtualBuffer = new Uint8Array(expectedBufferLength);
         }
         const buffer = this.virtualBuffer;
-        await renderer.readRenderTargetPixelsAsync(renderTarget, 0, 0, renderTarget.width, renderTarget.height, buffer);
-        flipBufferVertically(buffer, renderTarget.width, renderTarget.height);
+        await renderer.readRenderTargetPixelsAsync(virtualRenderTarget, 0, 0, virtualRenderTarget.width, virtualRenderTarget.height, buffer);
+        flipBufferVertically(buffer, virtualRenderTarget.width, virtualRenderTarget.height);
         const canvas = this.virtualCanvas ||
             (this.virtualCanvas = document.createElement('canvas'));
-        canvas.width = renderTarget.width;
-        canvas.height = renderTarget.height;
+        canvas.width = virtualRenderTarget.width;
+        canvas.height = virtualRenderTarget.height;
         const context = canvas.getContext('2d');
         if (!context) {
             throw new Error('Failed to get 2D context');
         }
-        const imageData = new ImageData(new Uint8ClampedArray(buffer), renderTarget.width, renderTarget.height);
+        const imageData = new ImageData(new Uint8ClampedArray(buffer), virtualRenderTarget.width, virtualRenderTarget.height);
         context.putImageData(imageData, 0, 0);
         return canvas.toDataURL();
     }
@@ -3538,24 +3970,31 @@ class ScreenshotSynthesizer {
         }
         this.pendingScreenshotRequests.length = remainingRequests;
     }
-    async createVirtualRealImageDataURL(renderer, deviceCamera) {
+    async createVirtualRealImageDataURL(renderer, renderSceneFn, deviceCamera) {
         if (!deviceCamera.loaded) {
-            console.log('Waiting for device camera to be loaded');
+            console.debug('Waiting for device camera to be loaded');
             return null;
         }
-        if (!this.realVirtualRenderTarget) {
-            this.realVirtualRenderTarget = new THREE.WebGLRenderTarget(640, 480);
-        }
-        const virtualRenderTarget = renderer.getRenderTarget();
-        const renderTarget = this.realVirtualRenderTarget;
+        const mainRenderTarget = renderer.getRenderTarget();
+        const isRenderingStereo = renderer.xr.isPresenting && renderer.xr.getCamera().cameras.length == 2;
+        const mainRenderTargetSingleViewWidth = isRenderingStereo ? mainRenderTarget.width / 2 : mainRenderTarget.width;
+        const scaledHeight = Math.round(mainRenderTarget.height *
+            (this.renderTargetWidth / mainRenderTargetSingleViewWidth));
+        if (!this.virtualRealRenderTarget ||
+            this.virtualRealRenderTarget.height != scaledHeight) {
+            this.virtualRealRenderTarget?.dispose();
+            this.virtualRealRenderTarget = new THREE.WebGLRenderTarget(this.renderTargetWidth, scaledHeight, { colorSpace: THREE.SRGBColorSpace });
+        }
+        const renderTarget = this.virtualRealRenderTarget;
         renderer.setRenderTarget(renderTarget);
+        const xrIsPresenting = renderer.xr.isPresenting;
+        renderer.xr.isPresenting = false;
         const quad = this.getFullScreenQuad();
         quad.material.map = deviceCamera.texture;
         quad.render(renderer);
-        quad.material.map =
-            virtualRenderTarget.texture;
-        quad.render(renderer);
-        renderer.setRenderTarget(virtualRenderTarget);
+        renderSceneFn();
+        renderer.xr.isPresenting = xrIsPresenting;
+        renderer.setRenderTarget(mainRenderTarget);
         if (this.virtualRealBuffer.length !=
             renderTarget.width * renderTarget.height * 4) {
             this.virtualRealBuffer =
@@ -3912,7 +4351,6 @@ class XRButton {
         this.sessionManager.addEventListener(WebXRSessionEventType.READY, () => this.onSessionReady());
         this.sessionManager.addEventListener(WebXRSessionEventType.SESSION_START, () => this.onSessionStarted());
         this.sessionManager.addEventListener(WebXRSessionEventType.SESSION_END, this.onSessionEnded.bind(this));
-        this.sessionManager.initialize();
     }
     createSimulatorButton() {
         this.simulatorButtonElement.classList.add(XRBUTTON_CLASS);
@@ -5503,6 +5941,8 @@ class SimulatorOptions {
         // canvas.
         // This is a temporary option until we figure out why splats look faded.
         this.renderToRenderTexture = true;
+        // Blending mode when rendering the virtual scene.
+        this.blendingMode = 'normal';
         deepMerge(this, options);
     }
 }
@@ -5700,6 +6140,13 @@ class Options {
          * Whether to request a stencil buffer.
          */
         this.stencil = false;
+        /**
+         * Any additional required features when initializing webxr.
+         */
+        this.webxrRequiredFeatures = [];
+        // "local-floor" sets the scene origin at the user's feet,
+        // "local" sets the scene origin near their head.
+        this.referenceSpaceType = 'local-floor';
         this.controllers = new InputOptions();
         this.depth = new DepthOptions();
         this.lighting = new LightingOptions();
@@ -6459,7 +6906,7 @@ class SimulatorDepth {
             data: this.depthBuffer.buffer,
             rawValueToMeters: 1.0,
         };
-        this.depth.updateDepthData(depthData, 0);
+        this.depth.updateCPUDepthData(depthData, 0);
     }
 }
 
@@ -8253,8 +8700,17 @@ class Simulator extends Script {
             this.setupStereoCameras(camera);
         }
         this.virtualSceneRenderTarget = new THREE.WebGLRenderTarget(renderer.domElement.width, renderer.domElement.height, { stencilBuffer: options.stencil });
+        const virtualSceneMaterial = new THREE.MeshBasicMaterial({ map: this.virtualSceneRenderTarget.texture, transparent: true });
+        if (this.options.blendingMode === 'screen') {
+            virtualSceneMaterial.blending = THREE.CustomBlending;
+            virtualSceneMaterial.blendSrc = THREE.OneFactor;
+            virtualSceneMaterial.blendDst =
+                THREE.OneMinusSrcColorFactor;
+            virtualSceneMaterial.blendEquation =
+                THREE.AddEquation;
+        }
         this.virtualSceneFullScreenQuad =
-            new FullScreenQuad(new THREE.MeshBasicMaterial({ map: this.virtualSceneRenderTarget.texture, transparent: true }));
+            new FullScreenQuad(virtualSceneMaterial);
         this.renderer = renderer;
         this.mainCamera = camera;
         this.mainScene = scene;
@@ -8365,6 +8821,8 @@ class AudioListener extends Script {
         super();
         this.isCapturing = false;
         this.latestAudioBuffer = null;
+        this.accumulatedChunks = [];
+        this.isAccumulating = false;
         this.options = {
             sampleRate: 16000,
             channelCount: 1,
@@ -8385,6 +8843,10 @@ class AudioListener extends Script {
             return;
         this.onAudioData = callbacks.onAudioData;
         this.onError = callbacks.onError;
+        this.isAccumulating = callbacks.accumulate || false;
+        if (this.isAccumulating) {
+            this.accumulatedChunks = [];
+        }
         try {
             await this.setupAudioCapture();
             this.isCapturing = true;
@@ -8414,6 +8876,10 @@ class AudioListener extends Script {
         this.processorNode.port.onmessage = (event) => {
             if (event.data.type === 'audioData') {
                 this.latestAudioBuffer = event.data.data;
+                // Accumulate chunks if requested
+                if (this.isAccumulating) {
+                    this.accumulatedChunks.push(event.data.data);
+                }
                 this.onAudioData?.(event.data.data);
                 this.streamToAI(event.data.data);
             }
@@ -8473,6 +8939,8 @@ class AudioListener extends Script {
         this.onAudioData = undefined;
         this.onError = undefined;
         this.latestAudioBuffer = null;
+        this.accumulatedChunks = [];
+        this.isAccumulating = false;
         this.aiService = undefined;
     }
     static isSupported() {
@@ -8487,6 +8955,34 @@ class AudioListener extends Script {
     clearLatestAudioBuffer() {
         this.latestAudioBuffer = null;
     }
+    /**
+     * Gets all accumulated audio chunks as a single combined buffer
+     */
+    getAccumulatedBuffer() {
+        if (this.accumulatedChunks.length === 0)
+            return null;
+        const totalLength = this.accumulatedChunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+        const combined = new ArrayBuffer(totalLength);
+        const combinedArray = new Uint8Array(combined);
+        let offset = 0;
+        for (const chunk of this.accumulatedChunks) {
+            combinedArray.set(new Uint8Array(chunk), offset);
+            offset += chunk.byteLength;
+        }
+        return combined;
+    }
+    /**
+     * Clears accumulated chunks
+     */
+    clearAccumulatedBuffer() {
+        this.accumulatedChunks = [];
+    }
+    /**
+     * Gets the number of accumulated chunks
+     */
+    getAccumulatedChunkCount() {
+        return this.accumulatedChunks.length;
+    }
     dispose() {
         this.stopCapture();
         super.dispose();
@@ -8498,15 +8994,55 @@ class AudioPlayer extends Script {
         super();
         this.options = {};
         this.audioQueue = [];
-        this.isPlaying = false;
         this.nextStartTime = 0;
+        this.volume = 1.0;
+        this.category = 'speech';
         this.options = { sampleRate: 24000, channelCount: 1, ...options };
+        if (options.category) {
+            this.category = options.category;
+        }
+    }
+    /**
+     * Sets the CategoryVolumes instance for this player to respect
+     * master/category volumes
+     */
+    setCategoryVolumes(categoryVolumes) {
+        this.categoryVolumes = categoryVolumes;
+        this.updateGainNodeVolume();
+    }
+    /**
+     * Sets the specific volume for this player (0.0 to 1.0)
+     */
+    setVolume(level) {
+        this.volume = Math.max(0, Math.min(1, level));
+        this.updateGainNodeVolume();
+    }
+    /**
+     * Updates the gain node volume based on category volumes
+     * Public so CoreSound can update it when master volume changes
+     */
+    updateGainNodeVolume() {
+        if (this.gainNode && this.categoryVolumes) {
+            const effectiveVolume = this.categoryVolumes.getEffectiveVolume(this.category, this.volume);
+            this.gainNode.gain.value = effectiveVolume;
+        }
+        else if (this.gainNode) {
+            this.gainNode.gain.value = this.volume;
+        }
     }
     async initializeAudioContext() {
         if (!this.audioContext) {
             this.audioContext =
                 new AudioContext({ sampleRate: this.options.sampleRate });
             this.nextStartTime = this.audioContext.currentTime;
+            // Create gain node for volume control
+            this.gainNode = this.audioContext.createGain();
+            this.gainNode.connect(this.audioContext.destination);
+            this.updateGainNodeVolume();
+        }
+        // Ensure audio context is running (not suspended)
+        if (this.audioContext.state === 'suspended') {
+            await this.audioContext.resume();
         }
     }
     async playAudioChunk(base64AudioData) {
@@ -8521,31 +9057,31 @@ class AudioPlayer extends Script {
             channelData[i] = int16View[i] / 32768.0;
         }
         this.audioQueue.push(audioBuffer);
-        if (!this.isPlaying)
-            this.playNextAudioBuffer();
-    }
-    playNextAudioBuffer() {
-        if (this.audioQueue.length === 0) {
-            this.isPlaying = false;
-            return;
+        this.scheduleAudioBuffers();
+    }
+    scheduleAudioBuffers() {
+        const SCHEDULE_AHEAD_TIME = 0.2;
+        while (this.audioQueue.length > 0 &&
+            this.nextStartTime <=
+                this.audioContext.currentTime + SCHEDULE_AHEAD_TIME) {
+            const audioBuffer = this.audioQueue.shift();
+            const currentTime = this.audioContext.currentTime;
+            const startTime = Math.max(this.nextStartTime, currentTime);
+            const source = this.audioContext.createBufferSource();
+            source.buffer = audioBuffer;
+            // Connect through gain node for volume control
+            source.connect(this.gainNode || this.audioContext.destination);
+            source.onended = () => this.scheduleAudioBuffers();
+            // Start playback
+            source.start(startTime);
+            this.nextStartTime = startTime + audioBuffer.duration;
         }
-        this.isPlaying = true;
-        const audioBuffer = this.audioQueue.shift();
-        const currentTime = this.audioContext.currentTime;
-        const startTime = Math.max(this.nextStartTime, currentTime);
-        const source = this.audioContext.createBufferSource();
-        source.buffer = audioBuffer;
-        source.connect(this.audioContext.destination);
-        source.onended = () => this.playNextAudioBuffer();
-        source.start(startTime);
-        this.nextStartTime = startTime + audioBuffer.duration;
     }
     clearQueue() {
         this.audioQueue = [];
-        this.isPlaying = false;
     }
     getIsPlaying() {
-        return this.isPlaying;
+        return this.nextStartTime > this.audioContext.currentTime;
     }
     getQueueLength() {
         return this.audioQueue.length;
@@ -8563,10 +9099,12 @@ class AudioPlayer extends Script {
         if (this.audioContext) {
             this.audioContext.close();
             this.audioContext = undefined;
+            this.gainNode = undefined;
+            this.nextStartTime = 0; // Reset timing
         }
     }
     static isSupported() {
-        return !!('AudioContext' in window || 'webkitAudioContext' in window);
+        return !!('AudioContext' in window);
     }
     dispose() {
         this.stop();
@@ -9262,7 +9800,10 @@ class CoreSound extends Script {
             new BackgroundMusic(this.listener, this.categoryVolumes);
         this.spatialAudio = new SpatialAudio(this.listener, this.categoryVolumes);
         this.audioListener = new AudioListener();
-        this.audioPlayer = new AudioPlayer();
+        // Initialize with 48kHz for general audio playback
+        // Gemini Live uses 24kHz but that gets handled automatically via playAIAudio
+        this.audioPlayer = new AudioPlayer({ sampleRate: 48000 });
+        this.audioPlayer.setCategoryVolumes(this.categoryVolumes);
         camera.add(this.listener);
         this.add(this.backgroundMusic);
         this.add(this.spatialAudio);
@@ -9283,6 +9824,7 @@ class CoreSound extends Script {
     }
     setMasterVolume(level) {
         this.categoryVolumes.masterVolume = THREE.MathUtils.clamp(level, 0.0, 1.0);
+        this.audioPlayer?.updateGainNodeVolume();
     }
     getMasterVolume() {
         return this.categoryVolumes.isMuted ? 0.0 :
@@ -9300,17 +9842,49 @@ class CoreSound extends Script {
             1.0;
     }
     async enableAudio(options = {}) {
-        const { streamToAI = true } = options;
+        const { streamToAI = true, accumulate = false } = options;
         if (streamToAI && this.speechRecognizer?.isListening) {
             console.log('Disabling SpeechRecognizer while streaming audio.');
             this.speechRecognizer.stop();
         }
         this.audioListener.setAIStreaming(streamToAI);
-        await this.audioListener.startCapture({});
+        await this.audioListener.startCapture({ accumulate });
     }
     disableAudio() {
         this.audioListener?.stopCapture();
     }
+    /**
+     * Starts recording audio with chunk accumulation
+     */
+    async startRecording() {
+        await this.audioListener.startCapture({ accumulate: true });
+    }
+    /**
+     * Stops recording and returns the accumulated audio buffer
+     */
+    stopRecording() {
+        const buffer = this.audioListener.getAccumulatedBuffer();
+        this.audioListener.stopCapture();
+        return buffer;
+    }
+    /**
+     * Gets the accumulated recording buffer without stopping
+     */
+    getRecordedBuffer() {
+        return this.audioListener.getAccumulatedBuffer();
+    }
+    /**
+     * Clears the accumulated recording buffer
+     */
+    clearRecordedBuffer() {
+        this.audioListener.clearAccumulatedBuffer();
+    }
+    /**
+     * Gets the sample rate being used for recording
+     */
+    getRecordingSampleRate() {
+        return this.audioListener.audioContext?.sampleRate || 48000;
+    }
     setAIStreaming(enabled) {
         this.audioListener?.setAIStreaming(enabled);
     }
@@ -9318,6 +9892,16 @@ class CoreSound extends Script {
         return this.audioListener?.aiService !== null;
     }
     async playAIAudio(base64AudioData) {
+        // Gemini Live API outputs audio at 24kHz
+        // Only recreate AudioContext if sample rate needs to change
+        const currentRate = this.audioPlayer['options'].sampleRate;
+        if (currentRate !== 24000) {
+            this.audioPlayer['options'].sampleRate = 24000;
+            // Only stop if context exists and is different sample rate
+            if (this.audioPlayer['audioContext']) {
+                this.audioPlayer.stop(); // Reset context with new sample rate
+            }
+        }
         await this.audioPlayer.playAudioChunk(base64AudioData);
     }
     stopAIAudio() {
@@ -9326,6 +9910,26 @@ class CoreSound extends Script {
     isAIAudioPlaying() {
         return this.audioPlayer?.getIsPlaying();
     }
+    /**
+     * Plays a raw audio buffer (Int16 PCM data) with proper sample rate
+     */
+    async playRecordedAudio(audioBuffer, sampleRate) {
+        if (!audioBuffer)
+            return;
+        // Update sample rate if needed
+        if (sampleRate && sampleRate !== this.audioPlayer['options'].sampleRate) {
+            this.audioPlayer['options'].sampleRate = sampleRate;
+            this.audioPlayer.stop(); // Reset context with new sample rate
+        }
+        // Convert ArrayBuffer to base64
+        const bytes = new Uint8Array(audioBuffer);
+        let binary = '';
+        for (let i = 0; i < bytes.length; i++) {
+            binary += String.fromCharCode(bytes[i]);
+        }
+        const base64Audio = btoa(binary);
+        await this.audioPlayer.playAudioChunk(base64Audio);
+    }
     isAudioEnabled() {
         return this.audioListener?.getIsCapturing();
     }
@@ -9849,7 +10453,6 @@ class TextView extends View {
                     texture.offset.x = this.imageOffsetX;
                     const textObj = this.textObj;
                     textObj.material.map = texture;
-                    textObj.material.needsUpdate = true;
                     textObj.sync();
                 });
             }
@@ -9930,9 +10533,22 @@ class IconButton extends TextView {
         // Applies all provided options to this instance.
         Object.assign(this, options);
     }
+    /**
+     * Initializes the component and sets the render order.
+     */
+    async init(_) {
+        await super.init();
+        if (this.mesh) {
+            this.mesh.renderOrder = this.renderOrder;
+        }
+        if (this.textObj) {
+            this.textObj.renderOrder = this.renderOrder + 1;
+        }
+    }
+    /**
+  
     /**
      * Handles behavior when the cursor hovers over the button.
-     * @override
      */
     onHoverOver() {
         if (!this.ux)
@@ -9941,7 +10557,6 @@ class IconButton extends TextView {
     }
     /**
      * Handles behavior when the cursor moves off the button.
-     * @override
      */
     onHoverOut() {
         if (!this.ux)
@@ -10037,7 +10652,6 @@ class ImageView extends View {
             // If no source, ensure no texture is displayed.
             if (this.material.map) {
                 this.material.map = null;
-                this.material.needsUpdate = true;
             }
             this.texture?.dispose();
             this.texture = undefined;
@@ -10047,7 +10661,6 @@ class ImageView extends View {
         this.texture = this.textureLoader.load(this.src, (loadedTexture) => {
             loadedTexture.colorSpace = THREE.SRGBColorSpace;
             this.material.map = loadedTexture;
-            this.material.needsUpdate = true;
             // Updates layout after the image has loaded to get correct dimensions.
             this.updateLayout();
         });
@@ -10291,14 +10904,19 @@ class TextButton extends TextView {
         // Applies our own overrides to the default values.
         this.fontSize = options.fontSize ?? this.fontSize;
         this.fontColor = options.fontColor ?? this.fontColor;
+        this.width = options.width ?? this.width;
+        this.height = options.height ?? this.height;
     }
     /**
      * Initializes the text object after async dependencies are loaded.
-     * @override
      */
     async init() {
         await super.init();
         this.textObj.position.set(0, 0, VIEW_DEPTH_GAP);
+        if (this.mesh) {
+            this.mesh.renderOrder = this.renderOrder;
+        }
+        this.textObj.renderOrder = this.renderOrder + 1;
         // Disable raycasting on the text part so it doesn't interfere
         // with the main button geometry's interaction.
         this.textObj.raycast = () => { };
@@ -10464,7 +11082,6 @@ class VideoView extends View {
         videoTextureInstance.colorSpace = THREE.SRGBColorSpace;
         this.texture = videoTextureInstance; // Update internal texture reference
         this.material.map = this.texture;
-        this.material.needsUpdate = true;
         const onLoadedMetadata = () => {
             if (this.video.videoWidth && this.video.videoHeight) {
                 this.videoAspectRatio =
@@ -10490,7 +11107,6 @@ class VideoView extends View {
     loadFromVideoTexture(videoTextureInstance) {
         this.texture = videoTextureInstance;
         this.material.map = this.texture;
-        this.material.needsUpdate = true;
         this.video = this.texture.image; // Underlying HTMLVideoElement
         if (this.video && this.video.videoWidth && this.video.videoHeight) {
             this.videoAspectRatio = this.video.videoWidth / this.video.videoHeight;
@@ -13132,8 +13748,9 @@ class Core {
         this.ui = new UI();
         /** Manages all (spatial) audio playback. */
         this.sound = new CoreSound();
+        this.renderSceneBound = this.renderScene.bind(this);
         /** Manages the desktop XR simulator. */
-        this.simulator = new Simulator(this.renderScene.bind(this));
+        this.simulator = new Simulator(this.renderSceneBound);
         /** Manages drag-and-drop interactions. */
         this.dragManager = new DragManager();
         /** Manages drag-and-drop interactions. */
@@ -13204,6 +13821,7 @@ class Core {
         this.registry.register(this.camera, THREE.Camera);
         this.registry.register(this.camera, THREE.PerspectiveCamera);
         this.renderer = new THREE.WebGLRenderer({
+            canvas: options.canvas,
             antialias: options.antialias,
             stencil: options.stencil,
             alpha: true,
@@ -13212,13 +13830,15 @@ class Core {
         this.renderer.setPixelRatio(window.devicePixelRatio);
         this.renderer.setSize(window.innerWidth, window.innerHeight);
         this.renderer.xr.enabled = true;
+        // disable built-in occlusion
+        this.renderer.xr.getDepthSensingMesh = function () { return null; };
         this.registry.register(this.renderer);
-        // "local-floor" sets the scene origin at the user's feet,
-        // "local" sets the scene origin near their head.
-        this.renderer.xr.setReferenceSpaceType('local-floor');
-        const xrContainer = document.createElement('div');
-        document.body.appendChild(xrContainer);
-        xrContainer.appendChild(this.renderer.domElement);
+        this.renderer.xr.setReferenceSpaceType(options.referenceSpaceType);
+        if (!options.canvas) {
+            const xrContainer = document.createElement('div');
+            document.body.appendChild(xrContainer);
+            xrContainer.appendChild(this.renderer.domElement);
+        }
         this.options = options;
         // Sets up controllers.
         if (options.controllers.enabled) {
@@ -13235,17 +13855,16 @@ class Core {
         // Sets up device camera.
         if (options.deviceCamera?.enabled) {
             this.deviceCamera = new XRDeviceCamera(options.deviceCamera);
-            await this.deviceCamera.init();
             this.registry.register(this.deviceCamera);
         }
-        const webXRRequiredFeatures = [];
+        const webXRRequiredFeatures = options.webxrRequiredFeatures;
         this.webXRSettings.requiredFeatures = webXRRequiredFeatures;
         // Sets up depth.
         if (options.depth.enabled) {
             webXRRequiredFeatures.push('depth-sensing');
             webXRRequiredFeatures.push('local-floor');
             this.webXRSettings.depthSensing = {
-                usagePreference: ['cpu-optimized'],
+                usagePreference: [],
                 dataFormatPreference: [this.options.depth.useFloat32 ? 'float32' : 'luminance-alpha'],
             };
             this.depth.init(this.camera, options.depth, this.renderer, this.registry, this.scene);
@@ -13364,7 +13983,7 @@ class Core {
             script.update(time, frame);
         }
         this.renderSimulatorAndScene();
-        this.screenshotSynthesizer.onAfterRender(this.renderer, this.deviceCamera);
+        this.screenshotSynthesizer.onAfterRender(this.renderer, this.renderSceneBound, this.deviceCamera);
         if (this.simulatorRunning) {
             this.simulator.renderSimulatorScene();
         }
@@ -13384,7 +14003,10 @@ class Core {
      * scripts.
      * @param session - The newly started WebXR session.
      */
-    onXRSessionStarted(session) {
+    async onXRSessionStarted(session) {
+        if (this.options.deviceCamera?.enabled) {
+            await this.deviceCamera.init();
+        }
         this.scriptsManager.onXRSessionStarted(session);
     }
     async startSimulator() {
@@ -14010,14 +14632,14 @@ class MaterialSymbolsView extends View {
         const paths = svgData.paths;
         const group = new THREE.Group();
         const scale = 1 / Math.max(viewWidth, viewHeight);
+        const material = new THREE.MeshBasicMaterial({
+            color: this.iconColor,
+            transparent: true,
+            side: THREE.DoubleSide,
+            depthWrite: false,
+        });
         for (let i = 0; i < paths.length; i++) {
             const path = paths[i];
-            const material = new THREE.MeshBasicMaterial({
-                color: this.iconColor,
-                side: THREE.DoubleSide,
-                depthWrite: false,
-                depthTest: false
-            });
             const shapes = SVGLoader.createShapes(path);
             for (let j = 0; j < shapes.length; j++) {
                 const shape = shapes[j];