xrblocks 0.1.0 → 0.2.0

package/README.md

@@ -2,18 +2,22 @@
 
 [![NPM Package](https://img.shields.io/npm/v/xrblocks)](https://www.npmjs.com/package/xrblocks)
 [![Build Size](https://badgen.net/bundlephobia/minzip/xrblocks)](https://bundlephobia.com/result?p=xrblocks)
-[![NPM Downloads](https://img.shields.io/npm/dw/xrblocks)](https://www.npmtrends.com/xrblocks)
+![jsDelivr hits (GitHub)](https://img.shields.io/jsdelivr/gh/hw/google/xrblocks)
 
 #### JavaScript library for rapid XR and AI prototyping
 
+[Site](https://xrblocks.github.io/) —
 [Manual](https://xrblocks.github.io/docs/) —
-[Templates](https://xrblocks.github.io/docs/templates/Basic) —
-[Samples](https://xrblocks.github.io/docs/samples/ModelViewer)
+[Templates](https://xrblocks.github.io/docs/templates/Basic/) —
+[Demos](https://xrblocks.github.io/docs/samples/ModelViewer/) —
+[YouTube](https://www.youtube.com/watch?v=75QJHTsAoB8) —
+[arXiv](https://arxiv.org/abs/2509.25504) —
+[Blog](https://research.google/blog/xr-blocks-accelerating-ai-xr-innovation/)
 
 <p align="center">
- <a href="https://xrblocks.github.io/docs/samples/Ballpit" target="_blank"><img width="32.3%" src="assets/ballpit-demo.webp" alt="Ballpit" /></a>
-  <a href="https://xrblocks.github.io/docs/samples/XR-Emoji" target="_blank"><img width="32.3%" src="assets/xremoji-demo.webp" alt="XR Emoji" /></a>
- <a href="https://xrblocks.github.io/docs/samples/Gemini-Icebreakers" target="_blank"><img width="32.3%" src="assets/gemini-icebreakers-demo.webp" alt="Gemini Icebreakers" /></a>
+  <a href="https://xrblocks.github.io/docs/samples/Ballpit/" target="_blank"><img width="32.3%" src="https://cdn.jsdelivr.net/gh/google/xrblocks@main/assets/ballpit-demo.webp" alt="Ballpit" /></a>
+  <a href="https://xrblocks.github.io/docs/samples/XR-Emoji/" target="_blank"><img width="32.3%" src="https://cdn.jsdelivr.net/gh/google/xrblocks@main/assets/xremoji-demo.webp" alt="XR Emoji" /></a>
+  <a href="https://xrblocks.github.io/docs/samples/Gemini-Icebreakers/" target="_blank"><img width="32.3%" src="https://cdn.jsdelivr.net/gh/google/xrblocks@main/assets/gemini-icebreakers-demo.webp" alt="Gemini Icebreakers" /></a>
 </p>
 
 ### Description
@@ -25,21 +29,27 @@ powerful desktop simulator for development. The framework emphasizes a
 user-centric, developer-friendly SDK designed to simplify the creation of
 immersive applications with features like:
 
--   **Hand Tracking & Gestures:** Access advanced hand tracking, custom
-    gestures with TensorFlow Lite / LiteRT models, and interaction events.
--   **World Understanding:** Present samples with depth sensing, geometry-aware
-    physics, and object recognition with Gemini in both XR and desktop.
--   **AI Integration:** Seamlessly connect to Gemini for multimodal
-    understanding and live conversational experiences.
--   **Cross-Platform:** Write once and deploy to both XR devices and desktop
-    browsers.
+- **Hand Tracking & Gestures:** Access advanced hand tracking, custom
+  gestures with TensorFlow Lite / PyTorch models, and interaction events.
+- **World Understanding:** Present samples with depth sensing, geometry-aware
+  physics, and object recognition with Gemini in both XR and desktop simulator.
+- **AI Integration:** Seamlessly connect to Gemini for multimodal
+  understanding and live conversational experiences.
+- **Cross-Platform:** Write once and deploy to both XR devices and desktop
+  Chrome browsers.
+
+We welcome all contributors to foster an AI + XR community! Read our
+[blog post](https://research.google/blog/xr-blocks-accelerating-ai-xr-innovation/)
+and [white paper](https://arxiv.org/abs/2509.25504) for a visionary roadmap.
 
 ### Usage
 
 XR Blocks can be imported directly into a webpage using an importmap. This code
 creates a basic XR scene containing a cylinder. When you view the scene, you can
 pinch your fingers (in XR) or click (in the desktop simulator) to change the
-cylinder's color.
+cylinder's color. Check out
+[this live demo](https://xrblocks.github.io/docs/templates/Basic/) with simple
+code below:
 
 ```html
 <!DOCTYPE html>
@@ -132,8 +142,9 @@ npm ci
 npm run build
 ```
 
-This is not an officially supported Google product. This project is not eligible
-for the
+This is not an officially supported Google product, but will be actively
+maintained by the XR Labs team and external collaborators. This project is not
+eligible for the
 [Google Open Source Software Vulnerability Rewards Program](https://bughunters.google.com/open-source-security).
 
 ### User Data & Permissions
@@ -183,11 +194,85 @@ To remove XR Blocks from your code, simple remove the lines from your `<script
 type="importmap">` tag in HTML, or `import * from xrblocks` in JavaScript, or
 use `npm uninstall xrblocks` from your project directory.
 
+### References
+
+If you find XR Blocks inspiring or useful in your research, please reference it
+as:
+
+```bibtex
+@misc{Li2025XR,
+  title={{XR Blocks: Accelerating Human-centered AI + XR Innovation}},
+  author={Li, David and Numan, Nels and Qian, Xun and Chen, Yanhe and Zhou, Zhongyi and Alekseev, Evgenii and Lee, Geonsun and Cooper, Alex and Xia, Min and Chung, Scott and Nelson, Jeremy and Yuan, Xiuxiu and Dias, Jolica and Bettridge, Tim and Hersh, Benjamin and Huynh, Michelle and Piascik, Konrad and Cabello, Ricardo and Kim, David and Du, Ruofei},
+  year={2025},
+  eprint={2509.25504},
+  archivePrefix={arXiv},
+  primaryClass={cs.HC},
+  url={https://arxiv.org/abs/2509.25504},
+}
+```
+
+#### Key Works Built with XR Blocks
+
+These references are built with XR Blocks:
+
+```bibtex
+@inproceedings{Lee2025Sensible,
+  title = {{Sensible Agent: A Framework for Unobtrusive Interaction with Proactive AR Agent}},
+  author = {Lee, Geonsun and Xia, Min and Numan, Nels and Qian, Xun and Li, David and Chen, Yanhe and Kulshrestha, Achin and Chatterjee, Ishan and Zhang, Yinda and Manocha, Dinesh and Kim, David and Du, Ruofei},
+  booktitle = {Proceedings of the 39th Annual ACM Symposium on User Interface Software and Technology},
+  year = {2025},
+  publisher = {ACM},
+  numpages = {22},
+  series = {UIST},
+  doi = {10.1145/3746059.3747748},
+}
+```
+
+#### Inspiring Related Works
+
+We call for contributors to integrate our prior art into XR Blocks to enhance
+reproducibility and knowledge sharing:
+
+E.g., integrating models from https://visualblocks.withgoogle.com to XR Blocks:
+
+```bibtex
+@inproceedings{Du2023Rapsai,
+  title = {{Rapsai: Accelerating Machine Learning Prototyping of Multimedia Applications Through Visual Programming}},
+  author = {Du, Ruofei and Li, Na and Jin, Jing and Carney, Michelle and Miles, Scott and Kleiner, Maria and Yuan, Xiuxiu and Zhang, Yinda and Kulkarni, Anuva and Liu, XingyuBruce and Sabie, Ahmed and Orts-Escolano, Sergio and Kar, Abhishek and Yu, Ping and Iyengar, Ram and Kowdle, Adarsh and Olwal, Alex},
+  booktitle = {Proceedings of the 2023 CHI Conference on Human Factors in Computing Systems},
+  year = {2023},
+  publisher = {ACM},
+  month = {Apr.},
+  day = {22-29},
+  number = {125},
+  pages = {1--23},
+  series = {CHI},
+  doi = {10.1145/3544548.3581338},
+}
+```
+
+Extending XR Blocks to XR communication:
+
+```bibtex
+@inproceedings{Hu2025DialogLab,
+  title = {{DialogLab: Authoring, Simulating, and Testing Dynamic Group Conversations in Hybrid Human-AI Conversations}},
+  author = {Hu, Erzhen and Chen, Yanhe and Li, Mingyi and Phadnis, Vrushank and Xu, Pingmei and Qian, Xun and Olwal, Alex and Kim, David and Heo, Seongkook and Du, Ruofei},
+  booktitle = {Proceedings of the 39th Annual ACM Symposium on User Interface Software and Technology},
+  year = {2025},
+  number = {210},
+  publisher = {ACM},
+  number = {210},
+  pages = {1--20},
+  series = {UIST},
+  doi = {10.1145/3746059.3747696},
+}
+```
+
 ### Terms of Service
 
--   Please follow
-    [Google's Privacy & Terms](https://ai.google.dev/gemini-api/terms) when
-    using this SDK.
+- Please follow
+  [Google's Privacy & Terms](https://ai.google.dev/gemini-api/terms) when
+  using this SDK.
 
--   When using AI features in this SDK, please follow
-    [Gemini's Privacy & Terms](https://ai.google.dev/gemini-api/terms).
+- When using AI features in this SDK, please follow
+  [Gemini's Privacy & Terms](https://ai.google.dev/gemini-api/terms).

package/build/addons/ai/GeminiManager.d.ts

@@ -19,23 +19,26 @@ export declare class GeminiManager extends xb.Script<GeminiManagerEventMap> {
     processorNode: AudioWorkletNode | null;
     isAIRunning: boolean;
     audioQueue: AudioBuffer[];
-    isPlayingAudio: boolean;
+    nextAudioStartTime: number;
     private screenshotInterval?;
     currentInputText: string;
     currentOutputText: string;
+    tools: xb.Tool[];
     constructor();
     init(): void;
-    startGeminiLive(): Promise<void>;
+    startGeminiLive({ liveParams }?: {
+        liveParams?: xb.GeminiStartLiveSessionParams;
+    }): Promise<void>;
     stopGeminiLive(): Promise<void>;
     setupAudioCapture(): Promise<void>;
-    startLiveAI(): Promise<void>;
+    startLiveAI(params: xb.GeminiStartLiveSessionParams): Promise<void>;
     startScreenshotCapture(intervalMs?: number): void;
     captureAndSendScreenshot(): void;
     sendAudioData(audioBuffer: ArrayBuffer): void;
     sendVideoFrame(base64Image: string): void;
     initializeAudioContext(): Promise<void>;
     playAudioChunk(audioData: string): Promise<void>;
-    playNextAudioBuffer(): void;
+    scheduleAudioBuffers(): void;
     cleanup(): void;
     handleAIMessage(message: GoogleGenAITypes.LiveServerMessage): void;
     arrayBufferToBase64(buffer: ArrayBuffer): string;

package/build/addons/ai/GeminiManager.js

@@ -12,23 +12,29 @@ class GeminiManager extends xb.Script {
         this.isAIRunning = false;
         // Audio playback setup
         this.audioQueue = [];
-        this.isPlayingAudio = false;
+        this.nextAudioStartTime = 0;
         // Transcription state
         this.currentInputText = '';
         this.currentOutputText = '';
+        this.tools = [];
     }
     init() {
         this.xrDeviceCamera = xb.core.deviceCamera;
         this.ai = xb.core.ai;
     }
-    async startGeminiLive() {
+    async startGeminiLive({ liveParams } = {}) {
         if (this.isAIRunning || !this.ai) {
             console.warn('AI already running or not available');
             return;
         }
+        liveParams = liveParams || {};
+        liveParams.tools = liveParams.tools || [];
+        for (const tool of this.tools) {
+            liveParams.tools.push(tool.toJSON());
+        }
         try {
             await this.setupAudioCapture();
-            await this.startLiveAI();
+            await this.startLiveAI(liveParams);
             this.startScreenshotCapture();
             this.isAIRunning = true;
         }
@@ -82,7 +88,7 @@ class GeminiManager extends xb.Script {
         this.sourceNode.connect(this.processorNode);
         this.processorNode.connect(this.audioContext.destination);
     }
-    async startLiveAI() {
+    async startLiveAI(params) {
         return new Promise((resolve, reject) => {
             this.ai.setLiveCallbacks({
                 onopen: () => {
@@ -99,7 +105,7 @@ class GeminiManager extends xb.Script {
                     this.isAIRunning = false;
                 }
             });
-            this.ai.startLiveSession().catch(reject);
+            this.ai.startLiveSession(params).catch(reject);
         });
     }
     startScreenshotCapture(intervalMs = 1000) {
@@ -174,28 +180,28 @@ class GeminiManager extends xb.Script {
                 channelData[i] = int16View[i] / 32768.0;
             }
             this.audioQueue.push(audioBuffer);
-            if (!this.isPlayingAudio) {
-                this.playNextAudioBuffer();
-            }
+            this.scheduleAudioBuffers();
         }
         catch (error) {
             console.error('Error playing audio chunk:', error);
         }
     }
-    playNextAudioBuffer() {
-        if (this.audioQueue.length === 0) {
-            this.isPlayingAudio = false;
-            return;
+    scheduleAudioBuffers() {
+        const SCHEDULE_AHEAD_TIME = 0.2;
+        while (this.audioQueue.length > 0 &&
+            this.nextAudioStartTime <=
+                this.audioContext.currentTime + SCHEDULE_AHEAD_TIME) {
+            const audioBuffer = this.audioQueue.shift();
+            const source = this.audioContext.createBufferSource();
+            source.buffer = audioBuffer;
+            source.connect(this.audioContext.destination);
+            source.onended = () => {
+                this.scheduleAudioBuffers();
+            };
+            const startTime = Math.max(this.nextAudioStartTime, this.audioContext.currentTime);
+            source.start(startTime);
+            this.nextAudioStartTime = startTime + audioBuffer.duration;
         }
-        this.isPlayingAudio = true;
-        const audioBuffer = this.audioQueue.shift();
-        const source = this.audioContext.createBufferSource();
-        source.buffer = audioBuffer;
-        source.connect(this.audioContext.destination);
-        source.onended = () => {
-            this.playNextAudioBuffer();
-        };
-        source.start();
     }
     cleanup() {
         if (this.screenshotInterval) {
@@ -204,7 +210,6 @@ class GeminiManager extends xb.Script {
         }
         // Clear audio queue and stop playback
         this.audioQueue = [];
-        this.isPlayingAudio = false;
         if (this.processorNode) {
             this.processorNode.disconnect();
             this.processorNode = null;
@@ -226,6 +231,22 @@ class GeminiManager extends xb.Script {
         if (message.data) {
             this.playAudioChunk(message.data);
         }
+        for (const functionCall of message.toolCall?.functionCalls ?? []) {
+            const tool = this.tools.find(tool => tool.name == functionCall.name);
+            if (tool) {
+                const exec = tool.execute(functionCall.args);
+                exec.then(result => {
+                    this.ai.sendToolResponse({
+                        functionResponses: {
+                            id: functionCall.id,
+                            name: functionCall.name,
+                            response: { 'output': result }
+                        }
+                    });
+                })
+                    .catch((error) => console.error('Tool error:', error));
+            }
+        }
         if (message.serverContent) {
             if (message.serverContent.inputTranscription) {
                 const text = message.serverContent.inputTranscription.text;

package/build/agent/Agent.d.ts

@@ -2,6 +2,19 @@ import { AI } from '../ai/AI';
 import { Context } from './Context';
 import { Memory } from './Memory';
 import { Tool } from './Tool';
+/**
+ * Lifecycle callbacks for agent events.
+ */
+export interface AgentLifecycleCallbacks {
+    /** Called when a session starts */
+    onSessionStart?: () => void | Promise<void>;
+    /** Called when a session ends */
+    onSessionEnd?: () => void | Promise<void>;
+    /** Called after a tool is executed */
+    onToolExecuted?: (toolName: string, result: unknown) => void;
+    /** Called when an error occurs */
+    onError?: (error: Error) => void;
+}
 /**
  * An agent that can use an AI to reason and execute tools.
  */
@@ -11,7 +24,9 @@ export declare class Agent {
     tools: Tool[];
     memory: Memory;
     contextBuilder: Context;
-    constructor(ai: AI, tools?: Tool[], instruction?: string);
+    lifecycleCallbacks?: AgentLifecycleCallbacks;
+    isSessionActive: boolean;
+    constructor(ai: AI, tools?: Tool[], instruction?: string, callbacks?: AgentLifecycleCallbacks);
     /**
      * Starts the agent's reasoning loop with an initial prompt.
      * @param prompt - The initial prompt from the user.
@@ -25,4 +40,13 @@ export declare class Agent {
      */
     private run;
     findTool(name: string): Tool | undefined;
+    /**
+     * Get the current session state.
+     * @returns Object containing session information
+     */
+    getSessionState(): {
+        isActive: boolean;
+        toolCount: number;
+        memorySize: number;
+    };
 }

package/build/agent/SkyboxAgent.d.ts

@@ -2,11 +2,127 @@ import type * as GoogleGenAITypes from '@google/genai';
 import * as THREE from 'three';
 import { AI } from '../ai/AI';
 import { CoreSound } from '../sound/CoreSound';
-import { Agent } from './Agent';
+import { Agent, AgentLifecycleCallbacks } from './Agent';
+import { ToolResult } from './Tool';
+/**
+ * State information for a live session.
+ */
+export interface LiveSessionState {
+    /** Whether the session is currently active */
+    isActive: boolean;
+    /** Timestamp when session started */
+    startTime?: number;
+    /** Timestamp when session ended */
+    endTime?: number;
+    /** Number of messages received */
+    messageCount: number;
+    /** Number of tool calls executed */
+    toolCallCount: number;
+    /** Last error message if any */
+    lastError?: string;
+}
+/**
+ * Skybox Agent for generating 360-degree equirectangular backgrounds through conversation.
+ *
+ * @example Basic usage
+ * ```typescript
+ * // 1. Enable audio (required for live sessions)
+ * await xb.core.sound.enableAudio();
+ *
+ * // 2. Create agent
+ * const agent = new xb.SkyboxAgent(xb.core.ai, xb.core.sound, xb.core.scene);
+ *
+ * // 3. Start session
+ * await agent.startLiveSession({
+ *   onopen: () => console.log('Session ready'),
+ *   onmessage: (msg) => handleMessage(msg),
+ *   onclose: () => console.log('Session closed')
+ * });
+ *
+ * // 4. Clean up when done
+ * await agent.stopLiveSession();
+ * xb.core.sound.disableAudio();
+ * ```
+ *
+ * @example With lifecycle callbacks
+ * ```typescript
+ * const agent = new xb.SkyboxAgent(
+ *   xb.core.ai,
+ *   xb.core.sound,
+ *   xb.core.scene,
+ *   {
+ *     onSessionStart: () => updateUI('active'),
+ *     onSessionEnd: () => updateUI('inactive'),
+ *     onError: (error) => showError(error)
+ *   }
+ * );
+ * ```
+ *
+ * @remarks
+ * - Audio must be enabled BEFORE starting live session using `xb.core.sound.enableAudio()`
+ * - Users are responsible for managing audio lifecycle
+ * - Always call `stopLiveSession()` before disabling audio
+ * - Session state can be checked using `getSessionState()` and `getLiveSessionState()`
+ */
 export declare class SkyboxAgent extends Agent {
     private sound;
-    constructor(ai: AI, sound: CoreSound, scene: THREE.Scene);
-    startLiveSession(callbacks: GoogleGenAITypes.LiveCallbacks): Promise<void>;
+    private sessionState;
+    constructor(ai: AI, sound: CoreSound, scene: THREE.Scene, callbacks?: AgentLifecycleCallbacks);
+    /**
+     * Starts a live AI session for real-time conversation.
+     *
+     * @param callbacks - Optional callbacks for session events. Can also be set using ai.setLiveCallbacks()
+     * @throws If AI model is not initialized or live session is not available
+     *
+     * @remarks
+     * Audio must be enabled separately using `xb.core.sound.enableAudio()` before starting the session.
+     * This gives users control over when microphone permissions are requested.
+     */
+    startLiveSession(callbacks?: GoogleGenAITypes.LiveCallbacks): Promise<void>;
+    /**
+     * Stops the live AI session.
+     *
+     * @remarks
+     * Audio must be disabled separately using `xb.core.sound.disableAudio()` after stopping the session.
+     */
     stopLiveSession(): Promise<void>;
+    /**
+     * Wraps user callbacks to track session state and trigger lifecycle events.
+     * @param callbacks - The callbacks to wrap.
+     * @returns The wrapped callbacks.
+     */
+    private wrapCallbacks;
+    /**
+     * Sends tool execution results back to the AI.
+     *
+     * @param response - The tool response containing function results
+     */
     sendToolResponse(response: GoogleGenAITypes.LiveSendToolResponseParameters): Promise<void>;
+    /**
+     * Validates that a tool response has the correct format.
+     * @param response - The tool response to validate.
+     * @returns True if the response is valid, false otherwise.
+     */
+    private validateToolResponse;
+    /**
+     * Helper to create a properly formatted tool response from a ToolResult.
+     *
+     * @param id - The function call ID
+     * @param name - The function name
+     * @param result - The ToolResult from tool execution
+     * @returns A properly formatted FunctionResponse
+     */
+    static createToolResponse(id: string, name: string, result: ToolResult): GoogleGenAITypes.FunctionResponse;
+    /**
+     * Gets the current live session state.
+     *
+     * @returns Read-only session state information
+     */
+    getLiveSessionState(): Readonly<LiveSessionState>;
+    /**
+     * Gets the duration of the session in milliseconds.
+     *
+     * @returns Duration in ms, or null if session hasn't started
+     */
+    getSessionDuration(): number | null;
 }

package/build/agent/Tool.d.ts

@@ -3,6 +3,20 @@ export interface ToolCall {
     name: string;
     args: unknown;
 }
+/**
+ * Standardized result type for tool execution.
+ * @typeParam T - The type of data returned on success.
+ */
+export interface ToolResult<T = unknown> {
+    /** Whether the tool execution succeeded */
+    success: boolean;
+    /** The result data if successful */
+    data?: T;
+    /** Error message if execution failed */
+    error?: string;
+    /** Additional metadata about the execution */
+    metadata?: Record<string, unknown>;
+}
 export type ToolSchema = Omit<GoogleGenAITypes.Schema, 'type' | 'properties'> & {
     properties?: Record<string, ToolSchema>;
     type?: keyof typeof GoogleGenAITypes.Type;
@@ -15,7 +29,7 @@ export type ToolOptions = {
     /** The parameters of the tool */
     parameters?: ToolSchema;
     /** A callback to execute when the tool is triggered */
-    onTriggered?: (args: unknown) => unknown;
+    onTriggered?: (args: unknown) => unknown | Promise<unknown>;
 };
 /**
  * A base class for tools that the agent can use.
@@ -30,11 +44,11 @@ export declare class Tool {
      */
     constructor(options: ToolOptions);
     /**
-     * Executes the tool's action.
+     * Executes the tool's action with standardized error handling.
      * @param args - The arguments for the tool.
-     * @returns The result of the tool's action.
+     * @returns A promise that resolves with a ToolResult containing success/error information.
      */
-    execute(args: unknown): unknown;
+    execute(args: unknown): Promise<ToolResult>;
     /**
      * Returns a JSON representation of the tool.
      * @returns A valid FunctionDeclaration object.

package/build/agent/index.d.ts

@@ -1,4 +1,5 @@
 export * from './Agent';
 export * from './Memory';
+export * from './SkyboxAgent';
 export * from './Tool';
 export * from './tools/index';

package/build/agent/tools/GenerateSkyboxTool.d.ts

@@ -1,6 +1,6 @@
 import * as THREE from 'three';
 import { AI } from '../../ai/AI';
-import { Tool } from '../Tool';
+import { Tool, ToolResult } from '../Tool';
 /**
  * A tool that generates a 360-degree equirectangular skybox image
  * based on a given prompt using an AI service.
@@ -12,9 +12,9 @@ export declare class GenerateSkyboxTool extends Tool {
     /**
      * Executes the tool's action.
      * @param args - The prompt to use to generate the skybox.
-     * @returns A promise that resolves with the result of the skybox generation.
+     * @returns A promise that resolves with a ToolResult containing success/error information.
      */
     execute(args: {
         prompt: string;
-    }): Promise<string>;
+    }): Promise<ToolResult<string>>;
 }

package/build/agent/tools/GetWeatherTool.d.ts

@@ -1,13 +1,12 @@
-import { Tool } from '../Tool';
+import { Tool, ToolResult } from '../Tool';
 export interface GetWeatherArgs {
     latitude: number;
     longitude: number;
 }
-export type GetWeatherToolResults = {
-    error?: string;
-    temperature?: number;
-    weathercode?: number;
-};
+export interface WeatherData {
+    temperature: number;
+    weathercode: number;
+}
 /**
  * A tool that gets the current weather for a specific location.
  */
@@ -16,7 +15,7 @@ export declare class GetWeatherTool extends Tool {
     /**
      * Executes the tool's action.
      * @param args - The arguments for the tool.
-     * @returns A promise that resolves with the weather information.
+     * @returns A promise that resolves with a ToolResult containing weather information.
      */
-    execute(args: GetWeatherArgs): Promise<GetWeatherToolResults>;
+    execute(args: GetWeatherArgs): Promise<ToolResult<WeatherData>>;
 }

package/build/ai/AI.d.ts

@@ -2,7 +2,7 @@ import type * as GoogleGenAITypes from '@google/genai';
 import { Script } from '../core/Script';
 import { AIOptions, GeminiOptions, OpenAIOptions } from './AIOptions';
 import { GeminiResponse } from './AITypes';
-import { Gemini } from './Gemini';
+import { Gemini, GeminiStartLiveSessionParams } from './Gemini';
 import { OpenAI } from './OpenAI';
 export type ModelClass = Gemini | OpenAI;
 export type ModelOptions = GeminiOptions | OpenAIOptions;
@@ -60,7 +60,7 @@ export declare class AI extends Script {
     query(input: {
         prompt: string;
     }, tools?: never[]): Promise<GeminiResponse | string | null>;
-    startLiveSession(config?: {}): Promise<GoogleGenAITypes.Session>;
+    startLiveSession(config?: GeminiStartLiveSessionParams): Promise<GoogleGenAITypes.Session>;
     stopLiveSession(): Promise<void>;
     setLiveCallbacks(callbacks: GoogleGenAITypes.LiveCallbacks): Promise<void>;
     sendToolResponse(response: GoogleGenAITypes.LiveSendToolResponseParameters): void;

package/build/ai/Gemini.d.ts

@@ -14,6 +14,10 @@ export interface GeminiQueryInput {
     config?: GoogleGenAITypes.LiveConnectConfig;
     data?: GoogleGenAITypes.LiveSendRealtimeInputParameters;
 }
+export type GeminiStartLiveSessionParams = {
+    tools?: GoogleGenAITypes.FunctionDeclaration[];
+    systemInstruction?: GoogleGenAITypes.ContentUnion | string;
+};
 export declare class Gemini extends BaseAIModel {
     protected options: GeminiOptions;
     inited: boolean;
@@ -25,10 +29,7 @@ export declare class Gemini extends BaseAIModel {
     init(): Promise<void>;
     isAvailable(): boolean;
     isLiveAvailable(): false | typeof GoogleGenAITypes.Modality | undefined;
-    startLiveSession(params?: {
-        tools?: GoogleGenAITypes.FunctionDeclaration[];
-        systemInstruction?: GoogleGenAITypes.ContentUnion | string;
-    }): Promise<GoogleGenAITypes.Session>;
+    startLiveSession(params?: GeminiStartLiveSessionParams): Promise<GoogleGenAITypes.Session>;
     stopLiveSession(): Promise<void>;
     setLiveCallbacks(callbacks: GoogleGenAITypes.LiveCallbacks): void;
     sendToolResponse(response: GoogleGenAITypes.LiveSendToolResponseParameters): void;

package/build/core/Core.d.ts

@@ -56,6 +56,7 @@ export declare class Core {
     ui: UI;
     /** Manages all (spatial) audio playback. */
     sound: CoreSound;
+    private renderSceneBound;
     /** Manages the desktop XR simulator. */
     simulator: Simulator;
     /** Manages drag-and-drop interactions. */