xrblocks 0.5.0 → 0.6.0

package/build/ai/AI.d.ts

@@ -76,7 +76,7 @@ export declare class AI extends Script {
      * In XR mode, show a 3D UI to instruct users to get an API key.
      */
     triggerKeyPopup(): void;
-    generate(prompt: string | string[], type?: 'image', systemInstruction?: string, model?: string): Promise<string | void | undefined>;
+    generate(prompt: string | string[], type?: 'image', systemInstruction?: string, model?: undefined): Promise<string | void | undefined>;
     /**
      * Create a sample keys.json file structure for reference
      * @returns Sample keys.json structure

package/build/camera/CameraOptions.d.ts

@@ -1,15 +1,46 @@
 import { DeepPartial, DeepReadonly } from '../utils/Types';
+/**
+ * Parameters for RGB to depth UV mapping given different aspect ratios.
+ * These parameters define the distortion model and affine transformations
+ * required to align the RGB camera feed with the depth map.
+ */
+export interface RgbToDepthParams {
+    scale: number;
+    scaleX: number;
+    scaleY: number;
+    translateU: number;
+    translateV: number;
+    k1: number;
+    k2: number;
+    k3: number;
+    p1: number;
+    p2: number;
+    xc: number;
+    yc: number;
+}
+/**
+ * Default parameters for rgb to depth projection.
+ * For RGB and depth, 4:3 and 1:1, respectively.
+ */
+export declare const DEFAULT_RGB_TO_DEPTH_PARAMS: RgbToDepthParams;
+/**
+ * Configuration options for the device camera.
+ */
 export declare class DeviceCameraOptions {
     enabled: boolean;
     /**
      * Constraints for `getUserMedia`. This will guide the initial camera
-     * selection. *
+     * selection.
      */
     videoConstraints?: MediaTrackConstraints;
     /**
      * Hint for performance optimization on frequent captures.
      */
     willCaptureFrequently: boolean;
+    /**
+     * Parameters for RGB to depth UV mapping given different aspect ratios.
+     */
+    rgbToDepthParams: RgbToDepthParams;
     constructor(options?: DeepReadonly<DeepPartial<DeviceCameraOptions>>);
 }
 export declare const xrDeviceCameraEnvironmentOptions: {
@@ -167,6 +198,20 @@ export declare const xrDeviceCameraEnvironmentOptions: {
         } | undefined;
     } | undefined;
     readonly willCaptureFrequently: boolean;
+    readonly rgbToDepthParams: {
+        readonly scale: number;
+        readonly scaleX: number;
+        readonly scaleY: number;
+        readonly translateU: number;
+        readonly translateV: number;
+        readonly k1: number;
+        readonly k2: number;
+        readonly k3: number;
+        readonly p1: number;
+        readonly p2: number;
+        readonly xc: number;
+        readonly yc: number;
+    };
 };
 export declare const xrDeviceCameraUserOptions: {
     readonly enabled: boolean;
@@ -323,6 +368,20 @@ export declare const xrDeviceCameraUserOptions: {
         } | undefined;
     } | undefined;
     readonly willCaptureFrequently: boolean;
+    readonly rgbToDepthParams: {
+        readonly scale: number;
+        readonly scaleX: number;
+        readonly scaleY: number;
+        readonly translateU: number;
+        readonly translateV: number;
+        readonly k1: number;
+        readonly k2: number;
+        readonly k3: number;
+        readonly p1: number;
+        readonly p2: number;
+        readonly xc: number;
+        readonly yc: number;
+    };
 };
 export declare const xrDeviceCameraEnvironmentContinuousOptions: {
     readonly enabled: boolean;
@@ -479,6 +538,20 @@ export declare const xrDeviceCameraEnvironmentContinuousOptions: {
         } | undefined;
     } | undefined;
     readonly willCaptureFrequently: boolean;
+    readonly rgbToDepthParams: {
+        readonly scale: number;
+        readonly scaleX: number;
+        readonly scaleY: number;
+        readonly translateU: number;
+        readonly translateV: number;
+        readonly k1: number;
+        readonly k2: number;
+        readonly k3: number;
+        readonly p1: number;
+        readonly p2: number;
+        readonly xc: number;
+        readonly yc: number;
+    };
 };
 export declare const xrDeviceCameraUserContinuousOptions: {
     readonly enabled: boolean;
@@ -635,4 +708,18 @@ export declare const xrDeviceCameraUserContinuousOptions: {
         } | undefined;
     } | undefined;
     readonly willCaptureFrequently: boolean;
+    readonly rgbToDepthParams: {
+        readonly scale: number;
+        readonly scaleX: number;
+        readonly scaleY: number;
+        readonly translateU: number;
+        readonly translateV: number;
+        readonly k1: number;
+        readonly k2: number;
+        readonly k3: number;
+        readonly p1: number;
+        readonly p2: number;
+        readonly xc: number;
+        readonly yc: number;
+    };
 };

package/build/camera/CameraUtils.d.ts

@@ -6,29 +6,27 @@ export declare const aspectRatios: {
     RGB: number;
 };
 /**
- * Parameters for RGB to depth UV mapping (manually calibrated for aspect
- * ratios. For RGB and depth, 4:3 and 1:1, respectively.
+ * Maps a UV coordinate from a RGB space to a destination depth space,
+ * applying Brown-Conrady distortion and affine transformations based on
+ * aspect ratios. If the simulator camera is used, no transformation is applied.
+ *
+ * @param rgbUv - The RGB UV coordinate, e.g., \{ u: 0.5, v: 0.5 \}.
+ * @param xrDeviceCamera - The device camera instance.
+ * @returns The transformed UV coordinate in the render camera clip space, or null if
+ *     inputs are invalid.
  */
-export declare const rgbToDepthParams: {
-    scale: number;
-    scaleX: number;
-    scaleY: number;
-    translateU: number;
-    translateV: number;
-    k1: number;
-    k2: number;
-    k3: number;
-    p1: number;
-    p2: number;
-    xc: number;
-    yc: number;
-};
+export declare function transformRgbToRenderCameraClip(rgbUv: {
+    u: number;
+    v: number;
+}, xrDeviceCamera?: XRDeviceCamera): THREE.Vector2 | null;
 /**
  * Maps a UV coordinate from a RGB space to a destination depth space,
  * applying Brown-Conrady distortion and affine transformations based on
  * aspect ratios. If the simulator camera is used, no transformation is applied.
  *
  * @param rgbUv - The RGB UV coordinate, e.g., \{ u: 0.5, v: 0.5 \}.
+ * @param renderCameraWorldFromClip - Render camera world from clip, i.e. inverse of the View Projection matrix.
+ * @param depthCameraClipFromWorld - Depth camera clip from world, i.e.
  * @param xrDeviceCamera - The device camera instance.
  * @returns The transformed UV coordinate in the depth image space, or null if
  *     inputs are invalid.
@@ -36,7 +34,7 @@ export declare const rgbToDepthParams: {
 export declare function transformRgbToDepthUv(rgbUv: {
     u: number;
     v: number;
-}, xrDeviceCamera?: XRDeviceCamera): {
+}, renderCameraWorldFromClip: THREE.Matrix4, depthCameraClipFromWorld: THREE.Matrix4, xrDeviceCamera?: XRDeviceCamera): {
     u: number;
     v: number;
 } | null;
@@ -48,9 +46,9 @@ export declare function transformRgbToDepthUv(rgbUv: {
  *
  * @param rgbUv - The RGB UV coordinate, e.g., \{ u: 0.5, v: 0.5 \}.
  * @param depthArray - Array containing depth data.
- * @param viewProjectionMatrix - XRView object with corresponding
+ * @param projectionMatrix - XRView object with corresponding
  * projection matrix.
- * @param matrixWorld - Matrix for view-to-world translation.
+ * @param matrixWorld - Rendering camera's model matrix.
  * @param xrDeviceCamera - The device camera instance.
  * @param xrDepth - The SDK's Depth module.
  * @returns Vertex at (u, v) in world space.
@@ -58,7 +56,7 @@ export declare function transformRgbToDepthUv(rgbUv: {
 export declare function transformRgbUvToWorld(rgbUv: {
     u: number;
     v: number;
-}, depthArray: number[] | Uint16Array | Float32Array, viewProjectionMatrix: THREE.Matrix4, matrixWorld: THREE.Matrix4, xrDeviceCamera?: XRDeviceCamera, xrDepth?: Depth | undefined): THREE.Vector3 | null;
+}, depthArray: number[] | Uint16Array | Float32Array, projectionMatrix: THREE.Matrix4, matrixWorld: THREE.Matrix4, xrDeviceCamera?: XRDeviceCamera, xrDepth?: Depth | undefined): THREE.Vector3;
 /**
  * Asynchronously crops a base64 encoded image using a THREE.Box2 bounding box.
  * This function creates an in-memory image, draws a specified portion of it to

package/build/camera/XRDeviceCamera.d.ts

@@ -1,7 +1,7 @@
 import { SimulatorCamera } from '../simulator/SimulatorCamera';
 import { SimulatorMediaDeviceInfo } from '../simulator/SimulatorMediaDeviceInfo';
 import { VideoStream, VideoStreamDetails } from '../video/VideoStream';
-import { DeviceCameraOptions } from './CameraOptions';
+import { DeviceCameraOptions, RgbToDepthParams } from './CameraOptions';
 export type MediaOrSimulatorMediaDeviceInfo = MediaDeviceInfo | SimulatorMediaDeviceInfo;
 type XRDeviceCameraDetails = VideoStreamDetails & {
     width?: number;
@@ -15,6 +15,7 @@ type XRDeviceCameraDetails = VideoStreamDetails & {
  */
 export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
     simulatorCamera?: SimulatorCamera;
+    rgbToDepthParams: RgbToDepthParams;
     protected videoConstraints_: MediaTrackConstraints;
     private isInitializing_;
     private availableDevices_;
@@ -23,7 +24,7 @@ export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
     /**
      * @param options - The configuration options.
      */
-    constructor({ videoConstraints, willCaptureFrequently, }?: Partial<DeviceCameraOptions>);
+    constructor({ videoConstraints, willCaptureFrequently, rgbToDepthParams, }?: Partial<DeviceCameraOptions>);
     /**
      * Retrieves the list of available video input devices.
      * @returns A promise that resolves with an
@@ -43,7 +44,7 @@ export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
     /**
      * Sets the active camera by its device ID. Removes potentially conflicting
      * constraints such as facingMode.
-     * @param deviceId - Device id.
+     * @param deviceId - Device ID
      */
     setDeviceId(deviceId: string): Promise<void>;
     /**

package/build/depth/Depth.d.ts

@@ -10,7 +10,6 @@ export declare class Depth {
     private renderer;
     private scene;
     private projectionMatrixInverse;
-    private xrRefSpace?;
     view: XRView[];
     cpuDepthData: XRCPUDepthInformation[];
     gpuDepthData: XRWebGLDepthInformation[];
@@ -25,6 +24,9 @@ export declare class Depth {
     private occlusionPass?;
     private depthClientsInitialized;
     private depthClients;
+    depthProjectionMatrices: THREE.Matrix4[];
+    depthViewMatrices: THREE.Matrix4[];
+    depthViewProjectionMatrices: THREE.Matrix4[];
     /**
      * Depth is a lightweight manager based on three.js to simply prototyping
      * with Depth in WebXR.
@@ -54,9 +56,10 @@ export declare class Depth {
      * @returns Vertex at (u, v)
      */
     getVertex(u: number, v: number): THREE.Vector3 | null;
-    updateCPUDepthData(depthData: XRCPUDepthInformation, view_id?: number): void;
-    updateGPUDepthData(depthData: XRWebGLDepthInformation, view_id?: number): void;
-    getTexture(view_id: number): THREE.DataTexture | THREE.ExternalTexture | undefined;
+    private updateDepthMatrices;
+    updateCPUDepthData(depthData: XRCPUDepthInformation, viewId?: number): void;
+    updateGPUDepthData(depthData: XRWebGLDepthInformation, viewId?: number): void;
+    getTexture(viewId: number): THREE.DataTexture | THREE.ExternalTexture | undefined;
     update(frame?: XRFrame): void;
     updateLocalDepth(frame: XRFrame): void;
     renderOcclusionPass(): void;

package/build/simulator/SimulatorDepth.d.ts

@@ -4,15 +4,25 @@ import { SimulatorDepthMaterial } from './SimulatorDepthMaterial';
 import { SimulatorScene } from './SimulatorScene';
 export declare class SimulatorDepth {
     private simulatorScene;
-    renderer: THREE.WebGLRenderer;
-    camera: THREE.Camera;
-    depth: Depth;
+    private renderer;
+    private camera;
+    private depth;
     depthWidth: number;
     depthHeight: number;
     depthBufferSlice: Float32Array<ArrayBuffer>;
     depthMaterial: SimulatorDepthMaterial;
     depthRenderTarget: THREE.WebGLRenderTarget;
     depthBuffer: Float32Array;
+    depthCamera: THREE.Camera;
+    /**
+     * If true, copies the rendering camera's projection matrix each frame.
+     */
+    autoUpdateDepthCameraProjection: boolean;
+    /**
+     * If true, copies the rendering camera's transform each frame.
+     */
+    autoUpdateDepthCameraTransform: boolean;
+    private projectionMatrixArray;
     constructor(simulatorScene: SimulatorScene);
     /**
      * Initialize Simulator Depth.
@@ -20,6 +30,7 @@ export declare class SimulatorDepth {
     init(renderer: THREE.WebGLRenderer, camera: THREE.Camera, depth: Depth): void;
     createRenderTarget(): void;
     update(): void;
-    renderDepthScene(): void;
-    updateDepth(): void;
+    private updateDepthCamera;
+    private renderDepthScene;
+    private updateDepth;
 }

package/build/ui/components/TextButton.d.ts

@@ -9,6 +9,8 @@ export type TextButtonOptions = TextViewOptions & {
     maxWidth?: number;
     radius?: number;
     boxSize?: number;
+    hoverColor?: string | number;
+    selectedFontColor?: string | number;
 };
 export declare class TextButton extends TextView {
     /** Default description of this view in Three.js DevTools. */

package/build/ui/components/TextView.d.ts

@@ -140,7 +140,7 @@ export declare class TextView extends View<TextViewEventMap> {
      */
     protected _initializeText(): void;
     protected syncTextObj(): void;
-    protected setTextColor(color: number): void;
+    protected setTextColor(color: number | string): void;
     /**
      * Disposes of resources used by the TextView, such as event listeners.
      */

package/build/world/objects/ObjectDetector.d.ts

@@ -61,7 +61,7 @@ export declare class ObjectDetector extends Script {
      * Retrieves a list of currently detected objects.
      *
      * @param label - The semantic label to filter by (e.g., 'chair'). If null,
-     *     all objects are returned.
+     * all objects are returned.
      * @returns An array of `Object` instances.
      */
     get(label?: null): DetectedObject[];
@@ -79,10 +79,15 @@ export declare class ObjectDetector extends Script {
      * Draws the detected bounding boxes on the input image and triggers a
      * download for debugging.
      * @param base64Image - The base64 encoded input image.
-     * @param detections - The array of detected objects from the
-     * AI response.
+     * @param detections - The array of detected objects from the AI response.
      */
     private _visualizeBoundingBoxesOnImage;
+    /**
+     * Generates a visual representation of the depth map, normalized to 0-1 range,
+     * and triggers a download for debugging.
+     * @param depthArray - The raw depth data array.
+     */
+    private _visualizeDepthMap;
     /**
      * Creates a simple debug visualization for an object based on its position
      * (center of its 2D detection bounding box).