xrblocks 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,15 +1,46 @@
1
1
  import { DeepPartial, DeepReadonly } from '../utils/Types';
2
+ /**
3
+ * Parameters for RGB to depth UV mapping given different aspect ratios.
4
+ * These parameters define the distortion model and affine transformations
5
+ * required to align the RGB camera feed with the depth map.
6
+ */
7
+ export interface RgbToDepthParams {
8
+ scale: number;
9
+ scaleX: number;
10
+ scaleY: number;
11
+ translateU: number;
12
+ translateV: number;
13
+ k1: number;
14
+ k2: number;
15
+ k3: number;
16
+ p1: number;
17
+ p2: number;
18
+ xc: number;
19
+ yc: number;
20
+ }
21
+ /**
22
+ * Default parameters for rgb to depth projection.
23
+ * For RGB and depth, 4:3 and 1:1, respectively.
24
+ */
25
+ export declare const DEFAULT_RGB_TO_DEPTH_PARAMS: RgbToDepthParams;
26
+ /**
27
+ * Configuration options for the device camera.
28
+ */
2
29
  export declare class DeviceCameraOptions {
3
30
  enabled: boolean;
4
31
  /**
5
32
  * Constraints for `getUserMedia`. This will guide the initial camera
6
- * selection. *
33
+ * selection.
7
34
  */
8
35
  videoConstraints?: MediaTrackConstraints;
9
36
  /**
10
37
  * Hint for performance optimization on frequent captures.
11
38
  */
12
39
  willCaptureFrequently: boolean;
40
+ /**
41
+ * Parameters for RGB to depth UV mapping given different aspect ratios.
42
+ */
43
+ rgbToDepthParams: RgbToDepthParams;
13
44
  constructor(options?: DeepReadonly<DeepPartial<DeviceCameraOptions>>);
14
45
  }
15
46
  export declare const xrDeviceCameraEnvironmentOptions: {
@@ -167,6 +198,20 @@ export declare const xrDeviceCameraEnvironmentOptions: {
167
198
  } | undefined;
168
199
  } | undefined;
169
200
  readonly willCaptureFrequently: boolean;
201
+ readonly rgbToDepthParams: {
202
+ readonly scale: number;
203
+ readonly scaleX: number;
204
+ readonly scaleY: number;
205
+ readonly translateU: number;
206
+ readonly translateV: number;
207
+ readonly k1: number;
208
+ readonly k2: number;
209
+ readonly k3: number;
210
+ readonly p1: number;
211
+ readonly p2: number;
212
+ readonly xc: number;
213
+ readonly yc: number;
214
+ };
170
215
  };
171
216
  export declare const xrDeviceCameraUserOptions: {
172
217
  readonly enabled: boolean;
@@ -323,6 +368,20 @@ export declare const xrDeviceCameraUserOptions: {
323
368
  } | undefined;
324
369
  } | undefined;
325
370
  readonly willCaptureFrequently: boolean;
371
+ readonly rgbToDepthParams: {
372
+ readonly scale: number;
373
+ readonly scaleX: number;
374
+ readonly scaleY: number;
375
+ readonly translateU: number;
376
+ readonly translateV: number;
377
+ readonly k1: number;
378
+ readonly k2: number;
379
+ readonly k3: number;
380
+ readonly p1: number;
381
+ readonly p2: number;
382
+ readonly xc: number;
383
+ readonly yc: number;
384
+ };
326
385
  };
327
386
  export declare const xrDeviceCameraEnvironmentContinuousOptions: {
328
387
  readonly enabled: boolean;
@@ -479,6 +538,20 @@ export declare const xrDeviceCameraEnvironmentContinuousOptions: {
479
538
  } | undefined;
480
539
  } | undefined;
481
540
  readonly willCaptureFrequently: boolean;
541
+ readonly rgbToDepthParams: {
542
+ readonly scale: number;
543
+ readonly scaleX: number;
544
+ readonly scaleY: number;
545
+ readonly translateU: number;
546
+ readonly translateV: number;
547
+ readonly k1: number;
548
+ readonly k2: number;
549
+ readonly k3: number;
550
+ readonly p1: number;
551
+ readonly p2: number;
552
+ readonly xc: number;
553
+ readonly yc: number;
554
+ };
482
555
  };
483
556
  export declare const xrDeviceCameraUserContinuousOptions: {
484
557
  readonly enabled: boolean;
@@ -635,4 +708,18 @@ export declare const xrDeviceCameraUserContinuousOptions: {
635
708
  } | undefined;
636
709
  } | undefined;
637
710
  readonly willCaptureFrequently: boolean;
711
+ readonly rgbToDepthParams: {
712
+ readonly scale: number;
713
+ readonly scaleX: number;
714
+ readonly scaleY: number;
715
+ readonly translateU: number;
716
+ readonly translateV: number;
717
+ readonly k1: number;
718
+ readonly k2: number;
719
+ readonly k3: number;
720
+ readonly p1: number;
721
+ readonly p2: number;
722
+ readonly xc: number;
723
+ readonly yc: number;
724
+ };
638
725
  };
@@ -5,24 +5,6 @@ export declare const aspectRatios: {
5
5
  depth: number;
6
6
  RGB: number;
7
7
  };
8
- /**
9
- * Parameters for RGB to depth UV mapping (manually calibrated for aspect
10
- * ratios. For RGB and depth, 4:3 and 1:1, respectively.
11
- */
12
- export declare const rgbToDepthParams: {
13
- scale: number;
14
- scaleX: number;
15
- scaleY: number;
16
- translateU: number;
17
- translateV: number;
18
- k1: number;
19
- k2: number;
20
- k3: number;
21
- p1: number;
22
- p2: number;
23
- xc: number;
24
- yc: number;
25
- };
26
8
  /**
27
9
  * Maps a UV coordinate from a RGB space to a destination depth space,
28
10
  * applying Brown-Conrady distortion and affine transformations based on
@@ -1,7 +1,7 @@
1
1
  import { SimulatorCamera } from '../simulator/SimulatorCamera';
2
2
  import { SimulatorMediaDeviceInfo } from '../simulator/SimulatorMediaDeviceInfo';
3
3
  import { VideoStream, VideoStreamDetails } from '../video/VideoStream';
4
- import { DeviceCameraOptions } from './CameraOptions';
4
+ import { DeviceCameraOptions, RgbToDepthParams } from './CameraOptions';
5
5
  export type MediaOrSimulatorMediaDeviceInfo = MediaDeviceInfo | SimulatorMediaDeviceInfo;
6
6
  type XRDeviceCameraDetails = VideoStreamDetails & {
7
7
  width?: number;
@@ -15,6 +15,7 @@ type XRDeviceCameraDetails = VideoStreamDetails & {
15
15
  */
16
16
  export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
17
17
  simulatorCamera?: SimulatorCamera;
18
+ rgbToDepthParams: RgbToDepthParams;
18
19
  protected videoConstraints_: MediaTrackConstraints;
19
20
  private isInitializing_;
20
21
  private availableDevices_;
@@ -23,7 +24,7 @@ export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
23
24
  /**
24
25
  * @param options - The configuration options.
25
26
  */
26
- constructor({ videoConstraints, willCaptureFrequently, }?: Partial<DeviceCameraOptions>);
27
+ constructor({ videoConstraints, willCaptureFrequently, rgbToDepthParams, }?: Partial<DeviceCameraOptions>);
27
28
  /**
28
29
  * Retrieves the list of available video input devices.
29
30
  * @returns A promise that resolves with an
@@ -43,7 +44,7 @@ export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
43
44
  /**
44
45
  * Sets the active camera by its device ID. Removes potentially conflicting
45
46
  * constraints such as facingMode.
46
- * @param deviceId - Device id.
47
+ * @param deviceId - Device ID
47
48
  */
48
49
  setDeviceId(deviceId: string): Promise<void>;
49
50
  /**
@@ -9,6 +9,8 @@ export type TextButtonOptions = TextViewOptions & {
9
9
  maxWidth?: number;
10
10
  radius?: number;
11
11
  boxSize?: number;
12
+ hoverColor?: string | number;
13
+ selectedFontColor?: string | number;
12
14
  };
13
15
  export declare class TextButton extends TextView {
14
16
  /** Default description of this view in Three.js DevTools. */
@@ -140,7 +140,7 @@ export declare class TextView extends View<TextViewEventMap> {
140
140
  */
141
141
  protected _initializeText(): void;
142
142
  protected syncTextObj(): void;
143
- protected setTextColor(color: number): void;
143
+ protected setTextColor(color: number | string): void;
144
144
  /**
145
145
  * Disposes of resources used by the TextView, such as event listeners.
146
146
  */
@@ -61,7 +61,7 @@ export declare class ObjectDetector extends Script {
61
61
  * Retrieves a list of currently detected objects.
62
62
  *
63
63
  * @param label - The semantic label to filter by (e.g., 'chair'). If null,
64
- * all objects are returned.
64
+ * all objects are returned.
65
65
  * @returns An array of `Object` instances.
66
66
  */
67
67
  get(label?: null): DetectedObject[];
@@ -79,10 +79,15 @@ export declare class ObjectDetector extends Script {
79
79
  * Draws the detected bounding boxes on the input image and triggers a
80
80
  * download for debugging.
81
81
  * @param base64Image - The base64 encoded input image.
82
- * @param detections - The array of detected objects from the
83
- * AI response.
82
+ * @param detections - The array of detected objects from the AI response.
84
83
  */
85
84
  private _visualizeBoundingBoxesOnImage;
85
+ /**
86
+ * Generates a visual representation of the depth map, normalized to 0-1 range,
87
+ * and triggers a download for debugging.
88
+ * @param depthArray - The raw depth data array.
89
+ */
90
+ private _visualizeDepthMap;
86
91
  /**
87
92
  * Creates a simple debug visualization for an object based on its position
88
93
  * (center of its 2D detection bounding box).
package/build/xrblocks.js CHANGED
@@ -14,9 +14,9 @@
14
14
  * limitations under the License.
15
15
  *
16
16
  * @file xrblocks.js
17
- * @version v0.5.0
18
- * @commitid c2f4b09
19
- * @builddate 2025-12-04T15:14:30.184Z
17
+ * @version v0.5.1
18
+ * @commitid 896d66f
19
+ * @builddate 2025-12-06T06:19:44.629Z
20
20
  * @description XR Blocks SDK, built from source with the above commit ID.
21
21
  * @agent When using with Gemini to create XR apps, use **Gemini Canvas** mode,
22
22
  * and follow rules below:
@@ -1849,6 +1849,27 @@ function deepMerge(obj1, obj2) {
1849
1849
  }
1850
1850
  }
1851
1851
 
1852
+ /**
1853
+ * Default parameters for rgb to depth projection.
1854
+ * For RGB and depth, 4:3 and 1:1, respectively.
1855
+ */
1856
+ const DEFAULT_RGB_TO_DEPTH_PARAMS = {
1857
+ scale: 1,
1858
+ scaleX: 0.75,
1859
+ scaleY: 0.63,
1860
+ translateU: 0.2,
1861
+ translateV: -0.02,
1862
+ k1: -0.046,
1863
+ k2: 0,
1864
+ k3: 0,
1865
+ p1: 0,
1866
+ p2: 0,
1867
+ xc: 0,
1868
+ yc: 0,
1869
+ };
1870
+ /**
1871
+ * Configuration options for the device camera.
1872
+ */
1852
1873
  class DeviceCameraOptions {
1853
1874
  constructor(options) {
1854
1875
  this.enabled = false;
@@ -1856,10 +1877,14 @@ class DeviceCameraOptions {
1856
1877
  * Hint for performance optimization on frequent captures.
1857
1878
  */
1858
1879
  this.willCaptureFrequently = false;
1880
+ /**
1881
+ * Parameters for RGB to depth UV mapping given different aspect ratios.
1882
+ */
1883
+ this.rgbToDepthParams = { ...DEFAULT_RGB_TO_DEPTH_PARAMS };
1859
1884
  deepMerge(this, options);
1860
1885
  }
1861
1886
  }
1862
- // Base configuration for all common capture settings
1887
+ // Base configuration for all common capture settings.
1863
1888
  const baseCaptureOptions = {
1864
1889
  enabled: true,
1865
1890
  videoConstraints: {
@@ -3316,24 +3341,6 @@ const aspectRatios = {
3316
3341
  depth: 1.0,
3317
3342
  RGB: 4 / 3,
3318
3343
  };
3319
- /**
3320
- * Parameters for RGB to depth UV mapping (manually calibrated for aspect
3321
- * ratios. For RGB and depth, 4:3 and 1:1, respectively.
3322
- */
3323
- const rgbToDepthParams = {
3324
- scale: 1,
3325
- scaleX: 0.75,
3326
- scaleY: 0.63,
3327
- translateU: 0.2,
3328
- translateV: -0.02,
3329
- k1: -0.046,
3330
- k2: 0,
3331
- k3: 0,
3332
- p1: 0,
3333
- p2: 0,
3334
- xc: 0,
3335
- yc: 0,
3336
- };
3337
3344
  /**
3338
3345
  * Maps a UV coordinate from a RGB space to a destination depth space,
3339
3346
  * applying Brown-Conrady distortion and affine transformations based on
@@ -3368,42 +3375,40 @@ function transformRgbToDepthUv(rgbUv, xrDeviceCamera) {
3368
3375
  console.error('Invalid aspect ratios provided.');
3369
3376
  return null;
3370
3377
  }
3371
- // Determine the relative scaling required to fit the overlay within the base
3378
+ const params = xrDeviceCamera?.rgbToDepthParams ?? DEFAULT_RGB_TO_DEPTH_PARAMS;
3379
+ // Determine the relative scaling required to fit the overlay within the base.
3372
3380
  let relativeScaleX, relativeScaleY;
3373
3381
  if (aspectRatios.depth > aspectRatios.RGB) {
3374
- // Base is wider than overlay ("letterboxing")
3382
+ // Base is wider than overlay ("letterboxing").
3375
3383
  relativeScaleY = 1.0;
3376
3384
  relativeScaleX = aspectRatios.RGB / aspectRatios.depth;
3377
3385
  }
3378
3386
  else {
3379
- // Base is narrower than overlay ("pillarboxing")
3387
+ // Base is narrower than overlay ("pillarboxing").
3380
3388
  relativeScaleX = 1.0;
3381
3389
  relativeScaleY = aspectRatios.depth / aspectRatios.RGB;
3382
3390
  }
3383
- // Convert input source UV [0, 1] to a normalized coordinate space [-0.5, 0.5]
3391
+ // Convert input source UV [0, 1] to normalized coordinates in [-0.5, 0.5].
3384
3392
  const u_norm = rgbUv.u - 0.5;
3385
3393
  const v_norm = rgbUv.v - 0.5;
3386
- // Apply the FORWARD Brown-Conrady distortion model
3387
- const u_centered = u_norm - rgbToDepthParams.xc;
3388
- const v_centered = v_norm - rgbToDepthParams.yc;
3394
+ // Apply the FORWARD Brown-Conrady distortion model.
3395
+ const u_centered = u_norm - params.xc;
3396
+ const v_centered = v_norm - params.yc;
3389
3397
  const r2 = u_centered * u_centered + v_centered * v_centered;
3390
- const radial = 1 +
3391
- rgbToDepthParams.k1 * r2 +
3392
- rgbToDepthParams.k2 * r2 * r2 +
3393
- rgbToDepthParams.k3 * r2 * r2 * r2;
3394
- const tanX = 2 * rgbToDepthParams.p1 * u_centered * v_centered +
3395
- rgbToDepthParams.p2 * (r2 + 2 * u_centered * u_centered);
3396
- const tanY = rgbToDepthParams.p1 * (r2 + 2 * v_centered * v_centered) +
3397
- 2 * rgbToDepthParams.p2 * u_centered * v_centered;
3398
- const u_distorted = u_centered * radial + tanX + rgbToDepthParams.xc;
3399
- const v_distorted = v_centered * radial + tanY + rgbToDepthParams.yc;
3400
- // Apply initial aspect ratio scaling and translation
3401
- const u_fitted = u_distorted * relativeScaleX + rgbToDepthParams.translateU;
3402
- const v_fitted = v_distorted * relativeScaleY + rgbToDepthParams.translateV;
3403
- // Apply the final user-controlled scaling (zoom and stretch)
3404
- const finalNormX = u_fitted * rgbToDepthParams.scale * rgbToDepthParams.scaleX;
3405
- const finalNormY = v_fitted * rgbToDepthParams.scale * rgbToDepthParams.scaleY;
3406
- // Convert the final normalized coordinate back to a UV coordinate [0, 1]
3398
+ const radial = 1 + params.k1 * r2 + params.k2 * r2 * r2 + params.k3 * r2 * r2 * r2;
3399
+ const tanX = 2 * params.p1 * u_centered * v_centered +
3400
+ params.p2 * (r2 + 2 * u_centered * u_centered);
3401
+ const tanY = params.p1 * (r2 + 2 * v_centered * v_centered) +
3402
+ 2 * params.p2 * u_centered * v_centered;
3403
+ const u_distorted = u_centered * radial + tanX + params.xc;
3404
+ const v_distorted = v_centered * radial + tanY + params.yc;
3405
+ // Apply initial aspect ratio scaling and translation.
3406
+ const u_fitted = u_distorted * relativeScaleX + params.translateU;
3407
+ const v_fitted = v_distorted * relativeScaleY + params.translateV;
3408
+ // Apply the final user-controlled scaling (zoom and stretch).
3409
+ const finalNormX = u_fitted * params.scale * params.scaleX;
3410
+ const finalNormY = v_fitted * params.scale * params.scaleY;
3411
+ // Convert the final normalized coordinate back to a UV coordinate [0, 1].
3407
3412
  const finalU = finalNormX + 0.5;
3408
3413
  const finalV = finalNormY + 0.5;
3409
3414
  return { u: finalU, v: 1.0 - finalV };
@@ -3667,12 +3672,13 @@ class XRDeviceCamera extends VideoStream {
3667
3672
  /**
3668
3673
  * @param options - The configuration options.
3669
3674
  */
3670
- constructor({ videoConstraints = { facingMode: 'environment' }, willCaptureFrequently = false, } = {}) {
3675
+ constructor({ videoConstraints = { facingMode: 'environment' }, willCaptureFrequently = false, rgbToDepthParams = DEFAULT_RGB_TO_DEPTH_PARAMS, } = {}) {
3671
3676
  super({ willCaptureFrequently });
3672
3677
  this.isInitializing_ = false;
3673
3678
  this.availableDevices_ = [];
3674
3679
  this.currentDeviceIndex_ = -1;
3675
3680
  this.videoConstraints_ = { ...videoConstraints };
3681
+ this.rgbToDepthParams = rgbToDepthParams;
3676
3682
  }
3677
3683
  /**
3678
3684
  * Retrieves the list of available video input devices.
@@ -3724,7 +3730,7 @@ class XRDeviceCamera extends VideoStream {
3724
3730
  return;
3725
3731
  this.isInitializing_ = true;
3726
3732
  this.setState_(StreamState.INITIALIZING);
3727
- // Reset state for the new stream
3733
+ // Reset state for the new stream.
3728
3734
  this.currentTrackSettings_ = undefined;
3729
3735
  this.currentDeviceIndex_ = -1;
3730
3736
  try {
@@ -3758,7 +3764,7 @@ class XRDeviceCamera extends VideoStream {
3758
3764
  if (!videoTracks.length) {
3759
3765
  throw new Error('MediaStream has no video tracks.');
3760
3766
  }
3761
- // After the stream is active, we can get the ID of the track
3767
+ // After the stream is active, we can get the track ID.
3762
3768
  const activeTrack = videoTracks[0];
3763
3769
  this.currentTrackSettings_ = activeTrack.getSettings();
3764
3770
  console.debug('Active track settings:', this.currentTrackSettings_);
@@ -3768,10 +3774,10 @@ class XRDeviceCamera extends VideoStream {
3768
3774
  else {
3769
3775
  console.warn('Stream started without deviceId as it was unavailable');
3770
3776
  }
3771
- this.stop_(); // Stop any previous stream before starting new one
3777
+ this.stop_(); // Stop any previous stream before starting new one.
3772
3778
  this.stream_ = stream;
3773
3779
  this.video_.srcObject = stream;
3774
- this.video_.src = ''; // Required for some browsers to reset the src
3780
+ this.video_.src = ''; // Required for some browsers to reset the src.
3775
3781
  await new Promise((resolve, reject) => {
3776
3782
  this.video_.onloadedmetadata = () => {
3777
3783
  this.handleVideoStreamLoadedMetadata(resolve, reject, true);
@@ -3783,7 +3789,7 @@ class XRDeviceCamera extends VideoStream {
3783
3789
  };
3784
3790
  this.video_.play();
3785
3791
  });
3786
- // Once the stream is loaded and dimensions are known, set the final state
3792
+ // Once stream is loaded and dimensions are known, set the final state.
3787
3793
  const details = {
3788
3794
  width: this.width,
3789
3795
  height: this.height,
@@ -3805,7 +3811,7 @@ class XRDeviceCamera extends VideoStream {
3805
3811
  /**
3806
3812
  * Sets the active camera by its device ID. Removes potentially conflicting
3807
3813
  * constraints such as facingMode.
3808
- * @param deviceId - Device id.
3814
+ * @param deviceId - Device ID
3809
3815
  */
3810
3816
  async setDeviceId(deviceId) {
3811
3817
  const newIndex = this.availableDevices_.findIndex((device) => device.deviceId === deviceId);
@@ -11796,7 +11802,7 @@ class TextView extends View {
11796
11802
  }
11797
11803
  setTextColor(color) {
11798
11804
  if (Text && this.textObj instanceof Text) {
11799
- this.textObj.color = color;
11805
+ this.textObj.color = getColorHex(color);
11800
11806
  }
11801
11807
  }
11802
11808
  /**
@@ -12186,7 +12192,7 @@ class TextButton extends TextView {
12186
12192
  */
12187
12193
  constructor(options = {}) {
12188
12194
  const geometry = new THREE.PlaneGeometry(1, 1);
12189
- const colorVec4 = getVec4ByColorString(options.backgroundColor ?? '#00000000');
12195
+ const colorVec4 = getVec4ByColorString(options.backgroundColor ?? '#000000');
12190
12196
  const { opacity = 0.0, radius = SquircleShader.uniforms.uRadius.value, boxSize = SquircleShader.uniforms.uBoxSize.value, } = options;
12191
12197
  const uniforms = {
12192
12198
  ...SquircleShader.uniforms,
@@ -12236,6 +12242,9 @@ class TextButton extends TextView {
12236
12242
  // Applies our own overrides to the default values.
12237
12243
  this.fontSize = options.fontSize ?? this.fontSize;
12238
12244
  this.fontColor = options.fontColor ?? this.fontColor;
12245
+ this.hoverColor = options.hoverColor ?? this.hoverColor;
12246
+ this.selectedFontColor =
12247
+ options.selectedFontColor ?? this.selectedFontColor;
12239
12248
  this.width = options.width ?? this.width;
12240
12249
  this.height = options.height ?? this.height;
12241
12250
  }
@@ -12258,20 +12267,19 @@ class TextButton extends TextView {
12258
12267
  if (!this.textObj) {
12259
12268
  return;
12260
12269
  }
12261
- if (this.textObj) {
12262
- this.textObj.renderOrder = this.renderOrder + 1;
12263
- }
12270
+ // Update render order to ensure text appears on top of the button mesh
12271
+ this.textObj.renderOrder = this.renderOrder + 1;
12264
12272
  const ux = this.ux;
12265
12273
  if (ux.isHovered()) {
12266
12274
  if (ux.isSelected()) {
12267
- this.setTextColor(0x666666);
12275
+ this.setTextColor(this.selectedFontColor);
12268
12276
  }
12269
12277
  else {
12270
- this.setTextColor(0xaaaaaa);
12278
+ this.setTextColor(this.hoverColor);
12271
12279
  }
12272
12280
  }
12273
12281
  else {
12274
- this.setTextColor(0xffffff);
12282
+ this.setTextColor(this.fontColor);
12275
12283
  this.uniforms.uOpacity.value = this.defaultOpacity * this.opacity;
12276
12284
  }
12277
12285
  }
@@ -13887,6 +13895,7 @@ class ObjectDetector extends Script {
13887
13895
  }
13888
13896
  if (this.options.objects.showDebugVisualizations) {
13889
13897
  this._visualizeBoundingBoxesOnImage(base64Image, parsedResponse);
13898
+ this._visualizeDepthMap(cachedDepthArray);
13890
13899
  }
13891
13900
  const detectionPromises = parsedResponse.map(async (item) => {
13892
13901
  const { ymin, xmin, ymax, xmax, objectName, ...additionalData } = item || {};
@@ -13935,7 +13944,7 @@ class ObjectDetector extends Script {
13935
13944
  * Retrieves a list of currently detected objects.
13936
13945
  *
13937
13946
  * @param label - The semantic label to filter by (e.g., 'chair'). If null,
13938
- * all objects are returned.
13947
+ * all objects are returned.
13939
13948
  * @returns An array of `Object` instances.
13940
13949
  */
13941
13950
  get(label = null) {
@@ -13972,8 +13981,7 @@ class ObjectDetector extends Script {
13972
13981
  * Draws the detected bounding boxes on the input image and triggers a
13973
13982
  * download for debugging.
13974
13983
  * @param base64Image - The base64 encoded input image.
13975
- * @param detections - The array of detected objects from the
13976
- * AI response.
13984
+ * @param detections - The array of detected objects from the AI response.
13977
13985
  */
13978
13986
  _visualizeBoundingBoxesOnImage(base64Image, detections) {
13979
13987
  const img = new Image();
@@ -14022,6 +14030,71 @@ class ObjectDetector extends Script {
14022
14030
  };
14023
14031
  img.src = base64Image;
14024
14032
  }
14033
+ /**
14034
+ * Generates a visual representation of the depth map, normalized to 0-1 range,
14035
+ * and triggers a download for debugging.
14036
+ * @param depthArray - The raw depth data array.
14037
+ */
14038
+ _visualizeDepthMap(depthArray) {
14039
+ const width = this.depth.width;
14040
+ const height = this.depth.height;
14041
+ if (!width || !height || depthArray.length === 0) {
14042
+ console.warn('Cannot visualize depth map: missing dimensions or data.');
14043
+ return;
14044
+ }
14045
+ // 1. Find Min/Max for normalization (ignoring 0/invalid depth).
14046
+ let min = Infinity;
14047
+ let max = -Infinity;
14048
+ for (let i = 0; i < depthArray.length; ++i) {
14049
+ const val = depthArray[i];
14050
+ if (val > 0) {
14051
+ if (val < min)
14052
+ min = val;
14053
+ if (val > max)
14054
+ max = val;
14055
+ }
14056
+ }
14057
+ // Handle edge case where no valid depth exists.
14058
+ if (min === Infinity) {
14059
+ min = 0;
14060
+ max = 1;
14061
+ }
14062
+ if (min === max) {
14063
+ max = min + 1; // Avoid divide by zero
14064
+ }
14065
+ // 2. Create Canvas.
14066
+ const canvas = document.createElement('canvas');
14067
+ canvas.width = width;
14068
+ canvas.height = height;
14069
+ const ctx = canvas.getContext('2d');
14070
+ const imageData = ctx.createImageData(width, height);
14071
+ const data = imageData.data;
14072
+ // 3. Fill Pixels.
14073
+ for (let i = 0; i < depthArray.length; ++i) {
14074
+ const raw = depthArray[i];
14075
+ // Normalize to 0-1.
14076
+ // Typically 0 means invalid/sky in some depth APIs, so we keep it black.
14077
+ // Otherwise, map [min, max] to [0, 1].
14078
+ const normalized = raw === 0 ? 0 : (raw - min) / (max - min);
14079
+ const byteVal = Math.floor(normalized * 255);
14080
+ const stride = i * 4;
14081
+ data[stride] = byteVal; // R
14082
+ data[stride + 1] = byteVal; // G
14083
+ data[stride + 2] = byteVal; // B
14084
+ data[stride + 3] = 255; // Alpha
14085
+ }
14086
+ ctx.putImageData(imageData, 0, 0);
14087
+ // 4. Download.
14088
+ const timestamp = new Date()
14089
+ .toISOString()
14090
+ .slice(0, 19)
14091
+ .replace('T', '_')
14092
+ .replace(/:/g, '-');
14093
+ const link = document.createElement('a');
14094
+ link.download = `depth_debug_${timestamp}.png`;
14095
+ link.href = canvas.toDataURL('image/png');
14096
+ link.click();
14097
+ }
14025
14098
  /**
14026
14099
  * Creates a simple debug visualization for an object based on its position
14027
14100
  * (center of its 2D detection bounding box).
@@ -17086,5 +17159,5 @@ class VideoFileStream extends VideoStream {
17086
17159
  }
17087
17160
  }
17088
17161
 
17089
- export { AI, AIOptions, AVERAGE_IPD_METERS, ActiveControllers, Agent, AnimatableNumber, AudioListener, AudioPlayer, BACK, BackgroundMusic, CategoryVolumes, Col, Core, CoreSound, DEFAULT_DEVICE_CAMERA_HEIGHT, DEFAULT_DEVICE_CAMERA_WIDTH, DOWN, Depth, DepthMesh, DepthMeshOptions, DepthOptions, DepthTextures, DetectedObject, DetectedPlane, DeviceCameraOptions, DragManager, DragMode, ExitButton, FORWARD, FreestandingSlider, GazeController, Gemini, GeminiOptions, GenerateSkyboxTool, GestureRecognition, GestureRecognitionOptions, GetWeatherTool, Grid, HAND_BONE_IDX_CONNECTION_MAP, HAND_JOINT_COUNT, HAND_JOINT_IDX_CONNECTION_MAP, HAND_JOINT_NAMES, Handedness, Hands, HandsOptions, HorizontalPager, IconButton, IconView, ImageView, Input, InputOptions, Keycodes, LEFT, LEFT_VIEW_ONLY_LAYER, LabelView, Lighting, LightingOptions, LoadingSpinnerManager, MaterialSymbolsView, MeshScript, ModelLoader, ModelViewer, MouseController, NEXT_SIMULATOR_MODE, NUM_HANDS, OCCLUDABLE_ITEMS_LAYER, ObjectDetector, ObjectsOptions, OcclusionPass, OcclusionUtils, OpenAI, OpenAIOptions, Options, PageIndicator, Pager, PagerState, Panel, PanelMesh, Physics, PhysicsOptions, PinchOnButtonAction, PlaneDetector, PlanesOptions, RIGHT, RIGHT_VIEW_ONLY_LAYER, Registry, Reticle, ReticleOptions, RotationRaycastMesh, Row, SIMULATOR_HAND_POSE_NAMES, SIMULATOR_HAND_POSE_TO_JOINTS_LEFT, SIMULATOR_HAND_POSE_TO_JOINTS_RIGHT, SOUND_PRESETS, ScreenshotSynthesizer, Script, ScriptMixin, ScriptsManager, ScrollingTroikaTextView, SetSimulatorModeEvent, ShowHandsAction, Simulator, SimulatorCamera, SimulatorControlMode, SimulatorControllerState, SimulatorControls, SimulatorDepth, SimulatorDepthMaterial, SimulatorHandPose, SimulatorHandPoseChangeRequestEvent, SimulatorHands, SimulatorInterface, SimulatorMediaDeviceInfo, SimulatorMode, SimulatorOptions, SimulatorRenderMode, SimulatorScene, SimulatorUser, SimulatorUserAction, SketchPanel, SkyboxAgent, SoundOptions, SoundSynthesizer, SpatialAudio, SpatialPanel, SpeechRecognizer, SpeechRecognizerOptions, SpeechSynthesizer, SpeechSynthesizerOptions, SplatAnchor, StreamState, TextButton, TextScrollerState, TextView, Tool, UI, UI_OVERLAY_LAYER, UP, UX, User, VIEW_DEPTH_GAP, VerticalPager, VideoFileStream, VideoStream, VideoView, View, VolumeCategory, WaitFrame, WalkTowardsPanelAction, World, WorldOptions, XRButton, XRDeviceCamera, XREffects, XRPass, XRTransitionOptions, XR_BLOCKS_ASSETS_PATH, ZERO_VECTOR3, add, ai, aspectRatios, callInitWithDependencyInjection, clamp, clampRotationToAngle, core, cropImage, extractYaw, getColorHex, getDeltaTime, getUrlParamBool, getUrlParamFloat, getUrlParamInt, getUrlParameter, getVec4ByColorString, getXrCameraLeft, getXrCameraRight, init, initScript, lerp, loadStereoImageAsTextures, loadingSpinnerManager, lookAtRotation, objectIsDescendantOf, parseBase64DataURL, placeObjectAtIntersectionFacingTarget, print, rgbToDepthParams, scene, showOnlyInLeftEye, showOnlyInRightEye, showReticleOnDepthMesh, transformRgbToDepthUv, transformRgbUvToWorld, traverseUtil, uninitScript, urlParams, user, world, xrDepthMeshOptions, xrDepthMeshPhysicsOptions, xrDepthMeshVisualizationOptions, xrDeviceCameraEnvironmentContinuousOptions, xrDeviceCameraEnvironmentOptions, xrDeviceCameraUserContinuousOptions, xrDeviceCameraUserOptions };
17162
+ export { AI, AIOptions, AVERAGE_IPD_METERS, ActiveControllers, Agent, AnimatableNumber, AudioListener, AudioPlayer, BACK, BackgroundMusic, CategoryVolumes, Col, Core, CoreSound, DEFAULT_DEVICE_CAMERA_HEIGHT, DEFAULT_DEVICE_CAMERA_WIDTH, DEFAULT_RGB_TO_DEPTH_PARAMS, DOWN, Depth, DepthMesh, DepthMeshOptions, DepthOptions, DepthTextures, DetectedObject, DetectedPlane, DeviceCameraOptions, DragManager, DragMode, ExitButton, FORWARD, FreestandingSlider, GazeController, Gemini, GeminiOptions, GenerateSkyboxTool, GestureRecognition, GestureRecognitionOptions, GetWeatherTool, Grid, HAND_BONE_IDX_CONNECTION_MAP, HAND_JOINT_COUNT, HAND_JOINT_IDX_CONNECTION_MAP, HAND_JOINT_NAMES, Handedness, Hands, HandsOptions, HorizontalPager, IconButton, IconView, ImageView, Input, InputOptions, Keycodes, LEFT, LEFT_VIEW_ONLY_LAYER, LabelView, Lighting, LightingOptions, LoadingSpinnerManager, MaterialSymbolsView, MeshScript, ModelLoader, ModelViewer, MouseController, NEXT_SIMULATOR_MODE, NUM_HANDS, OCCLUDABLE_ITEMS_LAYER, ObjectDetector, ObjectsOptions, OcclusionPass, OcclusionUtils, OpenAI, OpenAIOptions, Options, PageIndicator, Pager, PagerState, Panel, PanelMesh, Physics, PhysicsOptions, PinchOnButtonAction, PlaneDetector, PlanesOptions, RIGHT, RIGHT_VIEW_ONLY_LAYER, Registry, Reticle, ReticleOptions, RotationRaycastMesh, Row, SIMULATOR_HAND_POSE_NAMES, SIMULATOR_HAND_POSE_TO_JOINTS_LEFT, SIMULATOR_HAND_POSE_TO_JOINTS_RIGHT, SOUND_PRESETS, ScreenshotSynthesizer, Script, ScriptMixin, ScriptsManager, ScrollingTroikaTextView, SetSimulatorModeEvent, ShowHandsAction, Simulator, SimulatorCamera, SimulatorControlMode, SimulatorControllerState, SimulatorControls, SimulatorDepth, SimulatorDepthMaterial, SimulatorHandPose, SimulatorHandPoseChangeRequestEvent, SimulatorHands, SimulatorInterface, SimulatorMediaDeviceInfo, SimulatorMode, SimulatorOptions, SimulatorRenderMode, SimulatorScene, SimulatorUser, SimulatorUserAction, SketchPanel, SkyboxAgent, SoundOptions, SoundSynthesizer, SpatialAudio, SpatialPanel, SpeechRecognizer, SpeechRecognizerOptions, SpeechSynthesizer, SpeechSynthesizerOptions, SplatAnchor, StreamState, TextButton, TextScrollerState, TextView, Tool, UI, UI_OVERLAY_LAYER, UP, UX, User, VIEW_DEPTH_GAP, VerticalPager, VideoFileStream, VideoStream, VideoView, View, VolumeCategory, WaitFrame, WalkTowardsPanelAction, World, WorldOptions, XRButton, XRDeviceCamera, XREffects, XRPass, XRTransitionOptions, XR_BLOCKS_ASSETS_PATH, ZERO_VECTOR3, add, ai, aspectRatios, callInitWithDependencyInjection, clamp, clampRotationToAngle, core, cropImage, extractYaw, getColorHex, getDeltaTime, getUrlParamBool, getUrlParamFloat, getUrlParamInt, getUrlParameter, getVec4ByColorString, getXrCameraLeft, getXrCameraRight, init, initScript, lerp, loadStereoImageAsTextures, loadingSpinnerManager, lookAtRotation, objectIsDescendantOf, parseBase64DataURL, placeObjectAtIntersectionFacingTarget, print, scene, showOnlyInLeftEye, showOnlyInRightEye, showReticleOnDepthMesh, transformRgbToDepthUv, transformRgbUvToWorld, traverseUtil, uninitScript, urlParams, user, world, xrDepthMeshOptions, xrDepthMeshPhysicsOptions, xrDepthMeshVisualizationOptions, xrDeviceCameraEnvironmentContinuousOptions, xrDeviceCameraEnvironmentOptions, xrDeviceCameraUserContinuousOptions, xrDeviceCameraUserOptions };
17090
17163
  //# sourceMappingURL=xrblocks.js.map