xrblocks 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/camera/CameraOptions.d.ts +88 -1
- package/build/camera/CameraUtils.d.ts +0 -18
- package/build/camera/XRDeviceCamera.d.ts +4 -3
- package/build/ui/components/TextButton.d.ts +2 -0
- package/build/ui/components/TextView.d.ts +1 -1
- package/build/world/objects/ObjectDetector.d.ts +8 -3
- package/build/xrblocks.js +138 -65
- package/build/xrblocks.js.map +1 -1
- package/build/xrblocks.min.js +1 -1
- package/build/xrblocks.min.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,15 +1,46 @@
|
|
|
1
1
|
import { DeepPartial, DeepReadonly } from '../utils/Types';
|
|
2
|
+
/**
|
|
3
|
+
* Parameters for RGB to depth UV mapping given different aspect ratios.
|
|
4
|
+
* These parameters define the distortion model and affine transformations
|
|
5
|
+
* required to align the RGB camera feed with the depth map.
|
|
6
|
+
*/
|
|
7
|
+
export interface RgbToDepthParams {
|
|
8
|
+
scale: number;
|
|
9
|
+
scaleX: number;
|
|
10
|
+
scaleY: number;
|
|
11
|
+
translateU: number;
|
|
12
|
+
translateV: number;
|
|
13
|
+
k1: number;
|
|
14
|
+
k2: number;
|
|
15
|
+
k3: number;
|
|
16
|
+
p1: number;
|
|
17
|
+
p2: number;
|
|
18
|
+
xc: number;
|
|
19
|
+
yc: number;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Default parameters for rgb to depth projection.
|
|
23
|
+
* For RGB and depth, 4:3 and 1:1, respectively.
|
|
24
|
+
*/
|
|
25
|
+
export declare const DEFAULT_RGB_TO_DEPTH_PARAMS: RgbToDepthParams;
|
|
26
|
+
/**
|
|
27
|
+
* Configuration options for the device camera.
|
|
28
|
+
*/
|
|
2
29
|
export declare class DeviceCameraOptions {
|
|
3
30
|
enabled: boolean;
|
|
4
31
|
/**
|
|
5
32
|
* Constraints for `getUserMedia`. This will guide the initial camera
|
|
6
|
-
* selection.
|
|
33
|
+
* selection.
|
|
7
34
|
*/
|
|
8
35
|
videoConstraints?: MediaTrackConstraints;
|
|
9
36
|
/**
|
|
10
37
|
* Hint for performance optimization on frequent captures.
|
|
11
38
|
*/
|
|
12
39
|
willCaptureFrequently: boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Parameters for RGB to depth UV mapping given different aspect ratios.
|
|
42
|
+
*/
|
|
43
|
+
rgbToDepthParams: RgbToDepthParams;
|
|
13
44
|
constructor(options?: DeepReadonly<DeepPartial<DeviceCameraOptions>>);
|
|
14
45
|
}
|
|
15
46
|
export declare const xrDeviceCameraEnvironmentOptions: {
|
|
@@ -167,6 +198,20 @@ export declare const xrDeviceCameraEnvironmentOptions: {
|
|
|
167
198
|
} | undefined;
|
|
168
199
|
} | undefined;
|
|
169
200
|
readonly willCaptureFrequently: boolean;
|
|
201
|
+
readonly rgbToDepthParams: {
|
|
202
|
+
readonly scale: number;
|
|
203
|
+
readonly scaleX: number;
|
|
204
|
+
readonly scaleY: number;
|
|
205
|
+
readonly translateU: number;
|
|
206
|
+
readonly translateV: number;
|
|
207
|
+
readonly k1: number;
|
|
208
|
+
readonly k2: number;
|
|
209
|
+
readonly k3: number;
|
|
210
|
+
readonly p1: number;
|
|
211
|
+
readonly p2: number;
|
|
212
|
+
readonly xc: number;
|
|
213
|
+
readonly yc: number;
|
|
214
|
+
};
|
|
170
215
|
};
|
|
171
216
|
export declare const xrDeviceCameraUserOptions: {
|
|
172
217
|
readonly enabled: boolean;
|
|
@@ -323,6 +368,20 @@ export declare const xrDeviceCameraUserOptions: {
|
|
|
323
368
|
} | undefined;
|
|
324
369
|
} | undefined;
|
|
325
370
|
readonly willCaptureFrequently: boolean;
|
|
371
|
+
readonly rgbToDepthParams: {
|
|
372
|
+
readonly scale: number;
|
|
373
|
+
readonly scaleX: number;
|
|
374
|
+
readonly scaleY: number;
|
|
375
|
+
readonly translateU: number;
|
|
376
|
+
readonly translateV: number;
|
|
377
|
+
readonly k1: number;
|
|
378
|
+
readonly k2: number;
|
|
379
|
+
readonly k3: number;
|
|
380
|
+
readonly p1: number;
|
|
381
|
+
readonly p2: number;
|
|
382
|
+
readonly xc: number;
|
|
383
|
+
readonly yc: number;
|
|
384
|
+
};
|
|
326
385
|
};
|
|
327
386
|
export declare const xrDeviceCameraEnvironmentContinuousOptions: {
|
|
328
387
|
readonly enabled: boolean;
|
|
@@ -479,6 +538,20 @@ export declare const xrDeviceCameraEnvironmentContinuousOptions: {
|
|
|
479
538
|
} | undefined;
|
|
480
539
|
} | undefined;
|
|
481
540
|
readonly willCaptureFrequently: boolean;
|
|
541
|
+
readonly rgbToDepthParams: {
|
|
542
|
+
readonly scale: number;
|
|
543
|
+
readonly scaleX: number;
|
|
544
|
+
readonly scaleY: number;
|
|
545
|
+
readonly translateU: number;
|
|
546
|
+
readonly translateV: number;
|
|
547
|
+
readonly k1: number;
|
|
548
|
+
readonly k2: number;
|
|
549
|
+
readonly k3: number;
|
|
550
|
+
readonly p1: number;
|
|
551
|
+
readonly p2: number;
|
|
552
|
+
readonly xc: number;
|
|
553
|
+
readonly yc: number;
|
|
554
|
+
};
|
|
482
555
|
};
|
|
483
556
|
export declare const xrDeviceCameraUserContinuousOptions: {
|
|
484
557
|
readonly enabled: boolean;
|
|
@@ -635,4 +708,18 @@ export declare const xrDeviceCameraUserContinuousOptions: {
|
|
|
635
708
|
} | undefined;
|
|
636
709
|
} | undefined;
|
|
637
710
|
readonly willCaptureFrequently: boolean;
|
|
711
|
+
readonly rgbToDepthParams: {
|
|
712
|
+
readonly scale: number;
|
|
713
|
+
readonly scaleX: number;
|
|
714
|
+
readonly scaleY: number;
|
|
715
|
+
readonly translateU: number;
|
|
716
|
+
readonly translateV: number;
|
|
717
|
+
readonly k1: number;
|
|
718
|
+
readonly k2: number;
|
|
719
|
+
readonly k3: number;
|
|
720
|
+
readonly p1: number;
|
|
721
|
+
readonly p2: number;
|
|
722
|
+
readonly xc: number;
|
|
723
|
+
readonly yc: number;
|
|
724
|
+
};
|
|
638
725
|
};
|
|
@@ -5,24 +5,6 @@ export declare const aspectRatios: {
|
|
|
5
5
|
depth: number;
|
|
6
6
|
RGB: number;
|
|
7
7
|
};
|
|
8
|
-
/**
|
|
9
|
-
* Parameters for RGB to depth UV mapping (manually calibrated for aspect
|
|
10
|
-
* ratios. For RGB and depth, 4:3 and 1:1, respectively.
|
|
11
|
-
*/
|
|
12
|
-
export declare const rgbToDepthParams: {
|
|
13
|
-
scale: number;
|
|
14
|
-
scaleX: number;
|
|
15
|
-
scaleY: number;
|
|
16
|
-
translateU: number;
|
|
17
|
-
translateV: number;
|
|
18
|
-
k1: number;
|
|
19
|
-
k2: number;
|
|
20
|
-
k3: number;
|
|
21
|
-
p1: number;
|
|
22
|
-
p2: number;
|
|
23
|
-
xc: number;
|
|
24
|
-
yc: number;
|
|
25
|
-
};
|
|
26
8
|
/**
|
|
27
9
|
* Maps a UV coordinate from a RGB space to a destination depth space,
|
|
28
10
|
* applying Brown-Conrady distortion and affine transformations based on
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { SimulatorCamera } from '../simulator/SimulatorCamera';
|
|
2
2
|
import { SimulatorMediaDeviceInfo } from '../simulator/SimulatorMediaDeviceInfo';
|
|
3
3
|
import { VideoStream, VideoStreamDetails } from '../video/VideoStream';
|
|
4
|
-
import { DeviceCameraOptions } from './CameraOptions';
|
|
4
|
+
import { DeviceCameraOptions, RgbToDepthParams } from './CameraOptions';
|
|
5
5
|
export type MediaOrSimulatorMediaDeviceInfo = MediaDeviceInfo | SimulatorMediaDeviceInfo;
|
|
6
6
|
type XRDeviceCameraDetails = VideoStreamDetails & {
|
|
7
7
|
width?: number;
|
|
@@ -15,6 +15,7 @@ type XRDeviceCameraDetails = VideoStreamDetails & {
|
|
|
15
15
|
*/
|
|
16
16
|
export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
|
|
17
17
|
simulatorCamera?: SimulatorCamera;
|
|
18
|
+
rgbToDepthParams: RgbToDepthParams;
|
|
18
19
|
protected videoConstraints_: MediaTrackConstraints;
|
|
19
20
|
private isInitializing_;
|
|
20
21
|
private availableDevices_;
|
|
@@ -23,7 +24,7 @@ export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
|
|
|
23
24
|
/**
|
|
24
25
|
* @param options - The configuration options.
|
|
25
26
|
*/
|
|
26
|
-
constructor({ videoConstraints, willCaptureFrequently, }?: Partial<DeviceCameraOptions>);
|
|
27
|
+
constructor({ videoConstraints, willCaptureFrequently, rgbToDepthParams, }?: Partial<DeviceCameraOptions>);
|
|
27
28
|
/**
|
|
28
29
|
* Retrieves the list of available video input devices.
|
|
29
30
|
* @returns A promise that resolves with an
|
|
@@ -43,7 +44,7 @@ export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
|
|
|
43
44
|
/**
|
|
44
45
|
* Sets the active camera by its device ID. Removes potentially conflicting
|
|
45
46
|
* constraints such as facingMode.
|
|
46
|
-
* @param deviceId - Device
|
|
47
|
+
* @param deviceId - Device ID
|
|
47
48
|
*/
|
|
48
49
|
setDeviceId(deviceId: string): Promise<void>;
|
|
49
50
|
/**
|
|
@@ -9,6 +9,8 @@ export type TextButtonOptions = TextViewOptions & {
|
|
|
9
9
|
maxWidth?: number;
|
|
10
10
|
radius?: number;
|
|
11
11
|
boxSize?: number;
|
|
12
|
+
hoverColor?: string | number;
|
|
13
|
+
selectedFontColor?: string | number;
|
|
12
14
|
};
|
|
13
15
|
export declare class TextButton extends TextView {
|
|
14
16
|
/** Default description of this view in Three.js DevTools. */
|
|
@@ -140,7 +140,7 @@ export declare class TextView extends View<TextViewEventMap> {
|
|
|
140
140
|
*/
|
|
141
141
|
protected _initializeText(): void;
|
|
142
142
|
protected syncTextObj(): void;
|
|
143
|
-
protected setTextColor(color: number): void;
|
|
143
|
+
protected setTextColor(color: number | string): void;
|
|
144
144
|
/**
|
|
145
145
|
* Disposes of resources used by the TextView, such as event listeners.
|
|
146
146
|
*/
|
|
@@ -61,7 +61,7 @@ export declare class ObjectDetector extends Script {
|
|
|
61
61
|
* Retrieves a list of currently detected objects.
|
|
62
62
|
*
|
|
63
63
|
* @param label - The semantic label to filter by (e.g., 'chair'). If null,
|
|
64
|
-
*
|
|
64
|
+
* all objects are returned.
|
|
65
65
|
* @returns An array of `Object` instances.
|
|
66
66
|
*/
|
|
67
67
|
get(label?: null): DetectedObject[];
|
|
@@ -79,10 +79,15 @@ export declare class ObjectDetector extends Script {
|
|
|
79
79
|
* Draws the detected bounding boxes on the input image and triggers a
|
|
80
80
|
* download for debugging.
|
|
81
81
|
* @param base64Image - The base64 encoded input image.
|
|
82
|
-
* @param detections - The array of detected objects from the
|
|
83
|
-
* AI response.
|
|
82
|
+
* @param detections - The array of detected objects from the AI response.
|
|
84
83
|
*/
|
|
85
84
|
private _visualizeBoundingBoxesOnImage;
|
|
85
|
+
/**
|
|
86
|
+
* Generates a visual representation of the depth map, normalized to 0-1 range,
|
|
87
|
+
* and triggers a download for debugging.
|
|
88
|
+
* @param depthArray - The raw depth data array.
|
|
89
|
+
*/
|
|
90
|
+
private _visualizeDepthMap;
|
|
86
91
|
/**
|
|
87
92
|
* Creates a simple debug visualization for an object based on its position
|
|
88
93
|
* (center of its 2D detection bounding box).
|
package/build/xrblocks.js
CHANGED
|
@@ -14,9 +14,9 @@
|
|
|
14
14
|
* limitations under the License.
|
|
15
15
|
*
|
|
16
16
|
* @file xrblocks.js
|
|
17
|
-
* @version v0.5.
|
|
18
|
-
* @commitid
|
|
19
|
-
* @builddate 2025-12-
|
|
17
|
+
* @version v0.5.1
|
|
18
|
+
* @commitid 896d66f
|
|
19
|
+
* @builddate 2025-12-06T06:19:44.629Z
|
|
20
20
|
* @description XR Blocks SDK, built from source with the above commit ID.
|
|
21
21
|
* @agent When using with Gemini to create XR apps, use **Gemini Canvas** mode,
|
|
22
22
|
* and follow rules below:
|
|
@@ -1849,6 +1849,27 @@ function deepMerge(obj1, obj2) {
|
|
|
1849
1849
|
}
|
|
1850
1850
|
}
|
|
1851
1851
|
|
|
1852
|
+
/**
|
|
1853
|
+
* Default parameters for rgb to depth projection.
|
|
1854
|
+
* For RGB and depth, 4:3 and 1:1, respectively.
|
|
1855
|
+
*/
|
|
1856
|
+
const DEFAULT_RGB_TO_DEPTH_PARAMS = {
|
|
1857
|
+
scale: 1,
|
|
1858
|
+
scaleX: 0.75,
|
|
1859
|
+
scaleY: 0.63,
|
|
1860
|
+
translateU: 0.2,
|
|
1861
|
+
translateV: -0.02,
|
|
1862
|
+
k1: -0.046,
|
|
1863
|
+
k2: 0,
|
|
1864
|
+
k3: 0,
|
|
1865
|
+
p1: 0,
|
|
1866
|
+
p2: 0,
|
|
1867
|
+
xc: 0,
|
|
1868
|
+
yc: 0,
|
|
1869
|
+
};
|
|
1870
|
+
/**
|
|
1871
|
+
* Configuration options for the device camera.
|
|
1872
|
+
*/
|
|
1852
1873
|
class DeviceCameraOptions {
|
|
1853
1874
|
constructor(options) {
|
|
1854
1875
|
this.enabled = false;
|
|
@@ -1856,10 +1877,14 @@ class DeviceCameraOptions {
|
|
|
1856
1877
|
* Hint for performance optimization on frequent captures.
|
|
1857
1878
|
*/
|
|
1858
1879
|
this.willCaptureFrequently = false;
|
|
1880
|
+
/**
|
|
1881
|
+
* Parameters for RGB to depth UV mapping given different aspect ratios.
|
|
1882
|
+
*/
|
|
1883
|
+
this.rgbToDepthParams = { ...DEFAULT_RGB_TO_DEPTH_PARAMS };
|
|
1859
1884
|
deepMerge(this, options);
|
|
1860
1885
|
}
|
|
1861
1886
|
}
|
|
1862
|
-
// Base configuration for all common capture settings
|
|
1887
|
+
// Base configuration for all common capture settings.
|
|
1863
1888
|
const baseCaptureOptions = {
|
|
1864
1889
|
enabled: true,
|
|
1865
1890
|
videoConstraints: {
|
|
@@ -3316,24 +3341,6 @@ const aspectRatios = {
|
|
|
3316
3341
|
depth: 1.0,
|
|
3317
3342
|
RGB: 4 / 3,
|
|
3318
3343
|
};
|
|
3319
|
-
/**
|
|
3320
|
-
* Parameters for RGB to depth UV mapping (manually calibrated for aspect
|
|
3321
|
-
* ratios. For RGB and depth, 4:3 and 1:1, respectively.
|
|
3322
|
-
*/
|
|
3323
|
-
const rgbToDepthParams = {
|
|
3324
|
-
scale: 1,
|
|
3325
|
-
scaleX: 0.75,
|
|
3326
|
-
scaleY: 0.63,
|
|
3327
|
-
translateU: 0.2,
|
|
3328
|
-
translateV: -0.02,
|
|
3329
|
-
k1: -0.046,
|
|
3330
|
-
k2: 0,
|
|
3331
|
-
k3: 0,
|
|
3332
|
-
p1: 0,
|
|
3333
|
-
p2: 0,
|
|
3334
|
-
xc: 0,
|
|
3335
|
-
yc: 0,
|
|
3336
|
-
};
|
|
3337
3344
|
/**
|
|
3338
3345
|
* Maps a UV coordinate from a RGB space to a destination depth space,
|
|
3339
3346
|
* applying Brown-Conrady distortion and affine transformations based on
|
|
@@ -3368,42 +3375,40 @@ function transformRgbToDepthUv(rgbUv, xrDeviceCamera) {
|
|
|
3368
3375
|
console.error('Invalid aspect ratios provided.');
|
|
3369
3376
|
return null;
|
|
3370
3377
|
}
|
|
3371
|
-
|
|
3378
|
+
const params = xrDeviceCamera?.rgbToDepthParams ?? DEFAULT_RGB_TO_DEPTH_PARAMS;
|
|
3379
|
+
// Determine the relative scaling required to fit the overlay within the base.
|
|
3372
3380
|
let relativeScaleX, relativeScaleY;
|
|
3373
3381
|
if (aspectRatios.depth > aspectRatios.RGB) {
|
|
3374
|
-
// Base is wider than overlay ("letterboxing")
|
|
3382
|
+
// Base is wider than overlay ("letterboxing").
|
|
3375
3383
|
relativeScaleY = 1.0;
|
|
3376
3384
|
relativeScaleX = aspectRatios.RGB / aspectRatios.depth;
|
|
3377
3385
|
}
|
|
3378
3386
|
else {
|
|
3379
|
-
// Base is narrower than overlay ("pillarboxing")
|
|
3387
|
+
// Base is narrower than overlay ("pillarboxing").
|
|
3380
3388
|
relativeScaleX = 1.0;
|
|
3381
3389
|
relativeScaleY = aspectRatios.depth / aspectRatios.RGB;
|
|
3382
3390
|
}
|
|
3383
|
-
// Convert input source UV [0, 1] to
|
|
3391
|
+
// Convert input source UV [0, 1] to normalized coordinates in [-0.5, 0.5].
|
|
3384
3392
|
const u_norm = rgbUv.u - 0.5;
|
|
3385
3393
|
const v_norm = rgbUv.v - 0.5;
|
|
3386
|
-
// Apply the FORWARD Brown-Conrady distortion model
|
|
3387
|
-
const u_centered = u_norm -
|
|
3388
|
-
const v_centered = v_norm -
|
|
3394
|
+
// Apply the FORWARD Brown-Conrady distortion model.
|
|
3395
|
+
const u_centered = u_norm - params.xc;
|
|
3396
|
+
const v_centered = v_norm - params.yc;
|
|
3389
3397
|
const r2 = u_centered * u_centered + v_centered * v_centered;
|
|
3390
|
-
const radial = 1 +
|
|
3391
|
-
|
|
3392
|
-
|
|
3393
|
-
|
|
3394
|
-
|
|
3395
|
-
|
|
3396
|
-
const
|
|
3397
|
-
|
|
3398
|
-
const
|
|
3399
|
-
const
|
|
3400
|
-
// Apply
|
|
3401
|
-
const
|
|
3402
|
-
const
|
|
3403
|
-
//
|
|
3404
|
-
const finalNormX = u_fitted * rgbToDepthParams.scale * rgbToDepthParams.scaleX;
|
|
3405
|
-
const finalNormY = v_fitted * rgbToDepthParams.scale * rgbToDepthParams.scaleY;
|
|
3406
|
-
// Convert the final normalized coordinate back to a UV coordinate [0, 1]
|
|
3398
|
+
const radial = 1 + params.k1 * r2 + params.k2 * r2 * r2 + params.k3 * r2 * r2 * r2;
|
|
3399
|
+
const tanX = 2 * params.p1 * u_centered * v_centered +
|
|
3400
|
+
params.p2 * (r2 + 2 * u_centered * u_centered);
|
|
3401
|
+
const tanY = params.p1 * (r2 + 2 * v_centered * v_centered) +
|
|
3402
|
+
2 * params.p2 * u_centered * v_centered;
|
|
3403
|
+
const u_distorted = u_centered * radial + tanX + params.xc;
|
|
3404
|
+
const v_distorted = v_centered * radial + tanY + params.yc;
|
|
3405
|
+
// Apply initial aspect ratio scaling and translation.
|
|
3406
|
+
const u_fitted = u_distorted * relativeScaleX + params.translateU;
|
|
3407
|
+
const v_fitted = v_distorted * relativeScaleY + params.translateV;
|
|
3408
|
+
// Apply the final user-controlled scaling (zoom and stretch).
|
|
3409
|
+
const finalNormX = u_fitted * params.scale * params.scaleX;
|
|
3410
|
+
const finalNormY = v_fitted * params.scale * params.scaleY;
|
|
3411
|
+
// Convert the final normalized coordinate back to a UV coordinate [0, 1].
|
|
3407
3412
|
const finalU = finalNormX + 0.5;
|
|
3408
3413
|
const finalV = finalNormY + 0.5;
|
|
3409
3414
|
return { u: finalU, v: 1.0 - finalV };
|
|
@@ -3667,12 +3672,13 @@ class XRDeviceCamera extends VideoStream {
|
|
|
3667
3672
|
/**
|
|
3668
3673
|
* @param options - The configuration options.
|
|
3669
3674
|
*/
|
|
3670
|
-
constructor({ videoConstraints = { facingMode: 'environment' }, willCaptureFrequently = false, } = {}) {
|
|
3675
|
+
constructor({ videoConstraints = { facingMode: 'environment' }, willCaptureFrequently = false, rgbToDepthParams = DEFAULT_RGB_TO_DEPTH_PARAMS, } = {}) {
|
|
3671
3676
|
super({ willCaptureFrequently });
|
|
3672
3677
|
this.isInitializing_ = false;
|
|
3673
3678
|
this.availableDevices_ = [];
|
|
3674
3679
|
this.currentDeviceIndex_ = -1;
|
|
3675
3680
|
this.videoConstraints_ = { ...videoConstraints };
|
|
3681
|
+
this.rgbToDepthParams = rgbToDepthParams;
|
|
3676
3682
|
}
|
|
3677
3683
|
/**
|
|
3678
3684
|
* Retrieves the list of available video input devices.
|
|
@@ -3724,7 +3730,7 @@ class XRDeviceCamera extends VideoStream {
|
|
|
3724
3730
|
return;
|
|
3725
3731
|
this.isInitializing_ = true;
|
|
3726
3732
|
this.setState_(StreamState.INITIALIZING);
|
|
3727
|
-
// Reset state for the new stream
|
|
3733
|
+
// Reset state for the new stream.
|
|
3728
3734
|
this.currentTrackSettings_ = undefined;
|
|
3729
3735
|
this.currentDeviceIndex_ = -1;
|
|
3730
3736
|
try {
|
|
@@ -3758,7 +3764,7 @@ class XRDeviceCamera extends VideoStream {
|
|
|
3758
3764
|
if (!videoTracks.length) {
|
|
3759
3765
|
throw new Error('MediaStream has no video tracks.');
|
|
3760
3766
|
}
|
|
3761
|
-
// After the stream is active, we can get the ID
|
|
3767
|
+
// After the stream is active, we can get the track ID.
|
|
3762
3768
|
const activeTrack = videoTracks[0];
|
|
3763
3769
|
this.currentTrackSettings_ = activeTrack.getSettings();
|
|
3764
3770
|
console.debug('Active track settings:', this.currentTrackSettings_);
|
|
@@ -3768,10 +3774,10 @@ class XRDeviceCamera extends VideoStream {
|
|
|
3768
3774
|
else {
|
|
3769
3775
|
console.warn('Stream started without deviceId as it was unavailable');
|
|
3770
3776
|
}
|
|
3771
|
-
this.stop_(); // Stop any previous stream before starting new one
|
|
3777
|
+
this.stop_(); // Stop any previous stream before starting new one.
|
|
3772
3778
|
this.stream_ = stream;
|
|
3773
3779
|
this.video_.srcObject = stream;
|
|
3774
|
-
this.video_.src = ''; // Required for some browsers to reset the src
|
|
3780
|
+
this.video_.src = ''; // Required for some browsers to reset the src.
|
|
3775
3781
|
await new Promise((resolve, reject) => {
|
|
3776
3782
|
this.video_.onloadedmetadata = () => {
|
|
3777
3783
|
this.handleVideoStreamLoadedMetadata(resolve, reject, true);
|
|
@@ -3783,7 +3789,7 @@ class XRDeviceCamera extends VideoStream {
|
|
|
3783
3789
|
};
|
|
3784
3790
|
this.video_.play();
|
|
3785
3791
|
});
|
|
3786
|
-
// Once
|
|
3792
|
+
// Once stream is loaded and dimensions are known, set the final state.
|
|
3787
3793
|
const details = {
|
|
3788
3794
|
width: this.width,
|
|
3789
3795
|
height: this.height,
|
|
@@ -3805,7 +3811,7 @@ class XRDeviceCamera extends VideoStream {
|
|
|
3805
3811
|
/**
|
|
3806
3812
|
* Sets the active camera by its device ID. Removes potentially conflicting
|
|
3807
3813
|
* constraints such as facingMode.
|
|
3808
|
-
* @param deviceId - Device
|
|
3814
|
+
* @param deviceId - Device ID
|
|
3809
3815
|
*/
|
|
3810
3816
|
async setDeviceId(deviceId) {
|
|
3811
3817
|
const newIndex = this.availableDevices_.findIndex((device) => device.deviceId === deviceId);
|
|
@@ -11796,7 +11802,7 @@ class TextView extends View {
|
|
|
11796
11802
|
}
|
|
11797
11803
|
setTextColor(color) {
|
|
11798
11804
|
if (Text && this.textObj instanceof Text) {
|
|
11799
|
-
this.textObj.color = color;
|
|
11805
|
+
this.textObj.color = getColorHex(color);
|
|
11800
11806
|
}
|
|
11801
11807
|
}
|
|
11802
11808
|
/**
|
|
@@ -12186,7 +12192,7 @@ class TextButton extends TextView {
|
|
|
12186
12192
|
*/
|
|
12187
12193
|
constructor(options = {}) {
|
|
12188
12194
|
const geometry = new THREE.PlaneGeometry(1, 1);
|
|
12189
|
-
const colorVec4 = getVec4ByColorString(options.backgroundColor ?? '#
|
|
12195
|
+
const colorVec4 = getVec4ByColorString(options.backgroundColor ?? '#000000');
|
|
12190
12196
|
const { opacity = 0.0, radius = SquircleShader.uniforms.uRadius.value, boxSize = SquircleShader.uniforms.uBoxSize.value, } = options;
|
|
12191
12197
|
const uniforms = {
|
|
12192
12198
|
...SquircleShader.uniforms,
|
|
@@ -12236,6 +12242,9 @@ class TextButton extends TextView {
|
|
|
12236
12242
|
// Applies our own overrides to the default values.
|
|
12237
12243
|
this.fontSize = options.fontSize ?? this.fontSize;
|
|
12238
12244
|
this.fontColor = options.fontColor ?? this.fontColor;
|
|
12245
|
+
this.hoverColor = options.hoverColor ?? this.hoverColor;
|
|
12246
|
+
this.selectedFontColor =
|
|
12247
|
+
options.selectedFontColor ?? this.selectedFontColor;
|
|
12239
12248
|
this.width = options.width ?? this.width;
|
|
12240
12249
|
this.height = options.height ?? this.height;
|
|
12241
12250
|
}
|
|
@@ -12258,20 +12267,19 @@ class TextButton extends TextView {
|
|
|
12258
12267
|
if (!this.textObj) {
|
|
12259
12268
|
return;
|
|
12260
12269
|
}
|
|
12261
|
-
|
|
12262
|
-
|
|
12263
|
-
}
|
|
12270
|
+
// Update render order to ensure text appears on top of the button mesh
|
|
12271
|
+
this.textObj.renderOrder = this.renderOrder + 1;
|
|
12264
12272
|
const ux = this.ux;
|
|
12265
12273
|
if (ux.isHovered()) {
|
|
12266
12274
|
if (ux.isSelected()) {
|
|
12267
|
-
this.setTextColor(
|
|
12275
|
+
this.setTextColor(this.selectedFontColor);
|
|
12268
12276
|
}
|
|
12269
12277
|
else {
|
|
12270
|
-
this.setTextColor(
|
|
12278
|
+
this.setTextColor(this.hoverColor);
|
|
12271
12279
|
}
|
|
12272
12280
|
}
|
|
12273
12281
|
else {
|
|
12274
|
-
this.setTextColor(
|
|
12282
|
+
this.setTextColor(this.fontColor);
|
|
12275
12283
|
this.uniforms.uOpacity.value = this.defaultOpacity * this.opacity;
|
|
12276
12284
|
}
|
|
12277
12285
|
}
|
|
@@ -13887,6 +13895,7 @@ class ObjectDetector extends Script {
|
|
|
13887
13895
|
}
|
|
13888
13896
|
if (this.options.objects.showDebugVisualizations) {
|
|
13889
13897
|
this._visualizeBoundingBoxesOnImage(base64Image, parsedResponse);
|
|
13898
|
+
this._visualizeDepthMap(cachedDepthArray);
|
|
13890
13899
|
}
|
|
13891
13900
|
const detectionPromises = parsedResponse.map(async (item) => {
|
|
13892
13901
|
const { ymin, xmin, ymax, xmax, objectName, ...additionalData } = item || {};
|
|
@@ -13935,7 +13944,7 @@ class ObjectDetector extends Script {
|
|
|
13935
13944
|
* Retrieves a list of currently detected objects.
|
|
13936
13945
|
*
|
|
13937
13946
|
* @param label - The semantic label to filter by (e.g., 'chair'). If null,
|
|
13938
|
-
*
|
|
13947
|
+
* all objects are returned.
|
|
13939
13948
|
* @returns An array of `Object` instances.
|
|
13940
13949
|
*/
|
|
13941
13950
|
get(label = null) {
|
|
@@ -13972,8 +13981,7 @@ class ObjectDetector extends Script {
|
|
|
13972
13981
|
* Draws the detected bounding boxes on the input image and triggers a
|
|
13973
13982
|
* download for debugging.
|
|
13974
13983
|
* @param base64Image - The base64 encoded input image.
|
|
13975
|
-
* @param detections - The array of detected objects from the
|
|
13976
|
-
* AI response.
|
|
13984
|
+
* @param detections - The array of detected objects from the AI response.
|
|
13977
13985
|
*/
|
|
13978
13986
|
_visualizeBoundingBoxesOnImage(base64Image, detections) {
|
|
13979
13987
|
const img = new Image();
|
|
@@ -14022,6 +14030,71 @@ class ObjectDetector extends Script {
|
|
|
14022
14030
|
};
|
|
14023
14031
|
img.src = base64Image;
|
|
14024
14032
|
}
|
|
14033
|
+
/**
|
|
14034
|
+
* Generates a visual representation of the depth map, normalized to 0-1 range,
|
|
14035
|
+
* and triggers a download for debugging.
|
|
14036
|
+
* @param depthArray - The raw depth data array.
|
|
14037
|
+
*/
|
|
14038
|
+
_visualizeDepthMap(depthArray) {
|
|
14039
|
+
const width = this.depth.width;
|
|
14040
|
+
const height = this.depth.height;
|
|
14041
|
+
if (!width || !height || depthArray.length === 0) {
|
|
14042
|
+
console.warn('Cannot visualize depth map: missing dimensions or data.');
|
|
14043
|
+
return;
|
|
14044
|
+
}
|
|
14045
|
+
// 1. Find Min/Max for normalization (ignoring 0/invalid depth).
|
|
14046
|
+
let min = Infinity;
|
|
14047
|
+
let max = -Infinity;
|
|
14048
|
+
for (let i = 0; i < depthArray.length; ++i) {
|
|
14049
|
+
const val = depthArray[i];
|
|
14050
|
+
if (val > 0) {
|
|
14051
|
+
if (val < min)
|
|
14052
|
+
min = val;
|
|
14053
|
+
if (val > max)
|
|
14054
|
+
max = val;
|
|
14055
|
+
}
|
|
14056
|
+
}
|
|
14057
|
+
// Handle edge case where no valid depth exists.
|
|
14058
|
+
if (min === Infinity) {
|
|
14059
|
+
min = 0;
|
|
14060
|
+
max = 1;
|
|
14061
|
+
}
|
|
14062
|
+
if (min === max) {
|
|
14063
|
+
max = min + 1; // Avoid divide by zero
|
|
14064
|
+
}
|
|
14065
|
+
// 2. Create Canvas.
|
|
14066
|
+
const canvas = document.createElement('canvas');
|
|
14067
|
+
canvas.width = width;
|
|
14068
|
+
canvas.height = height;
|
|
14069
|
+
const ctx = canvas.getContext('2d');
|
|
14070
|
+
const imageData = ctx.createImageData(width, height);
|
|
14071
|
+
const data = imageData.data;
|
|
14072
|
+
// 3. Fill Pixels.
|
|
14073
|
+
for (let i = 0; i < depthArray.length; ++i) {
|
|
14074
|
+
const raw = depthArray[i];
|
|
14075
|
+
// Normalize to 0-1.
|
|
14076
|
+
// Typically 0 means invalid/sky in some depth APIs, so we keep it black.
|
|
14077
|
+
// Otherwise, map [min, max] to [0, 1].
|
|
14078
|
+
const normalized = raw === 0 ? 0 : (raw - min) / (max - min);
|
|
14079
|
+
const byteVal = Math.floor(normalized * 255);
|
|
14080
|
+
const stride = i * 4;
|
|
14081
|
+
data[stride] = byteVal; // R
|
|
14082
|
+
data[stride + 1] = byteVal; // G
|
|
14083
|
+
data[stride + 2] = byteVal; // B
|
|
14084
|
+
data[stride + 3] = 255; // Alpha
|
|
14085
|
+
}
|
|
14086
|
+
ctx.putImageData(imageData, 0, 0);
|
|
14087
|
+
// 4. Download.
|
|
14088
|
+
const timestamp = new Date()
|
|
14089
|
+
.toISOString()
|
|
14090
|
+
.slice(0, 19)
|
|
14091
|
+
.replace('T', '_')
|
|
14092
|
+
.replace(/:/g, '-');
|
|
14093
|
+
const link = document.createElement('a');
|
|
14094
|
+
link.download = `depth_debug_${timestamp}.png`;
|
|
14095
|
+
link.href = canvas.toDataURL('image/png');
|
|
14096
|
+
link.click();
|
|
14097
|
+
}
|
|
14025
14098
|
/**
|
|
14026
14099
|
* Creates a simple debug visualization for an object based on its position
|
|
14027
14100
|
* (center of its 2D detection bounding box).
|
|
@@ -17086,5 +17159,5 @@ class VideoFileStream extends VideoStream {
|
|
|
17086
17159
|
}
|
|
17087
17160
|
}
|
|
17088
17161
|
|
|
17089
|
-
export { AI, AIOptions, AVERAGE_IPD_METERS, ActiveControllers, Agent, AnimatableNumber, AudioListener, AudioPlayer, BACK, BackgroundMusic, CategoryVolumes, Col, Core, CoreSound, DEFAULT_DEVICE_CAMERA_HEIGHT, DEFAULT_DEVICE_CAMERA_WIDTH, DOWN, Depth, DepthMesh, DepthMeshOptions, DepthOptions, DepthTextures, DetectedObject, DetectedPlane, DeviceCameraOptions, DragManager, DragMode, ExitButton, FORWARD, FreestandingSlider, GazeController, Gemini, GeminiOptions, GenerateSkyboxTool, GestureRecognition, GestureRecognitionOptions, GetWeatherTool, Grid, HAND_BONE_IDX_CONNECTION_MAP, HAND_JOINT_COUNT, HAND_JOINT_IDX_CONNECTION_MAP, HAND_JOINT_NAMES, Handedness, Hands, HandsOptions, HorizontalPager, IconButton, IconView, ImageView, Input, InputOptions, Keycodes, LEFT, LEFT_VIEW_ONLY_LAYER, LabelView, Lighting, LightingOptions, LoadingSpinnerManager, MaterialSymbolsView, MeshScript, ModelLoader, ModelViewer, MouseController, NEXT_SIMULATOR_MODE, NUM_HANDS, OCCLUDABLE_ITEMS_LAYER, ObjectDetector, ObjectsOptions, OcclusionPass, OcclusionUtils, OpenAI, OpenAIOptions, Options, PageIndicator, Pager, PagerState, Panel, PanelMesh, Physics, PhysicsOptions, PinchOnButtonAction, PlaneDetector, PlanesOptions, RIGHT, RIGHT_VIEW_ONLY_LAYER, Registry, Reticle, ReticleOptions, RotationRaycastMesh, Row, SIMULATOR_HAND_POSE_NAMES, SIMULATOR_HAND_POSE_TO_JOINTS_LEFT, SIMULATOR_HAND_POSE_TO_JOINTS_RIGHT, SOUND_PRESETS, ScreenshotSynthesizer, Script, ScriptMixin, ScriptsManager, ScrollingTroikaTextView, SetSimulatorModeEvent, ShowHandsAction, Simulator, SimulatorCamera, SimulatorControlMode, SimulatorControllerState, SimulatorControls, SimulatorDepth, SimulatorDepthMaterial, SimulatorHandPose, SimulatorHandPoseChangeRequestEvent, SimulatorHands, SimulatorInterface, SimulatorMediaDeviceInfo, SimulatorMode, SimulatorOptions, SimulatorRenderMode, SimulatorScene, SimulatorUser, SimulatorUserAction, SketchPanel, SkyboxAgent, SoundOptions, SoundSynthesizer, SpatialAudio, SpatialPanel, SpeechRecognizer, SpeechRecognizerOptions, SpeechSynthesizer, SpeechSynthesizerOptions, SplatAnchor, StreamState, TextButton, TextScrollerState, TextView, Tool, UI, UI_OVERLAY_LAYER, UP, UX, User, VIEW_DEPTH_GAP, VerticalPager, VideoFileStream, VideoStream, VideoView, View, VolumeCategory, WaitFrame, WalkTowardsPanelAction, World, WorldOptions, XRButton, XRDeviceCamera, XREffects, XRPass, XRTransitionOptions, XR_BLOCKS_ASSETS_PATH, ZERO_VECTOR3, add, ai, aspectRatios, callInitWithDependencyInjection, clamp, clampRotationToAngle, core, cropImage, extractYaw, getColorHex, getDeltaTime, getUrlParamBool, getUrlParamFloat, getUrlParamInt, getUrlParameter, getVec4ByColorString, getXrCameraLeft, getXrCameraRight, init, initScript, lerp, loadStereoImageAsTextures, loadingSpinnerManager, lookAtRotation, objectIsDescendantOf, parseBase64DataURL, placeObjectAtIntersectionFacingTarget, print,
|
|
17162
|
+
export { AI, AIOptions, AVERAGE_IPD_METERS, ActiveControllers, Agent, AnimatableNumber, AudioListener, AudioPlayer, BACK, BackgroundMusic, CategoryVolumes, Col, Core, CoreSound, DEFAULT_DEVICE_CAMERA_HEIGHT, DEFAULT_DEVICE_CAMERA_WIDTH, DEFAULT_RGB_TO_DEPTH_PARAMS, DOWN, Depth, DepthMesh, DepthMeshOptions, DepthOptions, DepthTextures, DetectedObject, DetectedPlane, DeviceCameraOptions, DragManager, DragMode, ExitButton, FORWARD, FreestandingSlider, GazeController, Gemini, GeminiOptions, GenerateSkyboxTool, GestureRecognition, GestureRecognitionOptions, GetWeatherTool, Grid, HAND_BONE_IDX_CONNECTION_MAP, HAND_JOINT_COUNT, HAND_JOINT_IDX_CONNECTION_MAP, HAND_JOINT_NAMES, Handedness, Hands, HandsOptions, HorizontalPager, IconButton, IconView, ImageView, Input, InputOptions, Keycodes, LEFT, LEFT_VIEW_ONLY_LAYER, LabelView, Lighting, LightingOptions, LoadingSpinnerManager, MaterialSymbolsView, MeshScript, ModelLoader, ModelViewer, MouseController, NEXT_SIMULATOR_MODE, NUM_HANDS, OCCLUDABLE_ITEMS_LAYER, ObjectDetector, ObjectsOptions, OcclusionPass, OcclusionUtils, OpenAI, OpenAIOptions, Options, PageIndicator, Pager, PagerState, Panel, PanelMesh, Physics, PhysicsOptions, PinchOnButtonAction, PlaneDetector, PlanesOptions, RIGHT, RIGHT_VIEW_ONLY_LAYER, Registry, Reticle, ReticleOptions, RotationRaycastMesh, Row, SIMULATOR_HAND_POSE_NAMES, SIMULATOR_HAND_POSE_TO_JOINTS_LEFT, SIMULATOR_HAND_POSE_TO_JOINTS_RIGHT, SOUND_PRESETS, ScreenshotSynthesizer, Script, ScriptMixin, ScriptsManager, ScrollingTroikaTextView, SetSimulatorModeEvent, ShowHandsAction, Simulator, SimulatorCamera, SimulatorControlMode, SimulatorControllerState, SimulatorControls, SimulatorDepth, SimulatorDepthMaterial, SimulatorHandPose, SimulatorHandPoseChangeRequestEvent, SimulatorHands, SimulatorInterface, SimulatorMediaDeviceInfo, SimulatorMode, SimulatorOptions, SimulatorRenderMode, SimulatorScene, SimulatorUser, SimulatorUserAction, SketchPanel, SkyboxAgent, SoundOptions, SoundSynthesizer, SpatialAudio, SpatialPanel, SpeechRecognizer, SpeechRecognizerOptions, SpeechSynthesizer, SpeechSynthesizerOptions, SplatAnchor, StreamState, TextButton, TextScrollerState, TextView, Tool, UI, UI_OVERLAY_LAYER, UP, UX, User, VIEW_DEPTH_GAP, VerticalPager, VideoFileStream, VideoStream, VideoView, View, VolumeCategory, WaitFrame, WalkTowardsPanelAction, World, WorldOptions, XRButton, XRDeviceCamera, XREffects, XRPass, XRTransitionOptions, XR_BLOCKS_ASSETS_PATH, ZERO_VECTOR3, add, ai, aspectRatios, callInitWithDependencyInjection, clamp, clampRotationToAngle, core, cropImage, extractYaw, getColorHex, getDeltaTime, getUrlParamBool, getUrlParamFloat, getUrlParamInt, getUrlParameter, getVec4ByColorString, getXrCameraLeft, getXrCameraRight, init, initScript, lerp, loadStereoImageAsTextures, loadingSpinnerManager, lookAtRotation, objectIsDescendantOf, parseBase64DataURL, placeObjectAtIntersectionFacingTarget, print, scene, showOnlyInLeftEye, showOnlyInRightEye, showReticleOnDepthMesh, transformRgbToDepthUv, transformRgbUvToWorld, traverseUtil, uninitScript, urlParams, user, world, xrDepthMeshOptions, xrDepthMeshPhysicsOptions, xrDepthMeshVisualizationOptions, xrDeviceCameraEnvironmentContinuousOptions, xrDeviceCameraEnvironmentOptions, xrDeviceCameraUserContinuousOptions, xrDeviceCameraUserOptions };
|
|
17090
17163
|
//# sourceMappingURL=xrblocks.js.map
|