xrblocks 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -9
- package/build/addons/ai/AudioCaptureProcessorCode.d.ts +1 -0
- package/build/addons/ai/AudioCaptureProcessorCode.js +27 -0
- package/build/addons/ai/GeminiManager.d.ts +7 -3
- package/build/addons/ai/GeminiManager.js +48 -23
- package/build/addons/objects/SimpleDecalGeometry.js +9 -5
- package/build/addons/simulator/instructions/CustomInstruction.js +8 -9
- package/build/addons/simulator/instructions/HandsInstructions.js +17 -10
- package/build/addons/simulator/instructions/NavigationInstructions.js +10 -9
- package/build/addons/simulator/instructions/SimulatorInstructions.js +17 -18
- package/build/addons/simulator/instructions/SimulatorInstructionsCard.js +69 -75
- package/build/addons/simulator/instructions/SimulatorInstructionsEvents.js +4 -1
- package/build/addons/simulator/instructions/UserInstructions.js +18 -15
- package/build/addons/simulator/ui/EnterXRButton.js +17 -17
- package/build/addons/simulator/ui/GeminiLiveApiKeyInput.js +45 -39
- package/build/addons/simulator/ui/HandPosePanel.js +20 -10
- package/build/addons/simulator/ui/MicButton.js +23 -18
- package/build/addons/simulator/ui/ModeIndicator.js +17 -17
- package/build/addons/ui/TextBillboard.js +1 -1
- package/build/addons/utils/Palette.js +3 -15
- package/build/addons/virtualkeyboard/Keyboard.js +24 -21
- package/build/addons/volumes/VolumetricCloud.glsl.js +1 -1
- package/build/addons/volumes/VolumetricCloud.js +8 -5
- package/build/agent/Tool.d.ts +3 -1
- package/build/ai/AI.d.ts +2 -2
- package/build/ai/Gemini.d.ts +1 -5
- package/build/camera/XRDeviceCamera.d.ts +1 -1
- package/build/core/Core.d.ts +3 -1
- package/build/core/Options.d.ts +7 -0
- package/build/core/components/ScreenshotSynthesizer.d.ts +2 -2
- package/build/core/components/XRTransition.d.ts +1 -1
- package/build/depth/DepthMesh.d.ts +1 -1
- package/build/input/Hands.d.ts +1 -1
- package/build/input/Input.d.ts +1 -1
- package/build/input/gestures/GestureEvents.d.ts +23 -0
- package/build/input/gestures/GestureRecognition.d.ts +43 -0
- package/build/input/gestures/GestureRecognitionOptions.d.ts +43 -0
- package/build/input/gestures/GestureTypes.d.ts +16 -0
- package/build/input/gestures/providers/HeuristicGestureDetectors.d.ts +2 -0
- package/build/simulator/Simulator.d.ts +2 -0
- package/build/simulator/SimulatorControls.d.ts +1 -1
- package/build/simulator/controlModes/SimulatorControlMode.d.ts +1 -1
- package/build/simulator/handPoses/HandPoseJoints.d.ts +2 -2
- package/build/simulator/userActions/PinchOnButtonAction.d.ts +2 -2
- package/build/simulator/userActions/WalkTowardsPanelAction.d.ts +1 -1
- package/build/singletons.d.ts +2 -2
- package/build/sound/CoreSound.d.ts +1 -1
- package/build/stereo/utils.d.ts +1 -1
- package/build/ui/components/MaterialSymbolsView.d.ts +1 -1
- package/build/ui/components/ScrollingTroikaTextView.d.ts +1 -1
- package/build/ui/interaction/ModelViewer.d.ts +6 -2
- package/build/utils/ModelLoader.d.ts +1 -1
- package/build/utils/SparkRendererHolder.d.ts +5 -0
- package/build/utils/Types.d.ts +2 -2
- package/build/video/VideoStream.d.ts +1 -1
- package/build/world/World.d.ts +1 -1
- package/build/world/objects/ObjectDetector.d.ts +1 -1
- package/build/world/planes/PlaneDetector.d.ts +1 -1
- package/build/xrblocks.d.ts +3 -0
- package/build/xrblocks.js +6782 -6020
- package/build/xrblocks.js.map +1 -1
- package/build/xrblocks.min.js +1 -1
- package/build/xrblocks.min.js.map +1 -1
- package/package.json +13 -8
|
@@ -88,12 +88,13 @@ class Keyboard extends xb.Script {
|
|
|
88
88
|
this.onEnterPressed = null;
|
|
89
89
|
this.onTextChanged = null;
|
|
90
90
|
this.keyRows = [];
|
|
91
|
-
this.subspace =
|
|
92
|
-
|
|
91
|
+
this.subspace = new xb.SpatialPanel({
|
|
92
|
+
useDefaultPosition: false,
|
|
93
|
+
showEdge: false,
|
|
94
|
+
});
|
|
93
95
|
this.subspace.isRoot = true;
|
|
94
96
|
this.add(this.subspace);
|
|
95
|
-
this.mainPanel =
|
|
96
|
-
this.subspace.add(new xb.Panel({ backgroundColor: KEYBOARD_COLOR }));
|
|
97
|
+
this.mainPanel = this.subspace.add(new xb.Panel({ backgroundColor: KEYBOARD_COLOR }));
|
|
97
98
|
this.mainGrid = new xb.Grid();
|
|
98
99
|
this.mainPanel.add(this.mainGrid);
|
|
99
100
|
this.mainGrid.addRow({ weight: ROW_TOP_PADDING });
|
|
@@ -135,16 +136,16 @@ class Keyboard extends xb.Script {
|
|
|
135
136
|
if (textKeys.length === 0) {
|
|
136
137
|
let currentColumnWeight = 0;
|
|
137
138
|
row.addCol({ weight: 0.145 });
|
|
138
|
-
specialKeys.forEach(key => {
|
|
139
|
+
specialKeys.forEach((key) => {
|
|
139
140
|
currentColumnWeight += key.weight || COLUMN_WEIGHT;
|
|
140
141
|
});
|
|
141
142
|
const missingWeight = TARGET_COLUMN_WEIGHT - currentColumnWeight;
|
|
142
143
|
if (missingWeight > 1) {
|
|
143
|
-
const leftKeys = specialKeys.filter(key => key.position === 'left');
|
|
144
|
-
const rightKeys = specialKeys.filter(key => key.position === 'right');
|
|
145
|
-
const weightForLeftKeys =
|
|
146
|
-
const weightForRightKeys = (missingWeight * 2 / 3
|
|
147
|
-
specialKeys.forEach(key => {
|
|
144
|
+
const leftKeys = specialKeys.filter((key) => key.position === 'left');
|
|
145
|
+
const rightKeys = specialKeys.filter((key) => key.position === 'right');
|
|
146
|
+
const weightForLeftKeys = missingWeight / 3 / (leftKeys.length || 1);
|
|
147
|
+
const weightForRightKeys = (missingWeight * 2) / 3 / (rightKeys.length || 1);
|
|
148
|
+
specialKeys.forEach((key) => {
|
|
148
149
|
let finalWeight = key.weight || COLUMN_WEIGHT;
|
|
149
150
|
if (key.position === 'left') {
|
|
150
151
|
finalWeight += weightForLeftKeys;
|
|
@@ -156,14 +157,14 @@ class Keyboard extends xb.Script {
|
|
|
156
157
|
});
|
|
157
158
|
}
|
|
158
159
|
else {
|
|
159
|
-
specialKeys.forEach(key => this.createSpecialButtons(row, key));
|
|
160
|
+
specialKeys.forEach((key) => this.createSpecialButtons(row, key));
|
|
160
161
|
}
|
|
161
162
|
return;
|
|
162
163
|
}
|
|
163
|
-
const leftSpecialKeys = specialKeys.filter(key => key.position === 'left');
|
|
164
|
-
const rightSpecialKeys = specialKeys.filter(key => key.position === 'right');
|
|
164
|
+
const leftSpecialKeys = specialKeys.filter((key) => key.position === 'left');
|
|
165
|
+
const rightSpecialKeys = specialKeys.filter((key) => key.position === 'right');
|
|
165
166
|
let remainingWeight = TARGET_COLUMN_WEIGHT;
|
|
166
|
-
leftSpecialKeys.forEach(key => {
|
|
167
|
+
leftSpecialKeys.forEach((key) => {
|
|
167
168
|
this.createSpecialButtons(row, key);
|
|
168
169
|
remainingWeight -= key.weight;
|
|
169
170
|
});
|
|
@@ -172,7 +173,7 @@ class Keyboard extends xb.Script {
|
|
|
172
173
|
this.createTextButtons(row, key, shiftKey);
|
|
173
174
|
remainingWeight -= COLUMN_WEIGHT;
|
|
174
175
|
});
|
|
175
|
-
rightSpecialKeys.forEach(key => {
|
|
176
|
+
rightSpecialKeys.forEach((key) => {
|
|
176
177
|
const finalWeight = (key.weight || COLUMN_WEIGHT) + remainingWeight;
|
|
177
178
|
this.createSpecialButtons(row, { ...key, weight: finalWeight });
|
|
178
179
|
});
|
|
@@ -186,7 +187,7 @@ class Keyboard extends xb.Script {
|
|
|
186
187
|
fontSize: FONT_SIZE,
|
|
187
188
|
backgroundColor: DEFAULT_KEY_COLOR,
|
|
188
189
|
originalKey: key,
|
|
189
|
-
shiftKey: shiftKey
|
|
190
|
+
shiftKey: shiftKey,
|
|
190
191
|
});
|
|
191
192
|
keyPanel.add(textButton);
|
|
192
193
|
this.textButtons.push(textButton);
|
|
@@ -248,17 +249,19 @@ class Keyboard extends xb.Script {
|
|
|
248
249
|
}
|
|
249
250
|
}
|
|
250
251
|
refreshKeyboard() {
|
|
251
|
-
this.textButtons.forEach(button => {
|
|
252
|
+
this.textButtons.forEach((button) => {
|
|
252
253
|
const isLetter = button.originalKey.length === 1 && button.originalKey.match(/[a-z]/i);
|
|
253
254
|
let newText;
|
|
254
255
|
const produceUpper = this.isShifted !== this.isCapsLockOn;
|
|
255
256
|
if (isLetter) {
|
|
256
|
-
newText = produceUpper
|
|
257
|
-
button.originalKey.
|
|
257
|
+
newText = produceUpper
|
|
258
|
+
? button.originalKey.toUpperCase()
|
|
259
|
+
: button.originalKey.toLowerCase();
|
|
258
260
|
}
|
|
259
261
|
else {
|
|
260
|
-
newText = this.isShifted
|
|
261
|
-
button.originalKey
|
|
262
|
+
newText = this.isShifted
|
|
263
|
+
? button.shiftKey || button.originalKey
|
|
264
|
+
: button.originalKey;
|
|
262
265
|
}
|
|
263
266
|
button.setText(newText);
|
|
264
267
|
});
|
|
@@ -43,15 +43,18 @@ class VolumetricCloud extends THREE.Object3D {
|
|
|
43
43
|
for (let x = 0; x < this.size; x++) {
|
|
44
44
|
// Calculates the density based on the distance from the center.
|
|
45
45
|
const d = 1.0 -
|
|
46
|
-
vector
|
|
46
|
+
vector
|
|
47
|
+
.set(x, y, z)
|
|
47
48
|
.subScalar(this.size / 2)
|
|
48
49
|
.divideScalar(this.size)
|
|
49
50
|
.length();
|
|
50
51
|
// Populates the data array with density influenced by noise.
|
|
51
|
-
data[i] =
|
|
52
|
-
this.size
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
data[i] =
|
|
53
|
+
(this.size +
|
|
54
|
+
this.size *
|
|
55
|
+
perlin.noise((x * this.cloudScale) / 1.5, y * this.cloudScale, (z * this.cloudScale) / 1.5)) *
|
|
56
|
+
d *
|
|
57
|
+
d;
|
|
55
58
|
i++;
|
|
56
59
|
}
|
|
57
60
|
}
|
package/build/agent/Tool.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import * as GoogleGenAITypes from '@google/genai';
|
|
2
2
|
export interface ToolCall {
|
|
3
3
|
name: string;
|
|
4
4
|
args: unknown;
|
|
@@ -30,6 +30,7 @@ export type ToolOptions = {
|
|
|
30
30
|
parameters?: ToolSchema;
|
|
31
31
|
/** A callback to execute when the tool is triggered */
|
|
32
32
|
onTriggered?: (args: unknown) => unknown | Promise<unknown>;
|
|
33
|
+
behavior?: 'BLOCKING' | 'NON_BLOCKING' | GoogleGenAITypes.Behavior;
|
|
33
34
|
};
|
|
34
35
|
/**
|
|
35
36
|
* A base class for tools that the agent can use.
|
|
@@ -39,6 +40,7 @@ export declare class Tool {
|
|
|
39
40
|
description?: string;
|
|
40
41
|
parameters?: ToolSchema;
|
|
41
42
|
onTriggered?: (args: unknown) => unknown;
|
|
43
|
+
behavior?: 'BLOCKING' | 'NON_BLOCKING';
|
|
42
44
|
/**
|
|
43
45
|
* @param options - The options for the tool.
|
|
44
46
|
*/
|
package/build/ai/AI.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type * as GoogleGenAITypes from '@google/genai';
|
|
|
2
2
|
import { Script } from '../core/Script';
|
|
3
3
|
import { AIOptions, GeminiOptions, OpenAIOptions } from './AIOptions';
|
|
4
4
|
import { GeminiResponse } from './AITypes';
|
|
5
|
-
import { Gemini
|
|
5
|
+
import { Gemini } from './Gemini';
|
|
6
6
|
import { OpenAI } from './OpenAI';
|
|
7
7
|
export type ModelClass = Gemini | OpenAI;
|
|
8
8
|
export type ModelOptions = GeminiOptions | OpenAIOptions;
|
|
@@ -60,7 +60,7 @@ export declare class AI extends Script {
|
|
|
60
60
|
query(input: {
|
|
61
61
|
prompt: string;
|
|
62
62
|
}, tools?: never[]): Promise<GeminiResponse | string | null>;
|
|
63
|
-
startLiveSession(config?:
|
|
63
|
+
startLiveSession(config?: GoogleGenAITypes.LiveConnectConfig, model?: string): Promise<GoogleGenAITypes.Session>;
|
|
64
64
|
stopLiveSession(): Promise<void>;
|
|
65
65
|
setLiveCallbacks(callbacks: GoogleGenAITypes.LiveCallbacks): Promise<void>;
|
|
66
66
|
sendToolResponse(response: GoogleGenAITypes.LiveSendToolResponseParameters): void;
|
package/build/ai/Gemini.d.ts
CHANGED
|
@@ -14,10 +14,6 @@ export interface GeminiQueryInput {
|
|
|
14
14
|
config?: GoogleGenAITypes.LiveConnectConfig;
|
|
15
15
|
data?: GoogleGenAITypes.LiveSendRealtimeInputParameters;
|
|
16
16
|
}
|
|
17
|
-
export type GeminiStartLiveSessionParams = {
|
|
18
|
-
tools?: GoogleGenAITypes.FunctionDeclaration[];
|
|
19
|
-
systemInstruction?: GoogleGenAITypes.ContentUnion | string;
|
|
20
|
-
};
|
|
21
17
|
export declare class Gemini extends BaseAIModel {
|
|
22
18
|
protected options: GeminiOptions;
|
|
23
19
|
inited: boolean;
|
|
@@ -29,7 +25,7 @@ export declare class Gemini extends BaseAIModel {
|
|
|
29
25
|
init(): Promise<void>;
|
|
30
26
|
isAvailable(): boolean;
|
|
31
27
|
isLiveAvailable(): false | typeof GoogleGenAITypes.Modality | undefined;
|
|
32
|
-
startLiveSession(params?:
|
|
28
|
+
startLiveSession(params?: GoogleGenAITypes.LiveConnectConfig, model?: string): Promise<GoogleGenAITypes.Session>;
|
|
33
29
|
stopLiveSession(): Promise<void>;
|
|
34
30
|
setLiveCallbacks(callbacks: GoogleGenAITypes.LiveCallbacks): void;
|
|
35
31
|
sendToolResponse(response: GoogleGenAITypes.LiveSendToolResponseParameters): void;
|
|
@@ -23,7 +23,7 @@ export declare class XRDeviceCamera extends VideoStream<XRDeviceCameraDetails> {
|
|
|
23
23
|
/**
|
|
24
24
|
* @param options - The configuration options.
|
|
25
25
|
*/
|
|
26
|
-
constructor({ videoConstraints, willCaptureFrequently }?: Partial<DeviceCameraOptions>);
|
|
26
|
+
constructor({ videoConstraints, willCaptureFrequently, }?: Partial<DeviceCameraOptions>);
|
|
27
27
|
/**
|
|
28
28
|
* Retrieves the list of available video input devices.
|
|
29
29
|
* @returns A promise that resolves with an
|
package/build/core/Core.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import * as THREE from 'three';
|
|
|
2
2
|
import { AI } from '../ai/AI';
|
|
3
3
|
import { XRDeviceCamera } from '../camera/XRDeviceCamera';
|
|
4
4
|
import { Depth } from '../depth/Depth';
|
|
5
|
+
import { GestureRecognition } from '../input/gestures/GestureRecognition';
|
|
5
6
|
import { Input } from '../input/Input';
|
|
6
7
|
import { Lighting } from '../lighting/Lighting';
|
|
7
8
|
import { Physics } from '../physics/Physics';
|
|
@@ -49,7 +50,7 @@ export declare class Core {
|
|
|
49
50
|
/** The main camera for rendering. */
|
|
50
51
|
camera: THREE.PerspectiveCamera;
|
|
51
52
|
/** The root scene graph for all objects. */
|
|
52
|
-
scene: THREE.Scene
|
|
53
|
+
scene: THREE.Scene<THREE.Object3DEventMap>;
|
|
53
54
|
/** Represents the user in the XR scene. */
|
|
54
55
|
user: User;
|
|
55
56
|
/** Manages all UI elements. */
|
|
@@ -77,6 +78,7 @@ export declare class Core {
|
|
|
77
78
|
xrButton?: XRButton;
|
|
78
79
|
effects?: XREffects;
|
|
79
80
|
ai: AI;
|
|
81
|
+
gestureRecognition?: GestureRecognition;
|
|
80
82
|
transition?: XRTransition;
|
|
81
83
|
currentFrame?: XRFrame;
|
|
82
84
|
scriptsManager: ScriptsManager;
|
package/build/core/Options.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { AIOptions } from '../ai/AIOptions';
|
|
|
2
2
|
import { DeviceCameraOptions } from '../camera/CameraOptions.js';
|
|
3
3
|
import { DepthOptions } from '../depth/DepthOptions.js';
|
|
4
4
|
import { HandsOptions } from '../input/HandsOptions.js';
|
|
5
|
+
import { GestureRecognitionOptions } from '../input/gestures/GestureRecognitionOptions.js';
|
|
5
6
|
import { LightingOptions } from '../lighting/LightingOptions.js';
|
|
6
7
|
import { PhysicsOptions } from '../physics/PhysicsOptions';
|
|
7
8
|
import { SimulatorOptions } from '../simulator/SimulatorOptions';
|
|
@@ -78,6 +79,7 @@ export declare class Options {
|
|
|
78
79
|
lighting: LightingOptions;
|
|
79
80
|
deviceCamera: DeviceCameraOptions;
|
|
80
81
|
hands: HandsOptions;
|
|
82
|
+
gestures: GestureRecognitionOptions;
|
|
81
83
|
reticles: ReticleOptions;
|
|
82
84
|
sound: SoundOptions;
|
|
83
85
|
ai: AIOptions;
|
|
@@ -150,6 +152,11 @@ export declare class Options {
|
|
|
150
152
|
* @returns The instance for chaining.
|
|
151
153
|
*/
|
|
152
154
|
enableHands(): this;
|
|
155
|
+
/**
|
|
156
|
+
* Enables the gesture recognition block and ensures hands are available.
|
|
157
|
+
* @returns The instance for chaining.
|
|
158
|
+
*/
|
|
159
|
+
enableGestures(): this;
|
|
153
160
|
/**
|
|
154
161
|
* Enables the visualization of rays for hand tracking.
|
|
155
162
|
* @returns The instance for chaining.
|
|
@@ -20,7 +20,7 @@ export declare class ScreenshotSynthesizer {
|
|
|
20
20
|
* Requests a screenshot from the scene as a DataURL.
|
|
21
21
|
* @param overlayOnCamera - If true, overlays the image on a camera image
|
|
22
22
|
* without any projection or aspect ratio correction.
|
|
23
|
-
* @returns Promise which returns the screenshot.
|
|
23
|
+
* @returns Promise which returns the screenshot as a data uri.
|
|
24
24
|
*/
|
|
25
|
-
getScreenshot(overlayOnCamera?: boolean): Promise<
|
|
25
|
+
getScreenshot(overlayOnCamera?: boolean): Promise<string>;
|
|
26
26
|
}
|
|
@@ -35,7 +35,7 @@ export declare class XRTransition extends MeshScript<THREE.SphereGeometry, THREE
|
|
|
35
35
|
private targetAlpha;
|
|
36
36
|
private defaultBackgroundColor;
|
|
37
37
|
constructor();
|
|
38
|
-
init({ renderer, camera, timer, scene, options }: {
|
|
38
|
+
init({ renderer, camera, timer, scene, options, }: {
|
|
39
39
|
renderer: THREE.WebGLRenderer;
|
|
40
40
|
camera: THREE.Camera;
|
|
41
41
|
timer: THREE.Timer;
|
package/build/input/Hands.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as THREE from 'three';
|
|
2
2
|
import { HAND_JOINT_NAMES } from './components/HandJointNames.js';
|
|
3
|
-
type JointName = typeof HAND_JOINT_NAMES[number];
|
|
3
|
+
type JointName = (typeof HAND_JOINT_NAMES)[number];
|
|
4
4
|
/**
|
|
5
5
|
* Utility class for managing WebXR hand tracking data based on
|
|
6
6
|
* reported Handedness.
|
package/build/input/Input.d.ts
CHANGED
|
@@ -36,7 +36,7 @@ export declare class Input {
|
|
|
36
36
|
* Initializes an instance with XR controllers, grips, hands, raycaster, and
|
|
37
37
|
* default options. Only called by Core.
|
|
38
38
|
*/
|
|
39
|
-
init({ scene, options, renderer }: {
|
|
39
|
+
init({ scene, options, renderer, }: {
|
|
40
40
|
scene: THREE.Scene;
|
|
41
41
|
options: Options;
|
|
42
42
|
renderer: THREE.WebGLRenderer;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { BuiltInGestureName } from './GestureRecognitionOptions';
|
|
2
|
+
export type GestureEventType = 'gesturestart' | 'gestureupdate' | 'gestureend';
|
|
3
|
+
export type GestureHandedness = 'left' | 'right';
|
|
4
|
+
export interface GestureEventDetail {
|
|
5
|
+
/**
|
|
6
|
+
* The canonical gesture identifier. Built-in gestures map to
|
|
7
|
+
* `BuiltInGestureName` while custom providers may surface arbitrary strings.
|
|
8
|
+
*/
|
|
9
|
+
name: BuiltInGestureName | string;
|
|
10
|
+
/** Which hand triggered the gesture. */
|
|
11
|
+
hand: GestureHandedness;
|
|
12
|
+
/** Provider specific confidence score, normalized to [0, 1]. */
|
|
13
|
+
confidence: number;
|
|
14
|
+
/**
|
|
15
|
+
* Optional payload for provider specific values (e.g. pinch distance,
|
|
16
|
+
* velocity vectors).
|
|
17
|
+
*/
|
|
18
|
+
data?: Record<string, unknown>;
|
|
19
|
+
}
|
|
20
|
+
export interface GestureEvent {
|
|
21
|
+
type: GestureEventType;
|
|
22
|
+
detail: GestureEventDetail;
|
|
23
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import * as THREE from 'three';
|
|
2
|
+
import { Input } from '../Input';
|
|
3
|
+
import { User } from '../../core/User';
|
|
4
|
+
import { Script } from '../../core/Script';
|
|
5
|
+
import { GestureEventDetail, GestureEventType } from './GestureEvents';
|
|
6
|
+
import { GestureRecognitionOptions } from './GestureRecognitionOptions';
|
|
7
|
+
type GestureScriptEvent = THREE.Event & {
|
|
8
|
+
type: GestureEventType;
|
|
9
|
+
target: GestureRecognition;
|
|
10
|
+
detail: GestureEventDetail;
|
|
11
|
+
};
|
|
12
|
+
interface GestureRecognitionEventMap extends THREE.Object3DEventMap {
|
|
13
|
+
gesturestart: GestureScriptEvent;
|
|
14
|
+
gestureupdate: GestureScriptEvent;
|
|
15
|
+
gestureend: GestureScriptEvent;
|
|
16
|
+
}
|
|
17
|
+
export declare class GestureRecognition extends Script<GestureRecognitionEventMap> {
|
|
18
|
+
static dependencies: {
|
|
19
|
+
input: typeof Input;
|
|
20
|
+
user: typeof User;
|
|
21
|
+
options: typeof GestureRecognitionOptions;
|
|
22
|
+
};
|
|
23
|
+
private options;
|
|
24
|
+
private user;
|
|
25
|
+
private input;
|
|
26
|
+
private activeGestures;
|
|
27
|
+
private lastEvaluation;
|
|
28
|
+
private detectors;
|
|
29
|
+
private activeProvider;
|
|
30
|
+
private providerWarned;
|
|
31
|
+
init({ options, user, input, }: {
|
|
32
|
+
options: GestureRecognitionOptions;
|
|
33
|
+
user: User;
|
|
34
|
+
input: Input;
|
|
35
|
+
}): Promise<void>;
|
|
36
|
+
update(): void;
|
|
37
|
+
private configureProvider;
|
|
38
|
+
private assignDetectors;
|
|
39
|
+
private evaluateHand;
|
|
40
|
+
private buildHandContext;
|
|
41
|
+
private emitGesture;
|
|
42
|
+
}
|
|
43
|
+
export {};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { DeepPartial, DeepReadonly } from '../../utils/Types';
|
|
2
|
+
export type GestureProvider = 'heuristics' | 'mediapipe' | 'tfjs';
|
|
3
|
+
export type BuiltInGestureName = 'pinch' | 'open-palm' | 'fist' | 'thumbs-up' | 'point' | 'spread';
|
|
4
|
+
export type GestureConfiguration = {
|
|
5
|
+
enabled: boolean;
|
|
6
|
+
/**
|
|
7
|
+
* Optional override for gesture-specific score thresholds. For distance based
|
|
8
|
+
* gestures this is treated as a maximum distance; for confidence based
|
|
9
|
+
* gestures it is treated as a minimum score.
|
|
10
|
+
*/
|
|
11
|
+
threshold?: number;
|
|
12
|
+
};
|
|
13
|
+
export type GestureConfigurations = Partial<Record<BuiltInGestureName, Partial<GestureConfiguration>>>;
|
|
14
|
+
export declare class GestureRecognitionOptions {
|
|
15
|
+
/** Master switch for the gesture recognition block. */
|
|
16
|
+
enabled: boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Backing provider that extracts gesture information.
|
|
19
|
+
* - 'heuristics': WebXR joint heuristics only (no external ML dependency).
|
|
20
|
+
* - 'mediapipe': MediaPipe Hands running via Web APIs / wasm.
|
|
21
|
+
* - 'tfjs': TensorFlow.js hand-pose-detection models.
|
|
22
|
+
*/
|
|
23
|
+
provider: GestureProvider;
|
|
24
|
+
/**
|
|
25
|
+
* Minimum confidence score to emit gesture events. Different providers map to
|
|
26
|
+
* different score domains so this value is normalised to [0-1].
|
|
27
|
+
*/
|
|
28
|
+
minimumConfidence: number;
|
|
29
|
+
/**
|
|
30
|
+
* Optional throttle window for expensive providers.
|
|
31
|
+
*/
|
|
32
|
+
updateIntervalMs: number;
|
|
33
|
+
/**
|
|
34
|
+
* Default gesture catalogue.
|
|
35
|
+
*/
|
|
36
|
+
gestures: Record<BuiltInGestureName, GestureConfiguration>;
|
|
37
|
+
constructor(options?: DeepReadonly<DeepPartial<GestureRecognitionOptions>>);
|
|
38
|
+
enable(): this;
|
|
39
|
+
/**
|
|
40
|
+
* Convenience helper to toggle specific gestures.
|
|
41
|
+
*/
|
|
42
|
+
setGestureEnabled(name: BuiltInGestureName, enabled: boolean): this;
|
|
43
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import * as THREE from 'three';
|
|
2
|
+
import { Handedness } from '../Hands';
|
|
3
|
+
import { BuiltInGestureName, GestureConfiguration } from './GestureRecognitionOptions';
|
|
4
|
+
export type HandLabel = 'left' | 'right';
|
|
5
|
+
export type JointPositions = Map<string, THREE.Vector3>;
|
|
6
|
+
export type HandContext = {
|
|
7
|
+
handedness: Handedness;
|
|
8
|
+
handLabel: HandLabel;
|
|
9
|
+
joints: JointPositions;
|
|
10
|
+
};
|
|
11
|
+
export type GestureDetectionResult = {
|
|
12
|
+
confidence: number;
|
|
13
|
+
data?: Record<string, unknown>;
|
|
14
|
+
};
|
|
15
|
+
export type GestureDetector = (context: HandContext, config: GestureConfiguration) => GestureDetectionResult | undefined;
|
|
16
|
+
export type GestureDetectorMap = Partial<Record<BuiltInGestureName, GestureDetector>>;
|
|
@@ -49,6 +49,8 @@ export declare class Simulator extends Script {
|
|
|
49
49
|
mainScene: THREE.Scene;
|
|
50
50
|
private initialized;
|
|
51
51
|
private renderSimulatorSceneToCanvasBound;
|
|
52
|
+
private sparkRenderer?;
|
|
53
|
+
private registry?;
|
|
52
54
|
constructor(renderMainScene: (cameraOverride?: THREE.Camera) => void);
|
|
53
55
|
init({ simulatorOptions, input, timer, camera, renderer, scene, registry, options, depth, }: {
|
|
54
56
|
simulatorOptions: SimulatorOptions;
|
|
@@ -38,7 +38,7 @@ export declare class SimulatorControls {
|
|
|
38
38
|
/**
|
|
39
39
|
* Initialize the simulator controls.
|
|
40
40
|
*/
|
|
41
|
-
init({ camera, input, timer, renderer, simulatorOptions }: {
|
|
41
|
+
init({ camera, input, timer, renderer, simulatorOptions, }: {
|
|
42
42
|
camera: THREE.Camera;
|
|
43
43
|
input: Input;
|
|
44
44
|
timer: THREE.Timer;
|
|
@@ -18,7 +18,7 @@ export declare class PinchOnButtonAction extends SimulatorUserAction {
|
|
|
18
18
|
private timer;
|
|
19
19
|
private input;
|
|
20
20
|
constructor(target: THREE.Object3D);
|
|
21
|
-
init({ simulator, camera, timer, input }: {
|
|
21
|
+
init({ simulator, camera, timer, input, }: {
|
|
22
22
|
simulator: Simulator;
|
|
23
23
|
camera: THREE.Camera;
|
|
24
24
|
timer: THREE.Timer;
|
|
@@ -27,7 +27,7 @@ export declare class PinchOnButtonAction extends SimulatorUserAction {
|
|
|
27
27
|
controllerIsPointingAtButton(controls: SimulatorControls, camera: THREE.Camera): boolean;
|
|
28
28
|
rotateControllerTowardsButton(controls: SimulatorControls, camera: THREE.Camera, deltaTime: number): void;
|
|
29
29
|
pinchController(): void;
|
|
30
|
-
play({ simulatorUser, journeyId, waitFrame }: {
|
|
30
|
+
play({ simulatorUser, journeyId, waitFrame, }: {
|
|
31
31
|
simulatorUser: SimulatorUser;
|
|
32
32
|
journeyId: number;
|
|
33
33
|
waitFrame: WaitFrame;
|
|
@@ -23,7 +23,7 @@ export declare class WalkTowardsPanelAction extends SimulatorUserAction {
|
|
|
23
23
|
lookAtTarget(): void;
|
|
24
24
|
lookTowardsTarget(): void;
|
|
25
25
|
moveTowardsTarget(): void;
|
|
26
|
-
play({ simulatorUser, journeyId, waitFrame }: {
|
|
26
|
+
play({ simulatorUser, journeyId, waitFrame, }: {
|
|
27
27
|
simulatorUser: SimulatorUser;
|
|
28
28
|
journeyId: number;
|
|
29
29
|
waitFrame: WaitFrame;
|
package/build/singletons.d.ts
CHANGED
|
@@ -16,7 +16,7 @@ export declare const core: Core;
|
|
|
16
16
|
* scene.add(myObject);
|
|
17
17
|
* ```
|
|
18
18
|
*/
|
|
19
|
-
export declare const scene: THREE.Scene
|
|
19
|
+
export declare const scene: THREE.Scene<THREE.Object3DEventMap>;
|
|
20
20
|
/**
|
|
21
21
|
* A direct alias to the `User` instance, which represents the user in the XR
|
|
22
22
|
* scene and manages inputs like controllers and hands.
|
|
@@ -43,7 +43,7 @@ export declare const ai: import("./xrblocks").AI;
|
|
|
43
43
|
* @param object - The object(s) to add.
|
|
44
44
|
* @see {@link three#Object3D.add}
|
|
45
45
|
*/
|
|
46
|
-
export declare function add(...object: THREE.Object3D[]): THREE.Scene
|
|
46
|
+
export declare function add(...object: THREE.Object3D[]): THREE.Scene<THREE.Object3DEventMap>;
|
|
47
47
|
/**
|
|
48
48
|
* A shortcut for `core.init()`. Initializes the XR Blocks system and starts
|
|
49
49
|
* the render loop. This is the main entry point for any application.
|
|
@@ -24,7 +24,7 @@ export declare class CoreSound extends Script {
|
|
|
24
24
|
audioListener: AudioListener;
|
|
25
25
|
audioPlayer: AudioPlayer;
|
|
26
26
|
options: SoundOptions;
|
|
27
|
-
init({ camera, soundOptions }: {
|
|
27
|
+
init({ camera, soundOptions, }: {
|
|
28
28
|
camera: THREE.Camera;
|
|
29
29
|
soundOptions: SoundOptions;
|
|
30
30
|
}): void;
|
package/build/stereo/utils.d.ts
CHANGED
|
@@ -20,4 +20,4 @@ export declare function showOnlyInRightEye<T extends THREE.Object3D>(obj: T): T;
|
|
|
20
20
|
* @returns A promise that resolves to an array containing the left and right
|
|
21
21
|
* eye textures.
|
|
22
22
|
*/
|
|
23
|
-
export declare function loadStereoImageAsTextures(url: string): Promise<THREE.Texture[]>;
|
|
23
|
+
export declare function loadStereoImageAsTextures(url: string): Promise<THREE.Texture<unknown>[]>;
|
|
@@ -34,7 +34,7 @@ export declare class MaterialSymbolsView extends View {
|
|
|
34
34
|
* Construct a Material Symbol view.
|
|
35
35
|
* @param options - Options for the icon.
|
|
36
36
|
*/
|
|
37
|
-
constructor({ icon, iconWeight, iconStyle, iconScale, iconColor }: MaterialSymbolsViewOptions);
|
|
37
|
+
constructor({ icon, iconWeight, iconStyle, iconScale, iconColor, }: MaterialSymbolsViewOptions);
|
|
38
38
|
init(): Promise<void>;
|
|
39
39
|
/**
|
|
40
40
|
* Updates the icon displayed by loading the appropriate SVG from the Material
|
|
@@ -27,7 +27,7 @@ export declare class ScrollingTroikaTextView extends View {
|
|
|
27
27
|
private textView;
|
|
28
28
|
private onTextSyncCompleteBound;
|
|
29
29
|
private currentText;
|
|
30
|
-
constructor({ text, textAlign, scrollerState, fontSize }?: ScrollingTroikaTextViewOptions);
|
|
30
|
+
constructor({ text, textAlign, scrollerState, fontSize, }?: ScrollingTroikaTextViewOptions);
|
|
31
31
|
update(): void;
|
|
32
32
|
addText(text: string): void;
|
|
33
33
|
setText(text: string): void;
|
|
@@ -2,6 +2,7 @@ import * as THREE from 'three';
|
|
|
2
2
|
import { Script } from '../../core/Script';
|
|
3
3
|
import { Depth } from '../../depth/Depth';
|
|
4
4
|
import { Draggable, DragMode, HasDraggingMode } from '../../ux/DragManager';
|
|
5
|
+
import { Registry } from '../../core/components/Registry';
|
|
5
6
|
export interface GLTFData {
|
|
6
7
|
model: string;
|
|
7
8
|
path: string;
|
|
@@ -38,6 +39,7 @@ export declare class ModelViewer extends Script implements Draggable {
|
|
|
38
39
|
depth: typeof Depth;
|
|
39
40
|
scene: typeof THREE.Scene;
|
|
40
41
|
renderer: typeof THREE.WebGLRenderer;
|
|
42
|
+
registry: typeof Registry;
|
|
41
43
|
};
|
|
42
44
|
draggable: boolean;
|
|
43
45
|
rotatable: boolean;
|
|
@@ -65,16 +67,18 @@ export declare class ModelViewer extends Script implements Draggable {
|
|
|
65
67
|
private platform?;
|
|
66
68
|
private controlBar?;
|
|
67
69
|
private rotationRaycastMesh?;
|
|
70
|
+
private registry?;
|
|
68
71
|
constructor({ castShadow, receiveShadow, raycastToChildren, }: {
|
|
69
72
|
castShadow?: boolean | undefined;
|
|
70
73
|
receiveShadow?: boolean | undefined;
|
|
71
74
|
raycastToChildren?: boolean | undefined;
|
|
72
75
|
});
|
|
73
|
-
init({ camera, depth, scene, renderer }: {
|
|
76
|
+
init({ camera, depth, scene, renderer, registry, }: {
|
|
74
77
|
camera: THREE.Camera;
|
|
75
78
|
depth: Depth;
|
|
76
79
|
scene: THREE.Scene;
|
|
77
80
|
renderer: THREE.WebGLRenderer;
|
|
81
|
+
registry: Registry;
|
|
78
82
|
}): Promise<void>;
|
|
79
83
|
loadSplatModel({ data, onSceneLoaded, platformMargin, setupRaycastCylinder, setupRaycastBox, setupPlatform, }: {
|
|
80
84
|
data: SplatData;
|
|
@@ -84,7 +88,7 @@ export declare class ModelViewer extends Script implements Draggable {
|
|
|
84
88
|
setupRaycastBox?: boolean;
|
|
85
89
|
setupPlatform?: boolean;
|
|
86
90
|
}): Promise<void | SplatAnchor>;
|
|
87
|
-
loadGLTFModel({ data, onSceneLoaded, platformMargin, setupRaycastCylinder, setupRaycastBox, setupPlatform, renderer, addOcclusionToShader }: {
|
|
91
|
+
loadGLTFModel({ data, onSceneLoaded, platformMargin, setupRaycastCylinder, setupRaycastBox, setupPlatform, renderer, addOcclusionToShader, }: {
|
|
88
92
|
data: GLTFData;
|
|
89
93
|
onSceneLoaded?: (scene: THREE.Object3D) => void;
|
|
90
94
|
platformMargin?: THREE.Vector2;
|
|
@@ -34,7 +34,7 @@ export declare class ModelLoader {
|
|
|
34
34
|
* @returns A promise that resolves with the loaded model data (e.g., a glTF
|
|
35
35
|
* scene or a SplatMesh).
|
|
36
36
|
*/
|
|
37
|
-
load({ path, url, renderer, onProgress }: ModelLoaderLoadOptions): Promise<GLTF | import("@sparkjsdev/spark").SplatMesh | null>;
|
|
37
|
+
load({ path, url, renderer, onProgress, }: ModelLoaderLoadOptions): Promise<GLTF | import("@sparkjsdev/spark").SplatMesh | null>;
|
|
38
38
|
/**
|
|
39
39
|
* Loads a 3DGS model (.ply, .spz, .splat, .ksplat).
|
|
40
40
|
* @param url - The URL of the model file.
|