three-mediapipe-rig 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/meshcap/atlas-builder.d.ts +9 -0
- package/dist/meshcap/atlas-builder.d.ts.map +1 -1
- package/dist/meshcap/audio.d.ts +27 -0
- package/dist/meshcap/audio.d.ts.map +1 -0
- package/dist/meshcap/constants.d.ts +8 -1
- package/dist/meshcap/constants.d.ts.map +1 -1
- package/dist/meshcap/material.d.ts +40 -6
- package/dist/meshcap/material.d.ts.map +1 -1
- package/dist/meshcap/meshcap.d.ts +1 -0
- package/dist/meshcap/meshcap.d.ts.map +1 -1
- package/dist/meshcap/parse-mcap-file.d.ts +2 -1
- package/dist/meshcap/parse-mcap-file.d.ts.map +1 -1
- package/dist/meshcap/types.d.ts +59 -9
- package/dist/meshcap/types.d.ts.map +1 -1
- package/dist/meshcap/write-mcap-file.d.ts +13 -0
- package/dist/meshcap/write-mcap-file.d.ts.map +1 -1
- package/dist/meshcap.js +527 -271
- package/dist/rigger.js +440 -392
- package/dist/tracking/FaceTracker.d.ts +9 -2
- package/dist/tracking/FaceTracker.d.ts.map +1 -1
- package/dist/tracking/PoseTracker.d.ts +3 -0
- package/dist/tracking/PoseTracker.d.ts.map +1 -1
- package/dist/tracking/TrackerManager.d.ts +16 -7
- package/dist/tracking/TrackerManager.d.ts.map +1 -1
- package/dist/tracking/recoding/recorder.d.ts +1 -1
- package/dist/tracking/recoding/recorder.d.ts.map +1 -1
- package/package.json +11 -10
package/README.md
CHANGED
|
@@ -9,15 +9,20 @@ The motion from the webcam will be applied to a skeleton. Angle based so it work
|
|
|
9
9
|
|
|
10
10
|
This will run 3 models: face, body, hands. So expect a FPS drop.
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
-
|
|
14
|
-
- [
|
|
15
|
-
- [
|
|
12
|
+
## Explore the demos:
|
|
13
|
+
- Open the [PoseCap Editor](https://github.com/bandinopla/three-mediapipe-rig/blob/main/POSECAP.md), examples using it:
|
|
14
|
+
- [Bandinopla chibi](https://bandinopla.github.io/three-mediapipe-rig/?demo=bandinopla-chibi) ( pose + face )
|
|
15
|
+
- Open the [MeshCap Editor](https://github.com/bandinopla/three-mediapipe-rig/blob/main/MESHCAP.md), examples using it:
|
|
16
|
+
- [Loading .mcap files](https://bandinopla.github.io/three-mediapipe-rig/?demo=load-meshcap-files)
|
|
17
|
+
- [Clips with Audio: Memory game](https://bandinopla.github.io/three-mediapipe-rig/?demo=game-youtubers)
|
|
18
|
+
- [Characters](https://bandinopla.github.io/three-mediapipe-rig) ( pose + hands + face )
|
|
19
|
+
- [Hands Demo](https://bandinopla.github.io/three-mediapipe-rig/?demo=hands) ( pose + hands )
|
|
16
20
|
- [**Video to Face Geometry** !!](https://bandinopla.github.io/three-mediapipe-rig/?demo=face-uv)
|
|
17
21
|
|
|
18
22
|
---
|
|
19
23
|
|
|
20
24
|
## Table of Contents
|
|
25
|
+
- New! [**PoseCap Editor**](https://github.com/bandinopla/three-mediapipe-rig/blob/main/POSECAP.md)
|
|
21
26
|
- New! [**MeshCap Editor**](https://github.com/bandinopla/three-mediapipe-rig/blob/main/MESHCAP.md)
|
|
22
27
|
- [Features](#features)
|
|
23
28
|
- [Installation](#installation)
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Texture } from 'three';
|
|
1
2
|
import { MeshCapAtlas, RecordedClip } from './types';
|
|
2
3
|
/**
|
|
3
4
|
* Builds a texture atlas from a list of recorded clips.
|
|
@@ -7,4 +8,12 @@ import { MeshCapAtlas, RecordedClip } from './types';
|
|
|
7
8
|
* @returns The atlas
|
|
8
9
|
*/
|
|
9
10
|
export declare function buildMeshCapAtlas(clips: RecordedClip[], atlasSize: number, padding?: number): MeshCapAtlas;
|
|
11
|
+
/**
|
|
12
|
+
* Load an atlas from a URL or File. If will handle deobfuscation if the file is an .mcapatlas file.
|
|
13
|
+
* @param atlasSource URL or File of the atlas. If it is an .mcapatlas file, it will be deobfuscated.
|
|
14
|
+
* @param pass If the atlas is obfuscated, provide the passphrase to deobfuscate it.
|
|
15
|
+
* @returns
|
|
16
|
+
*/
|
|
17
|
+
export declare function loadMeshcapAtlas(atlasSource: string | File, pass?: string): Promise<Texture<HTMLImageElement>>;
|
|
18
|
+
export declare function obfuscateImage(image: HTMLImageElement, pass: string): Promise<Blob>;
|
|
10
19
|
//# sourceMappingURL=atlas-builder.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"atlas-builder.d.ts","sourceRoot":"","sources":["../../src/meshcap/atlas-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAY,YAAY,EAAW,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"atlas-builder.d.ts","sourceRoot":"","sources":["../../src/meshcap/atlas-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAiB,MAAM,OAAO,CAAC;AAC/C,OAAO,EAAE,YAAY,EAAY,YAAY,EAAW,MAAM,SAAS,CAAC;AAsHxE;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAE,KAAK,EAAC,YAAY,EAAE,EAAE,SAAS,EAAC,MAAM,EAAE,OAAO,GAAC,MAAQ,GAAG,YAAY,CA2HzG;AAmCD;;;;;GAKG;AACH,wBAAsB,gBAAgB,CAAE,WAAW,EAAC,MAAM,GAAC,IAAI,EAAE,IAAI,CAAC,EAAC,MAAM,sCAwC5E;AA+BD,wBAAsB,cAAc,CAAC,KAAK,EAAC,gBAAgB,EAAE,IAAI,EAAC,MAAM,iBAyBvE"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { MCapClip, RecordedClip } from './types';
|
|
2
|
+
export type AudioSpriteAtlas = {
|
|
3
|
+
blob: Blob;
|
|
4
|
+
sprites: ([start: number, duration: number] | undefined)[];
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Will complete the missing audio element on the clips by extracting the audio clip from the atlas.
|
|
8
|
+
*
|
|
9
|
+
* @param audioFile The audio file to extract sprites from.
|
|
10
|
+
* @param clips The clips to extract sprites for.
|
|
11
|
+
* @returns An array of audio elements, one for each clip ( undefined if no audio sprite is defined for that clip)
|
|
12
|
+
*/
|
|
13
|
+
export declare function extractAudioSprites(audioFile: File | Blob | string | ArrayBuffer, clips: RecordedClip[]): Promise<AudioSpriteAtlas>;
|
|
14
|
+
export declare function audioBufferToWav(buffer: AudioBuffer): ArrayBuffer;
|
|
15
|
+
export type AudioAtlasPlayer = {
|
|
16
|
+
stopCurrent: () => void;
|
|
17
|
+
playSprite: (clipIndex: number) => AudioBufferSourceNode | undefined;
|
|
18
|
+
setVolume: (value: number) => void;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Creates a player that can play the audio sprites of the clips.
|
|
22
|
+
* @param audioBuffer The audio buffer that contains the audio for the clips.
|
|
23
|
+
* @param clips The meshcap clips
|
|
24
|
+
* @returns A player that can play the audio sprites of the clips.
|
|
25
|
+
*/
|
|
26
|
+
export declare function createAudioAtlasPlayer(audioBuffer: AudioBuffer, clips: MCapClip[]): AudioAtlasPlayer;
|
|
27
|
+
//# sourceMappingURL=audio.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../../src/meshcap/audio.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEjD,MAAM,MAAM,gBAAgB,GAAG;IAC9B,IAAI,EAAC,IAAI,CAAA;IACT,OAAO,EAAE,CAAC,CAAC,KAAK,EAAC,MAAM,EAAE,QAAQ,EAAC,MAAM,CAAC,GAAC,SAAS,CAAC,EAAE,CAAA;CACtD,CAAA;AAmBD;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,SAAS,EAAC,IAAI,GAAG,IAAI,GAAG,MAAM,GAAG,WAAW,EAAE,KAAK,EAAC,YAAY,EAAE,GAAE,OAAO,CAAC,gBAAgB,CAAC,CA+CtI;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,CA+BjE;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC9B,WAAW,EAAC,MAAI,IAAI,CAAA;IACpB,UAAU,EAAC,CAAC,SAAS,EAAC,MAAM,KAAG,qBAAqB,GAAC,SAAS,CAAA;IAC9D,SAAS,EAAC,CAAC,KAAK,EAAE,MAAM,KAAG,IAAI,CAAA;CAC/B,CAAA;AAKD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAE,WAAW,EAAC,WAAW,EAAE,KAAK,EAAC,QAAQ,EAAE,GAAK,gBAAgB,CA+CrG"}
|
|
@@ -1,3 +1,10 @@
|
|
|
1
1
|
export declare const MCAP_MAGIC = 1296253264;
|
|
2
|
-
|
|
2
|
+
/**
|
|
3
|
+
* 2 - added audio + timestamps of the frames
|
|
4
|
+
* 3 - fix: clip's frames length from 1 byte to 2 bytes.
|
|
5
|
+
* + Also store frame deltas in 2 bytes instead of 1.
|
|
6
|
+
* + Made the frameCrop and landmarksCropUVs to be stored as round(v * 10000) instead of round(v * 1000)
|
|
7
|
+
* + added face transformation matrices (16 bytes per frame)
|
|
8
|
+
*/
|
|
9
|
+
export declare const MCAP_FILE_VERSION = 3;
|
|
3
10
|
//# sourceMappingURL=constants.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/meshcap/constants.ts"],"names":[],"mappings":"AACA,eAAO,MAAM,UAAU,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/meshcap/constants.ts"],"names":[],"mappings":"AACA,eAAO,MAAM,UAAU,aAAa,CAAC;AAErC;;;;;;GAMG;AACH,eAAO,MAAM,iBAAiB,IAAI,CAAC"}
|
|
@@ -1,24 +1,56 @@
|
|
|
1
1
|
import { Mesh, Texture } from 'three';
|
|
2
2
|
import { MCapClip } from './types';
|
|
3
|
-
import { NodeMaterial } from 'three/webgpu';
|
|
3
|
+
import { Matrix4, NodeMaterial } from 'three/webgpu';
|
|
4
4
|
export type MeshCapMaterialHandler = {
|
|
5
|
+
/**
|
|
6
|
+
* 4x4 transformation matrix from MediaPipe Face tracking.
|
|
7
|
+
* Represents the face pose in camera space (position, rotation, scale).
|
|
8
|
+
* Can be applied directly to 3D objects to align them with the tracked face.
|
|
9
|
+
*
|
|
10
|
+
* Returns the last known face transform matrix.
|
|
11
|
+
* Everytime you call `update` and the frames get evaluated, this will return the transformMatrix the face has in the
|
|
12
|
+
* current frame. You can use this to position hats, hair, etc. Remember this transformation is from the face's origin perspective.
|
|
13
|
+
* So if you do add a hat, make sure the origin point of it is at the same place as the origin point of the face mesh so the rotations look alright.
|
|
14
|
+
* @returns
|
|
15
|
+
*/
|
|
16
|
+
getLastKnownFaceTransform: () => Matrix4 | undefined;
|
|
17
|
+
/**
|
|
18
|
+
* If it should play a sound or not (if it has one. Default: false )
|
|
19
|
+
*/
|
|
20
|
+
muted: boolean;
|
|
21
|
+
/**
|
|
22
|
+
* This hook will be called everytime a clip starts playing.
|
|
23
|
+
* @param clipIndex Index of the clip that is starting to play
|
|
24
|
+
* @param clipStartTime Start time of sound clip withing the sound atlas (in seconds)
|
|
25
|
+
* @param clipDuration Duration of the clip (in seconds)
|
|
26
|
+
* @returns
|
|
27
|
+
*/
|
|
28
|
+
playClipAudioHook?: (clipIndex: number, clipStartTime: number, clipDuration: number) => void;
|
|
5
29
|
/**
|
|
6
30
|
* Moves to a particular clip
|
|
7
31
|
* @param clipIndex The index of the clip to move to
|
|
32
|
+
* @param _loop Optional: Whether the clip should loop
|
|
33
|
+
* @param _onEndReached Optional: Callback to be called when the clip reaches the end
|
|
8
34
|
*/
|
|
9
|
-
goto: (clipIndex: number | string, _loop?: boolean) => void;
|
|
35
|
+
goto: (clipIndex: number | string, _loop?: boolean, _onEndOrLoopReached?: (timeOffset: number) => void, playSound?: boolean) => void;
|
|
10
36
|
/**
|
|
11
37
|
* Play a clip and when it reaches the end, it will loop back
|
|
12
38
|
* @param clipName Name of the clip
|
|
13
39
|
* @returns
|
|
14
40
|
*/
|
|
15
|
-
gotoAndLoop: (clipIndex: number | string) => void;
|
|
41
|
+
gotoAndLoop: (clipIndex: number | string, _onLoop?: (timeOffset: number) => void) => void;
|
|
16
42
|
/**
|
|
17
43
|
* Play a clip and when it reaches the end, it will not loop back
|
|
18
44
|
* @param clipName Name of the clip
|
|
19
45
|
* @returns
|
|
20
46
|
*/
|
|
21
|
-
gotoAndPlay: (clipIndex: number | string) => void;
|
|
47
|
+
gotoAndPlay: (clipIndex: number | string, _onEndReached?: () => void) => void;
|
|
48
|
+
/**
|
|
49
|
+
* Moves to a particular clip and stops at the first frame
|
|
50
|
+
* @param clipIndex The index of the clip to move to
|
|
51
|
+
* @returns
|
|
52
|
+
*/
|
|
53
|
+
gotoAndStop: (clipIndex: number | string, frame?: number) => void;
|
|
22
54
|
/**
|
|
23
55
|
* Updates the material with the given delta time
|
|
24
56
|
* @param delta The time to add to the current time
|
|
@@ -43,11 +75,13 @@ export type MeshCapMaterialHandler = {
|
|
|
43
75
|
};
|
|
44
76
|
/**
|
|
45
77
|
* Creates or setups a MeshCap material handler (not the material itself) for a given mesh.
|
|
46
|
-
*
|
|
78
|
+
*
|
|
79
|
+
* @param atlasTexture The texture atlas that contains the frames used by the clips. You may need to `flipY=false` on the texture atlas.
|
|
47
80
|
* @param clips The clips previously obtained by loading an .mcap file
|
|
48
81
|
* @param targetMesh The mesh to apply the material to. (It will be updated with a landmarkIndex attribute if it doesn't have one)
|
|
49
82
|
* @param host Optional: The material to use as a base. Defaults to a MeshPhysicalNodeMaterial.
|
|
83
|
+
* @param audioAtlas Optional: The audio atlas that contains the audio for the clips if you want to let the handler play the audio clips automatically on it's own. Else you will have to hook on the `playClipAudioHook` callback from the returned handler and play them yourself.
|
|
50
84
|
* @returns A handler that allows you to control the material.
|
|
51
85
|
*/
|
|
52
|
-
export declare function createMeshCapMaterial(atlasTexture: Texture, clips: MCapClip[], targetMesh: Mesh, host?: NodeMaterial): MeshCapMaterialHandler;
|
|
86
|
+
export declare function createMeshCapMaterial(atlasTexture: Texture, clips: MCapClip[], targetMesh: Mesh, host?: NodeMaterial, audioAtlas?: AudioBuffer): MeshCapMaterialHandler;
|
|
53
87
|
//# sourceMappingURL=material.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"material.d.ts","sourceRoot":"","sources":["../../src/meshcap/material.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAA4B,YAAY,EAAW,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"material.d.ts","sourceRoot":"","sources":["../../src/meshcap/material.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,OAAO,EAA4B,YAAY,EAAW,MAAM,cAAc,CAAC;AAOxF,MAAM,MAAM,sBAAsB,GAAG;IAEpC;;;;;;;;;;OAUG;IACH,yBAAyB,EAAC,MAAI,OAAO,GAAC,SAAS,CAAC;IAEhD;;OAEG;IACH,KAAK,EAAC,OAAO,CAAC;IAEd;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAC,CAAC,SAAS,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,EAAE,YAAY,EAAC,MAAM,KAAG,IAAI,CAAC;IAGvF;;;;;OAKG;IACH,IAAI,EAAC,CAAE,SAAS,EAAC,MAAM,GAAC,MAAM,EAAE,KAAK,CAAC,EAAC,OAAO,EAAE,mBAAmB,CAAC,EAAC,CAAE,UAAU,EAAC,MAAM,KAAI,IAAI,EAAE,SAAS,CAAC,EAAC,OAAO,KAAI,IAAI,CAAA;IAE5H;;;;OAIG;IACH,WAAW,EAAC,CAAE,SAAS,EAAC,MAAM,GAAC,MAAM,EAAE,OAAO,CAAC,EAAC,CAAE,UAAU,EAAC,MAAM,KAAI,IAAI,KAAI,IAAI,CAAA;IAEnF;;;;OAIG;IACH,WAAW,EAAC,CAAE,SAAS,EAAC,MAAM,GAAC,MAAM,EAAE,aAAa,CAAC,EAAC,MAAI,IAAI,KAAI,IAAI,CAAA;IAEtE;;;;OAIG;IACH,WAAW,EAAC,CAAE,SAAS,EAAC,MAAM,GAAC,MAAM,EAAE,KAAK,CAAC,EAAC,MAAM,KAAI,IAAI,CAAA;IAE5D;;;OAGG;IACH,MAAM,EAAC,CAAE,KAAK,EAAC,MAAM,KAAI,IAAI,CAAA;IAE7B;;OAEG;IACH,KAAK,EAAC,QAAQ,EAAE,CAAA;IAEhB;;OAEG;IACH,YAAY,EAAC,OAAO,CAAA;IAEpB;;OAEG;IACH,OAAO,EAAC,YAAY,CAAA;IAEpB;;OAEG;IACH,QAAQ,EAAC,YAAY,CAAA;CACrB,CAAA;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CAAE,YAAY,EAAC,OAAO,EAAE,KAAK,EAAC,QAAQ,EAAE,EAAE,UAAU,EAAC,IAAI,EAAE,IAAI,CAAC,EAAC,YAAY,EAAE,UAAU,CAAC,EAAC,WAAW,GAAG,sBAAsB,CAmUnK"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"meshcap.d.ts","sourceRoot":"","sources":["../../src/meshcap/meshcap.ts"],"names":[],"mappings":"AAAA,mBAAmB,SAAS,CAAA;AAC5B,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,YAAY,CAAA"}
|
|
1
|
+
{"version":3,"file":"meshcap.d.ts","sourceRoot":"","sources":["../../src/meshcap/meshcap.ts"],"names":[],"mappings":"AAAA,mBAAmB,SAAS,CAAA;AAC5B,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,YAAY,CAAA;AAC1B,cAAc,SAAS,CAAA"}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import { MCapFile } from './types';
|
|
1
|
+
import { MCapFile, MeshCapAtlas } from './types';
|
|
2
2
|
/**
|
|
3
3
|
* Loads a MeshCap (.mcap) file from a URL or File object. This is the file that contains the metadata for the clips.
|
|
4
4
|
* @param mcapFileSource URL or File object
|
|
5
5
|
* @returns
|
|
6
6
|
*/
|
|
7
7
|
export declare function loadMeshCapFile(mcapFileSource: string | File): Promise<MCapFile>;
|
|
8
|
+
export declare function readAsMCapFile(atlas: MeshCapAtlas): Promise<MCapFile>;
|
|
8
9
|
//# sourceMappingURL=parse-mcap-file.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-mcap-file.d.ts","sourceRoot":"","sources":["../../src/meshcap/parse-mcap-file.ts"],"names":[],"mappings":"AACA,OAAO,EAAY,QAAQ,EAAyB,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"parse-mcap-file.d.ts","sourceRoot":"","sources":["../../src/meshcap/parse-mcap-file.ts"],"names":[],"mappings":"AACA,OAAO,EAAY,QAAQ,EAAE,YAAY,EAAyB,MAAM,SAAS,CAAC;AAUlF;;;;GAIG;AACH,wBAAsB,eAAe,CAAE,cAAc,EAAC,MAAM,GAAC,IAAI,GAAK,OAAO,CAAC,QAAQ,CAAC,CAmBtF;AAgVD,wBAAsB,cAAc,CAAE,KAAK,EAAC,YAAY,qBAIvD"}
|
package/dist/meshcap/types.d.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import { Vector3Like } from 'three';
|
|
1
|
+
import { Mesh, Texture, Vector3Like } from 'three';
|
|
2
|
+
import { AudioSpriteAtlas } from './audio';
|
|
3
|
+
import { MeshCapMaterialHandler } from './material';
|
|
4
|
+
import { Matrix4, NodeMaterial } from 'three/webgpu';
|
|
2
5
|
export interface UVCoord {
|
|
3
6
|
u: number;
|
|
4
7
|
v: number;
|
|
@@ -11,18 +14,41 @@ export interface Clip {
|
|
|
11
14
|
landmarks: Vector3Like[][];
|
|
12
15
|
scale: number;
|
|
13
16
|
aspectRatio: number;
|
|
17
|
+
audioSprite?: {
|
|
18
|
+
start: number;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Duration of this clip (in seconds)
|
|
22
|
+
*/
|
|
23
|
+
duration: number;
|
|
14
24
|
}
|
|
25
|
+
export type FrameBase = {
|
|
26
|
+
/**
|
|
27
|
+
* This is the bounding box in atlas UV space in which the landmarks are expressed.
|
|
28
|
+
*/
|
|
29
|
+
cropUV: UVCoord;
|
|
30
|
+
startTime: number;
|
|
31
|
+
/**
|
|
32
|
+
* The general transformation of the face in this frame.
|
|
33
|
+
*/
|
|
34
|
+
transformMatrix?: Matrix4;
|
|
35
|
+
};
|
|
15
36
|
export interface RecordedClip extends Clip {
|
|
16
|
-
frames: {
|
|
37
|
+
frames: (FrameBase & {
|
|
17
38
|
canvas: HTMLCanvasElement;
|
|
18
|
-
|
|
19
|
-
|
|
39
|
+
})[];
|
|
40
|
+
audioSprite?: {
|
|
41
|
+
domElement?: HTMLAudioElement;
|
|
42
|
+
/**
|
|
43
|
+
* Start time in the audio sprite atlas (seconds)
|
|
44
|
+
*/
|
|
45
|
+
start: number;
|
|
46
|
+
};
|
|
20
47
|
}
|
|
21
48
|
export interface MCapClip extends Clip {
|
|
22
|
-
frames: {
|
|
23
|
-
cropUV: UVCoord;
|
|
49
|
+
frames: (FrameBase & {
|
|
24
50
|
frameUV: UVCoord;
|
|
25
|
-
}[];
|
|
51
|
+
})[];
|
|
26
52
|
}
|
|
27
53
|
export interface MCapFile {
|
|
28
54
|
clips: MCapClip[];
|
|
@@ -31,9 +57,21 @@ export interface MCapFile {
|
|
|
31
57
|
atlasPadding: number;
|
|
32
58
|
/**
|
|
33
59
|
* Extract the clips from the atlas image using the metadata as a guide to know where the clips are.
|
|
60
|
+
* If the clips use a sound atlas, and one is provided, their audio clips will be reconstructed.
|
|
34
61
|
* @param atlas
|
|
35
62
|
*/
|
|
36
|
-
unpackClips: (atlas: File | string | HTMLImageElement | HTMLCanvasElement) => Promise<
|
|
63
|
+
unpackClips: (atlas: File | string | HTMLImageElement | HTMLCanvasElement, audioFile?: File | Blob | string | ArrayBuffer) => Promise<{
|
|
64
|
+
clips: RecordedClip[];
|
|
65
|
+
audioAtlas?: AudioSpriteAtlas;
|
|
66
|
+
}>;
|
|
67
|
+
/**
|
|
68
|
+
* Creates a material on the mesh that updates its texture to match the clip frames.
|
|
69
|
+
* If you pass `audioAtlas`, the handler will play the audio for the clip when it is played.
|
|
70
|
+
* @param mesh
|
|
71
|
+
* @param atlasTexture
|
|
72
|
+
* @returns
|
|
73
|
+
*/
|
|
74
|
+
createMaterialHandlerOnMesh: (mesh: Mesh, atlasTexture: Texture, host?: NodeMaterial, audioAtlas?: AudioBuffer) => MeshCapMaterialHandler;
|
|
37
75
|
}
|
|
38
76
|
export interface MeshCapAtlas {
|
|
39
77
|
canvas: HTMLCanvasElement;
|
|
@@ -46,8 +84,20 @@ export interface MeshCapAtlas {
|
|
|
46
84
|
*/
|
|
47
85
|
padding: number;
|
|
48
86
|
/**
|
|
49
|
-
*
|
|
87
|
+
* prefered max atlas dimension
|
|
88
|
+
*/
|
|
89
|
+
atlasSize: number;
|
|
90
|
+
/**
|
|
91
|
+
* Save the cuyrrent meshcap as a .mcap file.
|
|
92
|
+
* @param downloadFile If true, the file will be downloaded to the user's computer.
|
|
93
|
+
* @returns
|
|
50
94
|
*/
|
|
51
95
|
save(downloadFile: boolean): Promise<Blob>;
|
|
96
|
+
/**
|
|
97
|
+
* Triggers a download of the atlas texture
|
|
98
|
+
* @param phrase Optional phrase to use to perform a simple obfuscation of the texture.
|
|
99
|
+
* If the phrase is not provided, the atlas will not be obfuscated. If you use one, you will have to provide it once again on your app or game when you try to load it calling `createMaterialHandlerOnMesh` by defining the `onAtlasKeyPhraseRequired` hook, which is a functionthat should return the phrase.
|
|
100
|
+
*/
|
|
101
|
+
saveImageAtlas(phrase?: string, asJpg?: boolean): Promise<void>;
|
|
52
102
|
}
|
|
53
103
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/meshcap/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAA;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/meshcap/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAA;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAIpD,MAAM,WAAW,OAAO;IAAE,CAAC,EAAC,MAAM,CAAC;IAAA,CAAC,EAAC,MAAM,CAAC;IAAA,CAAC,EAAC,MAAM,CAAC;IAAA,CAAC,EAAC,MAAM,CAAA;CAAC;AAE9D,MAAM,WAAW,IAAI;IACpB,GAAG,EAAC,MAAM,CAAC;IACX,IAAI,EAAC,MAAM,CAAC;IACZ,SAAS,EAAC,WAAW,EAAE,EAAE,CAAC;IAC1B,KAAK,EAAC,MAAM,CAAA;IACZ,WAAW,EAAC,MAAM,CAAC;IACnB,WAAW,CAAC,EAAC;QACZ,KAAK,EAAC,MAAM,CAAC;KACb,CAAA;IAED;;OAEG;IACH,QAAQ,EAAC,MAAM,CAAA;CACf;AAGD,MAAM,MAAM,SAAS,GAAG;IAEvB;;OAEG;IACH,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAC,MAAM,CAAC;IAEjB;;OAEG;IACH,eAAe,CAAC,EAAC,OAAO,CAAA;CACxB,CAAA;AAED,MAAM,WAAW,YAAa,SAAQ,IAAI;IACzC,MAAM,EAAC,CAAC,SAAS,GAAG;QACnB,MAAM,EAAC,iBAAiB,CAAC;KACzB,CAAC,EAAE,CAAC;IACL,WAAW,CAAC,EAAC;QACZ,UAAU,CAAC,EAAC,gBAAgB,CAAC;QAE7B;;WAEG;QACH,KAAK,EAAC,MAAM,CAAC;KACb,CAAA;CACD;AAED,MAAM,WAAW,QAAS,SAAQ,IAAI;IACrC,MAAM,EAAE,CAAC,SAAS,GAAG;QACpB,OAAO,EAAC,OAAO,CAAC;KAChB,CAAC,EAAE,CAAA;CACJ;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAC,MAAM,CAAC;IACjB,YAAY,EAAC,MAAM,CAAC;IAEpB;;;;OAIG;IACH,WAAW,EAAE,CAAC,KAAK,EAAC,IAAI,GAAC,MAAM,GAAC,gBAAgB,GAAC,iBAAiB,EAAE,SAAS,CAAC,EAAC,IAAI,GAAC,IAAI,GAAC,MAAM,GAAC,WAAW,KAAG,OAAO,CAAC;QAAC,KAAK,EAAC,YAAY,EAAE,CAAC;QAAC,UAAU,CAAC,EAAC,gBAAgB,CAAA;KAAC,CAAC,CAAC;IAE5K;;;;;;OAMG;IACH,2BAA2B,EAAE,CAAC,IAAI,EAAC,IAAI,EAAE,YAAY,EAAC,OAAO,EAAE,IAAI,CAAC,EAAC,YAAY,EAAE,UAAU,CAAC,EAAC,WAAW,KAAG,sBAAsB,CAAC;CACpI;AAED,MAAM,WAAW,YAAY;IACzB,MAAM,EAAE,iBAAiB,CAAC;IAE7B;;OAEG;IACA,KAAK,EAAE,QAAQ,EAAE,CAAC;IAErB;;OAEG;IACH,OAAO,EAAC,MAAM,CAAA;IAEd;;OAEG;IACH,SAAS,EAAC,MAAM,CAAA;IAEhB;;;;OAIG;IACH,IAAI,CAAE,YAAY,EAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IAE1C;;;;OAIG;IACH,cAAc,CAAE,MAAM,CAAC,EAAC,MAAM,EAAE,KAAK,CAAC,EAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9D"}
|
|
@@ -1,3 +1,16 @@
|
|
|
1
1
|
import { MeshCapAtlas } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* generates an .mcap binary from the atlas
|
|
4
|
+
* @param atlas
|
|
5
|
+
* @param useRelativeLandmarks Used internally to save space.
|
|
6
|
+
* @returns
|
|
7
|
+
*/
|
|
2
8
|
export declare function atlasToMCap(atlas: MeshCapAtlas, useRelativeLandmarks?: boolean): Promise<Blob>;
|
|
9
|
+
/**
|
|
10
|
+
* generates an .mcap array buffer. Used for simulating ( in the meshcap editor ) opening the file in a real life situation
|
|
11
|
+
* @param atlas
|
|
12
|
+
* @param useRelativeLandmarks Used internally to save space.
|
|
13
|
+
* @returns
|
|
14
|
+
*/
|
|
15
|
+
export declare function atlasToMCapBuffer(atlas: MeshCapAtlas, useRelativeLandmarks?: boolean): Promise<ArrayBuffer>;
|
|
3
16
|
//# sourceMappingURL=write-mcap-file.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"write-mcap-file.d.ts","sourceRoot":"","sources":["../../src/meshcap/write-mcap-file.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAIvC,wBAAsB,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,oBAAoB,GAAC,OAAc,
|
|
1
|
+
{"version":3,"file":"write-mcap-file.d.ts","sourceRoot":"","sources":["../../src/meshcap/write-mcap-file.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAIvC;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,oBAAoB,GAAC,OAAc,iBAczF;AAED;;;;;GAKG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,YAAY,EAAE,oBAAoB,GAAC,OAAc,wBA4N/F"}
|