three-mediapipe-rig 0.1.1 → 0.1.3

package/README.md

@@ -9,14 +9,18 @@ The motion from the webcam will be applied to a skeleton. Angle based so it work
 
 This will run 3 models: face, body, hands. So expect a FPS drop.
 
-Expolore the demos:
-- [Characters](https://bandinopla.github.io/three-mediapipe-rig) 
-- [Hands Demo](https://bandinopla.github.io/three-mediapipe-rig/?demo=hands)
+## Explore the demos:
+- Open the [MeshCap Editor](https://github.com/bandinopla/three-mediapipe-rig/blob/main/MESHCAP.md), examples using it:
+	- [Loading .mcap files](https://bandinopla.github.io/three-mediapipe-rig/?demo=load-meshcap-files)
+	- [Clips with Audio: Memory game](https://bandinopla.github.io/three-mediapipe-rig/?demo=game-youtubers)
+- [Characters](https://bandinopla.github.io/three-mediapipe-rig)  ( pose + hands + face )
+- [Hands Demo](https://bandinopla.github.io/three-mediapipe-rig/?demo=hands) ( pose + hands )
 - [**Video to Face Geometry** !!](https://bandinopla.github.io/three-mediapipe-rig/?demo=face-uv)
+
 ---
 
 ## Table of Contents
-
+- New! [**MeshCap Editor**](https://github.com/bandinopla/three-mediapipe-rig/blob/main/MESHCAP.md)
 - [Features](#features)
 - [Installation](#installation)
 - [Quick Start](#quick-start)

package/dist/face-tracker-utils-xt9__vBF.js

@@ -0,0 +1,16 @@
+import { BufferAttribute as c } from "three";
+const d = 478, g = (r) => {
+  if (r.geometry.hasAttribute("landmarkIndex")) return;
+  const s = r.geometry, e = s.attributes.position, a = [], n = /* @__PURE__ */ new Map(), i = [];
+  for (let t = 0; t < e.count; t++) {
+    const o = Math.round(e.getX(t) * 1e6) + "," + Math.round(e.getY(t) * 1e6) + "," + Math.round(e.getZ(t) * 1e6);
+    n.has(o) || (n.set(o, a.length), a.push(t));
+    const u = n.get(o);
+    i.push(u < d ? u : 65535);
+  }
+  s.setAttribute("landmarkIndex", new c(new Uint16Array(i), 1));
+};
+export {
+  d as F,
+  g as c
+};

package/dist/meshcap/atlas-builder.d.ts

@@ -0,0 +1,10 @@
+import { MeshCapAtlas, RecordedClip } from './types';
+/**
+ * Builds a texture atlas from a list of recorded clips.
+ * @param clips The clips to build the atlas from
+ * @param atlasSize Max size of the atlas (width or height)
+ * @param padding The padding to add between frames
+ * @returns The atlas
+ */
+export declare function buildMeshCapAtlas(clips: RecordedClip[], atlasSize: number, padding?: number): MeshCapAtlas;
+//# sourceMappingURL=atlas-builder.d.ts.map

package/dist/meshcap/atlas-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"atlas-builder.d.ts","sourceRoot":"","sources":["../../src/meshcap/atlas-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAY,YAAY,EAAW,MAAM,SAAS,CAAC;AAsNxE;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAE,KAAK,EAAC,YAAY,EAAE,EAAE,SAAS,EAAC,MAAM,EAAE,OAAO,GAAC,MAAQ,GAAG,YAAY,CAwGzG"}

package/dist/meshcap/audio.d.ts

@@ -0,0 +1,27 @@
+import { MCapClip, RecordedClip } from './types';
+export type AudioSpriteAtlas = {
+    blob: Blob;
+    sprites: ([start: number, duration: number] | undefined)[];
+};
+/**
+ * Will complete the missing audio element on the clips by extracting the audio clip from the atlas.
+ *
+ * @param audioFile The audio file to extract sprites from.
+ * @param clips The clips to extract sprites for.
+ * @returns An array of audio elements, one for each clip ( undefined if no audio sprite is defined for that clip)
+ */
+export declare function extractAudioSprites(audioFile: File | Blob | string | ArrayBuffer, clips: RecordedClip[]): Promise<AudioSpriteAtlas>;
+export declare function audioBufferToWav(buffer: AudioBuffer): ArrayBuffer;
+export type AudioAtlasPlayer = {
+    stopCurrent: () => void;
+    playSprite: (clipIndex: number) => AudioBufferSourceNode | undefined;
+    setVolume: (value: number) => void;
+};
+/**
+ * Creates a player that can play the audio sprites of the clips.
+ * @param audioBuffer The audio buffer that contains the audio for the clips.
+ * @param clips The meshcap clips
+ * @returns A player that can play the audio sprites of the clips.
+ */
+export declare function createAudioAtlasPlayer(audioBuffer: AudioBuffer, clips: MCapClip[]): AudioAtlasPlayer;
+//# sourceMappingURL=audio.d.ts.map

package/dist/meshcap/audio.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio.d.ts","sourceRoot":"","sources":["../../src/meshcap/audio.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEjD,MAAM,MAAM,gBAAgB,GAAG;IAC9B,IAAI,EAAC,IAAI,CAAA;IACT,OAAO,EAAE,CAAC,CAAC,KAAK,EAAC,MAAM,EAAE,QAAQ,EAAC,MAAM,CAAC,GAAC,SAAS,CAAC,EAAE,CAAA;CACtD,CAAA;AAmBD;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,SAAS,EAAC,IAAI,GAAG,IAAI,GAAG,MAAM,GAAG,WAAW,EAAE,KAAK,EAAC,YAAY,EAAE,GAAE,OAAO,CAAC,gBAAgB,CAAC,CA+CtI;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,CA+BjE;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC9B,WAAW,EAAC,MAAI,IAAI,CAAA;IACpB,UAAU,EAAC,CAAC,SAAS,EAAC,MAAM,KAAG,qBAAqB,GAAC,SAAS,CAAA;IAC9D,SAAS,EAAC,CAAC,KAAK,EAAE,MAAM,KAAG,IAAI,CAAA;CAC/B,CAAA;AAKD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAE,WAAW,EAAC,WAAW,EAAE,KAAK,EAAC,QAAQ,EAAE,GAAK,gBAAgB,CA+CrG"}

package/dist/meshcap/constants.d.ts

@@ -0,0 +1,3 @@
+export declare const MCAP_MAGIC = 1296253264;
+export declare const MCAP_FILE_VERSION = 2;
+//# sourceMappingURL=constants.d.ts.map

package/dist/meshcap/constants.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/meshcap/constants.ts"],"names":[],"mappings":"AACA,eAAO,MAAM,UAAU,aAAa,CAAC;AACrC,eAAO,MAAM,iBAAiB,IAAI,CAAC"}

package/dist/meshcap/material.d.ts

@@ -0,0 +1,75 @@
+import { Mesh, Texture } from 'three';
+import { MCapClip } from './types';
+import { NodeMaterial } from 'three/webgpu';
+export type MeshCapMaterialHandler = {
+    /**
+     * If it should play a sound or not (if it has one. Default: false )
+     */
+    muted: boolean;
+    /**
+     * This hook will be called everytime a clip starts playing.
+     * @param clipIndex Index of the clip that is starting to play
+     * @param clipStartTime Start time of sound clip withing the sound atlas (in seconds)
+     * @param clipDuration Duration of the clip (in seconds)
+     * @returns
+     */
+    playClipAudioHook?: (clipIndex: number, clipStartTime: number, clipDuration: number) => void;
+    /**
+     * Moves to a particular clip
+     * @param clipIndex The index of the clip to move to
+     * @param _loop Optional: Whether the clip should loop
+     * @param _onEndReached Optional: Callback to be called when the clip reaches the end
+     */
+    goto: (clipIndex: number | string, _loop?: boolean, _onEndOrLoopReached?: (timeOffset: number) => void, playSound?: boolean) => void;
+    /**
+     * Play a clip and when it reaches the end, it will loop back
+     * @param clipName Name of the clip
+     * @returns
+     */
+    gotoAndLoop: (clipIndex: number | string, _onLoop?: (timeOffset: number) => void) => void;
+    /**
+     * Play a clip and when it reaches the end, it will not loop back
+     * @param clipName Name of the clip
+     * @returns
+     */
+    gotoAndPlay: (clipIndex: number | string, _onEndReached?: () => void) => void;
+    /**
+     * Moves to a particular clip and stops at the first frame
+     * @param clipIndex The index of the clip to move to
+     * @returns
+     */
+    gotoAndStop: (clipIndex: number | string, frame?: number) => void;
+    /**
+     * Updates the material with the given delta time
+     * @param delta The time to add to the current time
+     */
+    update: (delta: number) => void;
+    /**
+     * The clips available to play
+     */
+    clips: MCapClip[];
+    /**
+     * The texture atlas that contains the frames used by the clips
+     */
+    atlasTexture: Texture;
+    /**
+     * Disposes the material and the texture atlas
+     */
+    dispose: VoidFunction;
+    /**
+     * The material used
+     */
+    material: NodeMaterial;
+};
+/**
+ * Creates or setups a MeshCap material handler (not the material itself) for a given mesh.
+ *
+ * @param atlasTexture The texture atlas that contains the frames used by the clips. You may need to `flipY=false` on the texture atlas.
+ * @param clips The clips previously obtained by loading an .mcap file
+ * @param targetMesh The mesh to apply the material to. (It will be updated with a landmarkIndex attribute if it doesn't have one)
+ * @param host Optional: The material to use as a base. Defaults to a MeshPhysicalNodeMaterial.
+ * @param audioAtlas Optional: The audio atlas that contains the audio for the clips if you want to let the handler play the audio clips automatically on it's own. Else you will have to hook on the `playClipAudioHook` callback from the returned handler and play them yourself.
+ * @returns A handler that allows you to control the material.
+ */
+export declare function createMeshCapMaterial(atlasTexture: Texture, clips: MCapClip[], targetMesh: Mesh, host?: NodeMaterial, audioAtlas?: AudioBuffer): MeshCapMaterialHandler;
+//# sourceMappingURL=material.d.ts.map

package/dist/meshcap/material.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"material.d.ts","sourceRoot":"","sources":["../../src/meshcap/material.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAA4B,YAAY,EAAW,MAAM,cAAc,CAAC;AAO/E,MAAM,MAAM,sBAAsB,GAAG;IAEpC;;OAEG;IACH,KAAK,EAAC,OAAO,CAAC;IAEd;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAC,CAAC,SAAS,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,EAAE,YAAY,EAAC,MAAM,KAAG,IAAI,CAAC;IAGvF;;;;;OAKG;IACH,IAAI,EAAC,CAAE,SAAS,EAAC,MAAM,GAAC,MAAM,EAAE,KAAK,CAAC,EAAC,OAAO,EAAE,mBAAmB,CAAC,EAAC,CAAE,UAAU,EAAC,MAAM,KAAI,IAAI,EAAE,SAAS,CAAC,EAAC,OAAO,KAAI,IAAI,CAAA;IAE5H;;;;OAIG;IACH,WAAW,EAAC,CAAE,SAAS,EAAC,MAAM,GAAC,MAAM,EAAE,OAAO,CAAC,EAAC,CAAE,UAAU,EAAC,MAAM,KAAI,IAAI,KAAI,IAAI,CAAA;IAEnF;;;;OAIG;IACH,WAAW,EAAC,CAAE,SAAS,EAAC,MAAM,GAAC,MAAM,EAAE,aAAa,CAAC,EAAC,MAAI,IAAI,KAAI,IAAI,CAAA;IAEtE;;;;OAIG;IACH,WAAW,EAAC,CAAE,SAAS,EAAC,MAAM,GAAC,MAAM,EAAE,KAAK,CAAC,EAAC,MAAM,KAAI,IAAI,CAAA;IAE5D;;;OAGG;IACH,MAAM,EAAC,CAAE,KAAK,EAAC,MAAM,KAAI,IAAI,CAAA;IAE7B;;OAEG;IACH,KAAK,EAAC,QAAQ,EAAE,CAAA;IAEhB;;OAEG;IACH,YAAY,EAAC,OAAO,CAAA;IAEpB;;OAEG;IACH,OAAO,EAAC,YAAY,CAAA;IAEpB;;OAEG;IACH,QAAQ,EAAC,YAAY,CAAA;CACrB,CAAA;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CAAE,YAAY,EAAC,OAAO,EAAE,KAAK,EAAC,QAAQ,EAAE,EAAE,UAAU,EAAC,IAAI,EAAE,IAAI,CAAC,EAAC,YAAY,EAAE,UAAU,CAAC,EAAC,WAAW,GAAG,sBAAsB,CA4SnK"}

package/dist/meshcap/meshcap.d.ts

@@ -0,0 +1,6 @@
+export type * from './types';
+export * from './parse-mcap-file';
+export * from './atlas-builder';
+export * from './material';
+export * from './audio';
+//# sourceMappingURL=meshcap.d.ts.map

package/dist/meshcap/meshcap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"meshcap.d.ts","sourceRoot":"","sources":["../../src/meshcap/meshcap.ts"],"names":[],"mappings":"AAAA,mBAAmB,SAAS,CAAA;AAC5B,cAAc,mBAAmB,CAAA;AACjC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,YAAY,CAAA;AAC1B,cAAc,SAAS,CAAA"}

package/dist/meshcap/parse-mcap-file.d.ts

@@ -0,0 +1,8 @@
+import { MCapFile } from './types';
+/**
+ * Loads a MeshCap (.mcap) file from a URL or File object. This is the file that contains the metadata for the clips.
+ * @param mcapFileSource URL or File object
+ * @returns
+ */
+export declare function loadMeshCapFile(mcapFileSource: string | File): Promise<MCapFile>;
+//# sourceMappingURL=parse-mcap-file.d.ts.map

package/dist/meshcap/parse-mcap-file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-mcap-file.d.ts","sourceRoot":"","sources":["../../src/meshcap/parse-mcap-file.ts"],"names":[],"mappings":"AACA,OAAO,EAAY,QAAQ,EAAyB,MAAM,SAAS,CAAC;AASpE;;;;GAIG;AACH,wBAAsB,eAAe,CAAE,cAAc,EAAC,MAAM,GAAC,IAAI,GAAK,OAAO,CAAC,QAAQ,CAAC,CAmBtF"}

package/dist/meshcap/types.d.ts

@@ -0,0 +1,88 @@
+import { Mesh, Texture, Vector3Like } from 'three';
+import { AudioSpriteAtlas } from './audio';
+import { MeshCapMaterialHandler } from './material';
+import { NodeMaterial } from 'three/webgpu';
+export interface UVCoord {
+    u: number;
+    v: number;
+    w: number;
+    h: number;
+}
+export interface Clip {
+    fps: number;
+    name: string;
+    landmarks: Vector3Like[][];
+    scale: number;
+    aspectRatio: number;
+    audioSprite?: {
+        start: number;
+    };
+    /**
+     * Duration of this clip (in seconds)
+     */
+    duration: number;
+}
+export interface RecordedClip extends Clip {
+    frames: {
+        canvas: HTMLCanvasElement;
+        cropUV: UVCoord;
+        startTime: number;
+    }[];
+    audioSprite?: {
+        domElement?: HTMLAudioElement;
+        /**
+         * Start time in the audio sprite atlas (seconds)
+         */
+        start: number;
+    };
+}
+export interface MCapClip extends Clip {
+    frames: {
+        cropUV: UVCoord;
+        frameUV: UVCoord;
+        startTime: number;
+    }[];
+}
+export interface MCapFile {
+    clips: MCapClip[];
+    version: number;
+    atlasSize: number;
+    atlasPadding: number;
+    /**
+     * Extract the clips from the atlas image using the metadata as a guide to know where the clips are.
+     * If the clips use a sound atlas, and one is provided, their audio clips will be reconstructed.
+     * @param atlas
+     */
+    unpackClips: (atlas: File | string | HTMLImageElement | HTMLCanvasElement, audioFile?: File | Blob | string | ArrayBuffer) => Promise<{
+        clips: RecordedClip[];
+        audioAtlas?: AudioSpriteAtlas;
+    }>;
+    /**
+     * Creates a material on the mesh that updates its texture to match the clip frames.
+     * If you pass `audioAtlas`, the handler will play the audio for the clip when it is played.
+     * @param mesh
+     * @param atlasTexture
+     * @returns
+     */
+    createMaterialHandlerOnMesh: (mesh: Mesh, atlasTexture: Texture, host?: NodeMaterial, audioAtlas?: AudioBuffer) => MeshCapMaterialHandler;
+}
+export interface MeshCapAtlas {
+    canvas: HTMLCanvasElement;
+    /**
+     * the index of each will correspond to the provided recorded clip's frames array at the time of creation
+     */
+    clips: MCapClip[];
+    /**
+     * padding used in the creation of the atlas
+     */
+    padding: number;
+    /**
+     * prefered max atlas dimension
+     */
+    atlasSize: number;
+    /**
+     *
+     */
+    save(downloadFile: boolean): Promise<Blob>;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/meshcap/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/meshcap/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAA;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAI3C,MAAM,WAAW,OAAO;IAAE,CAAC,EAAC,MAAM,CAAC;IAAA,CAAC,EAAC,MAAM,CAAC;IAAA,CAAC,EAAC,MAAM,CAAC;IAAA,CAAC,EAAC,MAAM,CAAA;CAAC;AAE9D,MAAM,WAAW,IAAI;IACpB,GAAG,EAAC,MAAM,CAAC;IACX,IAAI,EAAC,MAAM,CAAC;IACZ,SAAS,EAAC,WAAW,EAAE,EAAE,CAAC;IAC1B,KAAK,EAAC,MAAM,CAAA;IACZ,WAAW,EAAC,MAAM,CAAC;IACnB,WAAW,CAAC,EAAC;QACZ,KAAK,EAAC,MAAM,CAAC;KACb,CAAA;IAED;;OAEG;IACH,QAAQ,EAAC,MAAM,CAAA;CACf;AAGD,MAAM,WAAW,YAAa,SAAQ,IAAI;IACzC,MAAM,EAAC;QAAE,MAAM,EAAC,iBAAiB,CAAC;QAAC,MAAM,EAAE,OAAO,CAAC;QAAC,SAAS,EAAC,MAAM,CAAA;KAAE,EAAE,CAAC;IACzE,WAAW,CAAC,EAAC;QACZ,UAAU,CAAC,EAAC,gBAAgB,CAAC;QAE7B;;WAEG;QACH,KAAK,EAAC,MAAM,CAAC;KACb,CAAA;CACD;AAED,MAAM,WAAW,QAAS,SAAQ,IAAI;IACrC,MAAM,EAAE;QAAE,MAAM,EAAC,OAAO,CAAC;QAAC,OAAO,EAAC,OAAO,CAAC;QAAC,SAAS,EAAC,MAAM,CAAA;KAAE,EAAE,CAAA;CAC/D;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAC,MAAM,CAAC;IACjB,YAAY,EAAC,MAAM,CAAC;IAEpB;;;;OAIG;IACH,WAAW,EAAE,CAAC,KAAK,EAAC,IAAI,GAAC,MAAM,GAAC,gBAAgB,GAAC,iBAAiB,EAAE,SAAS,CAAC,EAAC,IAAI,GAAC,IAAI,GAAC,MAAM,GAAC,WAAW,KAAG,OAAO,CAAC;QAAC,KAAK,EAAC,YAAY,EAAE,CAAC;QAAC,UAAU,CAAC,EAAC,gBAAgB,CAAA;KAAC,CAAC,CAAC;IAE5K;;;;;;OAMG;IACH,2BAA2B,EAAE,CAAC,IAAI,EAAC,IAAI,EAAE,YAAY,EAAC,OAAO,EAAE,IAAI,CAAC,EAAC,YAAY,EAAE,UAAU,CAAC,EAAC,WAAW,KAAG,sBAAsB,CAAC;CACpI;AAED,MAAM,WAAW,YAAY;IACzB,MAAM,EAAE,iBAAiB,CAAC;IAE7B;;OAEG;IACA,KAAK,EAAE,QAAQ,EAAE,CAAC;IAErB;;OAEG;IACH,OAAO,EAAC,MAAM,CAAA;IAEd;;OAEG;IACH,SAAS,EAAC,MAAM,CAAA;IAEhB;;OAEG;IACH,IAAI,CAAE,YAAY,EAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC1C"}

package/dist/meshcap/write-mcap-file.d.ts

@@ -0,0 +1,3 @@
+import { MeshCapAtlas } from './types';
+export declare function atlasToMCap(atlas: MeshCapAtlas, useRelativeLandmarks?: boolean): Promise<Blob>;
+//# sourceMappingURL=write-mcap-file.d.ts.map

package/dist/meshcap/write-mcap-file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"write-mcap-file.d.ts","sourceRoot":"","sources":["../../src/meshcap/write-mcap-file.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAIvC,wBAAsB,WAAW,CAAC,KAAK,EAAE,YAAY,EAAE,oBAAoB,GAAC,OAAc,iBAwMzF"}

package/dist/meshcap.d.ts

@@ -0,0 +1,2 @@
+export * from './meshcap/meshcap'
+export {}