three-mediapipe-rig 0.0.3 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -28
- package/dist/three-mediapipe-rig.js +351 -285
- package/dist/three-mediapipe-rig.js.map +1 -1
- package/dist/tracking/FaceTracker.d.ts +20 -5
- package/dist/tracking/FaceTracker.d.ts.map +1 -1
- package/dist/tracking/HandTracker.d.ts +3 -1
- package/dist/tracking/HandTracker.d.ts.map +1 -1
- package/dist/tracking/PoseTracker.d.ts +1 -0
- package/dist/tracking/PoseTracker.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,43 +1,44 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var
|
|
4
|
-
import { PoseLandmarker as
|
|
5
|
-
import * as
|
|
6
|
-
import { Vector3 as P, Quaternion as
|
|
7
|
-
import * as
|
|
8
|
-
import { Bone as
|
|
9
|
-
import {
|
|
10
|
-
|
|
1
|
+
var pe = Object.defineProperty;
|
|
2
|
+
var me = (c, s, e) => s in c ? pe(c, s, { enumerable: !0, configurable: !0, writable: !0, value: e }) : c[s] = e;
|
|
3
|
+
var x = (c, s, e) => me(c, typeof s != "symbol" ? s + "" : s, e);
|
|
4
|
+
import { PoseLandmarker as de, HandLandmarker as z, FaceLandmarker as B, FilesetResolver as ue, DrawingUtils as fe } from "@mediapipe/tasks-vision";
|
|
5
|
+
import * as m from "three/webgpu";
|
|
6
|
+
import { Vector3 as P, Quaternion as ce, BufferAttribute as ye, VideoTexture as we, SRGBColorSpace as Le, MeshPhysicalNodeMaterial as ke } from "three/webgpu";
|
|
7
|
+
import * as O from "three";
|
|
8
|
+
import { Bone as be } from "three";
|
|
9
|
+
import { uniform as xe, attribute as Re, instancedArray as Pe, varying as ve, vec3 as Ee, texture as We, vec2 as Fe } from "three/tsl";
|
|
10
|
+
import { GLTFExporter as Ae } from "three/examples/jsm/Addons.js";
|
|
11
|
+
const _ = new P(), $ = new P(), ge = new P(1, 0, 0), Me = new P(-1, 0, 0), Se = new P(0, 1, 0), Ne = new P(0, -1, 0), H = new P(0, 0, 1);
|
|
11
12
|
new P(0, 0, -1);
|
|
12
|
-
const
|
|
13
|
-
function
|
|
14
|
-
|
|
15
|
-
const o = t == "+x" ?
|
|
16
|
-
|
|
17
|
-
const
|
|
18
|
-
|
|
13
|
+
const j = new P(), Ve = new P(), Oe = new P(), q = new ce(), D = new ce();
|
|
14
|
+
function M(c, s, e, t = "+x") {
|
|
15
|
+
c.lookAt(s);
|
|
16
|
+
const o = t == "+x" ? ge : t == "-x" ? Me : t == "+y" ? Se : Ne;
|
|
17
|
+
c.getWorldPosition($), c.getWorldQuaternion(D), _.subVectors(e, $).normalize(), j.copy(o).applyQuaternion(D);
|
|
18
|
+
const n = j, r = Ve.copy(H).applyQuaternion(D), i = _.clone().addScaledVector(r, -_.dot(r)).normalize(), a = Oe.crossVectors(n, i), l = Math.atan2(a.dot(r), n.dot(i));
|
|
19
|
+
q.setFromAxisAngle(H, l), c.quaternion.multiply(q);
|
|
19
20
|
}
|
|
20
|
-
const
|
|
21
|
-
new
|
|
22
|
-
class
|
|
21
|
+
const T = new m.Vector3(), S = new m.Vector3(), C = new m.Vector3();
|
|
22
|
+
new m.Vector3();
|
|
23
|
+
class I {
|
|
23
24
|
constructor(s, e) {
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
x(this, "objectGhost");
|
|
26
|
+
x(this, "root");
|
|
26
27
|
/**
|
|
27
28
|
* per landmark index, it points to it's object3D equivalent.
|
|
28
29
|
*/
|
|
29
|
-
|
|
30
|
-
this.points = s, this.debugConnections = e, this.root = new
|
|
30
|
+
x(this, "marks", {});
|
|
31
|
+
this.points = s, this.debugConnections = e, this.root = new m.Object3D(), this.objectGhost = /* @__PURE__ */ new Map();
|
|
31
32
|
for (let t in this.points)
|
|
32
|
-
this.marks[t] = new
|
|
33
|
+
this.marks[t] = new Te(), this.root.add(this.marks[t]);
|
|
33
34
|
}
|
|
34
35
|
updateLandmarks(s, e, t) {
|
|
35
36
|
for (let o in this.points) {
|
|
36
|
-
const
|
|
37
|
-
r && (
|
|
38
|
-
s[
|
|
39
|
-
s[
|
|
40
|
-
).divideScalar(2).add(s[
|
|
37
|
+
const n = this.points[o], r = this.marks[o];
|
|
38
|
+
r && (n instanceof Array ? (T.copy(s[n[0]]), r.position.copy(s[n[1]]).sub(T).divideScalar(2).add(s[n[0]]), n.length == 4 && (T.subVectors(
|
|
39
|
+
s[n[3]],
|
|
40
|
+
s[n[2]]
|
|
41
|
+
).divideScalar(2).add(s[n[2]]).sub(r.position).divideScalar(2), r.position.add(T))) : r.position.copy(s[n]));
|
|
41
42
|
}
|
|
42
43
|
t && e && t.drawConnectors(
|
|
43
44
|
e,
|
|
@@ -50,7 +51,7 @@ class _ {
|
|
|
50
51
|
getGhost(s) {
|
|
51
52
|
var e;
|
|
52
53
|
if (!this.objectGhost.has(s)) {
|
|
53
|
-
const t = new
|
|
54
|
+
const t = new _e();
|
|
54
55
|
t.position.copy(s.position), t.quaternion.copy(s.quaternion), (e = s.parent) == null || e.add(t), this.objectGhost.set(s, t);
|
|
55
56
|
}
|
|
56
57
|
return this.objectGhost.get(s);
|
|
@@ -65,47 +66,47 @@ class _ {
|
|
|
65
66
|
this.marks[s].position.set(1, 2, 3);
|
|
66
67
|
}
|
|
67
68
|
syncObjects(s, e, t) {
|
|
68
|
-
for (const [o,
|
|
69
|
-
this.marks[r].getWorldPosition(
|
|
70
|
-
const a =
|
|
71
|
-
o.getWorldPosition(
|
|
72
|
-
const
|
|
73
|
-
|
|
69
|
+
for (const [o, n, r, i] of s) {
|
|
70
|
+
this.marks[r].getWorldPosition(C), this.marks[n].getWorldPosition(S);
|
|
71
|
+
const a = C.sub(S);
|
|
72
|
+
o.getWorldPosition(S), S.add(a);
|
|
73
|
+
const l = S, p = o.getWorldPosition(C).sub(t), f = this.getGhost(o);
|
|
74
|
+
M(f, l, p, i), f.rotateX(Math.PI / 2), o.position.lerp(f.position, e * 4), o.quaternion.slerp(f.quaternion, e * 4);
|
|
74
75
|
}
|
|
75
76
|
}
|
|
76
77
|
getBone(s, e) {
|
|
77
78
|
return s.getObjectByName(e.replace(/[\.\:]/g, ""));
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
|
-
class
|
|
81
|
+
class Te extends m.Mesh {
|
|
81
82
|
constructor() {
|
|
82
|
-
super(new
|
|
83
|
-
|
|
84
|
-
this.add(new
|
|
83
|
+
super(new m.SphereGeometry(0.01, 3, 3), new m.MeshStandardMaterial({ color: 16711680, wireframe: !0 }));
|
|
84
|
+
x(this, "_worldPosition", new m.Vector3());
|
|
85
|
+
this.add(new m.AxesHelper(1e-3));
|
|
85
86
|
}
|
|
86
87
|
get worldPosition() {
|
|
87
88
|
return this.getWorldPosition(this._worldPosition), this._worldPosition;
|
|
88
89
|
}
|
|
89
90
|
}
|
|
90
|
-
class
|
|
91
|
+
class _e extends m.Object3D {
|
|
91
92
|
lerp(s, e, t = 8) {
|
|
92
93
|
s.position.lerp(this.position, e * t), s.quaternion.slerp(this.quaternion, e * t);
|
|
93
94
|
}
|
|
94
95
|
}
|
|
95
|
-
function
|
|
96
|
-
return e.worldToLocal(s.getWorldPosition(
|
|
96
|
+
function A(c, s, e) {
|
|
97
|
+
return e.worldToLocal(s.getWorldPosition(c)), c;
|
|
97
98
|
}
|
|
98
|
-
function
|
|
99
|
-
return
|
|
99
|
+
function De(c) {
|
|
100
|
+
return c.replace(/[\.\:]/g, "");
|
|
100
101
|
}
|
|
101
|
-
function
|
|
102
|
+
function v(c, s) {
|
|
102
103
|
let e;
|
|
103
|
-
return s =
|
|
104
|
-
t.name.indexOf(s) === 0 && t instanceof
|
|
105
|
-
}), e || console.log("Bone not found: ", s,
|
|
104
|
+
return s = De(s), c.traverse((t) => {
|
|
105
|
+
t.name.indexOf(s) === 0 && t instanceof be && (e = t);
|
|
106
|
+
}), e || console.log("Bone not found: ", s, c.name), e;
|
|
106
107
|
}
|
|
107
|
-
async function
|
|
108
|
-
const e = await
|
|
108
|
+
async function Ce(c, s) {
|
|
109
|
+
const e = await de.createFromOptions(c, {
|
|
109
110
|
baseOptions: {
|
|
110
111
|
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "pose_landmarker_lite.task",
|
|
111
112
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task`,
|
|
@@ -115,9 +116,9 @@ async function Pe(d, s) {
|
|
|
115
116
|
runningMode: "VIDEO",
|
|
116
117
|
numPoses: 1
|
|
117
118
|
});
|
|
118
|
-
return new
|
|
119
|
+
return new $e(e, s);
|
|
119
120
|
}
|
|
120
|
-
const
|
|
121
|
+
const ze = {
|
|
121
122
|
hips: [24, 23],
|
|
122
123
|
neck: [12, 11],
|
|
123
124
|
leftLeg: 23,
|
|
@@ -139,12 +140,12 @@ const We = {
|
|
|
139
140
|
torso: [24, 23, 12, 11],
|
|
140
141
|
leftEar: 7,
|
|
141
142
|
rightEar: 8
|
|
142
|
-
},
|
|
143
|
-
class
|
|
143
|
+
}, Be = new m.Vector3(), K = new m.Vector3(), Ie = new m.Vector3(), Ue = new m.Vector3(), Ge = new m.Vector3(), X = new m.Vector3(), Q = new m.Vector3();
|
|
144
|
+
class $e extends I {
|
|
144
145
|
constructor(e, t) {
|
|
145
|
-
super(
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
super(ze, de.POSE_CONNECTIONS);
|
|
147
|
+
x(this, "_leftWristNormalizedPosition");
|
|
148
|
+
x(this, "_rightWristNormalizedPosition");
|
|
148
149
|
this.poseLandmarker = e, this.config = t, this.root.scale.y *= -2, this.root.scale.z *= -2, this.root.scale.x *= 2;
|
|
149
150
|
}
|
|
150
151
|
/**
|
|
@@ -161,7 +162,8 @@ class ve extends _ {
|
|
|
161
162
|
}
|
|
162
163
|
predict(e, t) {
|
|
163
164
|
this.poseLandmarker.detectForVideo(e, performance.now(), (o) => {
|
|
164
|
-
|
|
165
|
+
var n;
|
|
166
|
+
o.landmarks.length != 0 && (this.updateLandmarks(o.worldLandmarks[0], ((n = this.config) == null ? void 0 : n.drawLandmarks) === !1 ? void 0 : o.landmarks[0], t), this._leftWristNormalizedPosition = o.landmarks[0][this.points.leftWrist], this._rightWristNormalizedPosition = o.landmarks[0][this.points.rightWrist]);
|
|
165
167
|
});
|
|
166
168
|
}
|
|
167
169
|
// override sync ( delta:number, objects: BoneBinding[] ) {
|
|
@@ -174,42 +176,42 @@ class ve extends _ {
|
|
|
174
176
|
bind(e, t) {
|
|
175
177
|
var a;
|
|
176
178
|
const o = {
|
|
177
|
-
hips:
|
|
178
|
-
neck:
|
|
179
|
-
leftArm:
|
|
180
|
-
leftElbow:
|
|
181
|
-
leftWrist:
|
|
182
|
-
rightArm:
|
|
183
|
-
rightElbow:
|
|
184
|
-
rightWrist:
|
|
185
|
-
head:
|
|
186
|
-
torso:
|
|
187
|
-
leftLeg:
|
|
188
|
-
leftKnee:
|
|
189
|
-
leftFoot:
|
|
190
|
-
rightLeg:
|
|
191
|
-
rightKnee:
|
|
192
|
-
rightFoot:
|
|
179
|
+
hips: v(e, t.hips),
|
|
180
|
+
neck: v(e, t.neck),
|
|
181
|
+
leftArm: v(e, t.armL),
|
|
182
|
+
leftElbow: v(e, t.forearmL),
|
|
183
|
+
leftWrist: v(e, t.handL),
|
|
184
|
+
rightArm: v(e, t.armR),
|
|
185
|
+
rightElbow: v(e, t.forearmR),
|
|
186
|
+
rightWrist: v(e, t.handR),
|
|
187
|
+
head: v(e, t.head),
|
|
188
|
+
torso: v(e, t.torso),
|
|
189
|
+
leftLeg: v(e, t.thighL),
|
|
190
|
+
leftKnee: v(e, t.shinL),
|
|
191
|
+
leftFoot: v(e, t.footL),
|
|
192
|
+
rightLeg: v(e, t.thighR),
|
|
193
|
+
rightKnee: v(e, t.shinR),
|
|
194
|
+
rightFoot: v(e, t.footR)
|
|
193
195
|
};
|
|
194
196
|
(a = this.config) != null && a.ignoreLegs && (delete o.leftLeg, delete o.leftKnee, delete o.leftFoot, delete o.leftToes, delete o.rightLeg, delete o.rightKnee, delete o.rightFoot, delete o.rightToes);
|
|
195
|
-
const
|
|
196
|
-
if (!
|
|
197
|
-
const w = this.marks[R].getWorldPosition(
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
|
|
197
|
+
const n = new m.Vector3(), r = new m.Vector3(), i = (l, p, f, R, d, u) => {
|
|
198
|
+
if (!p) return;
|
|
199
|
+
const w = this.marks[R].getWorldPosition(n).sub(this.marks[f].getWorldPosition(r)).normalize();
|
|
200
|
+
A(X, p, e).add(w).applyMatrix4(e.matrixWorld), A(Q, p, e).add(d).applyMatrix4(e.matrixWorld);
|
|
201
|
+
const h = this.getGhost(p);
|
|
202
|
+
M(h, X, Q, u), h.rotateX(Math.PI / 2), h.lerp(p, l);
|
|
201
203
|
};
|
|
202
204
|
return {
|
|
203
|
-
update: (
|
|
204
|
-
const
|
|
205
|
-
|
|
205
|
+
update: (l) => {
|
|
206
|
+
const p = this.marks.leftLeg.getWorldPosition(Be).sub(this.marks.rightLeg.getWorldPosition(K)).normalize(), f = this.marks.leftArm.getWorldPosition(K).sub(this.marks.rightArm.getWorldPosition(Ie)).normalize(), R = this.marks.leftEar.getWorldPosition(Ue).sub(this.marks.rightEar.getWorldPosition(Ge)).normalize();
|
|
207
|
+
i(l, o.hips, "hips", "torso", p, "+x"), i(l, o.torso, "torso", "neck", f, "+x"), i(l, o.neck, "neck", "head", R, "+x"), i(l, o.head, "neck", "head", R, "+x"), i(l, o.leftArm, "leftArm", "leftElbow", f, "-x"), i(l, o.leftElbow, "leftElbow", "leftWrist", f, "-x"), i(l, o.leftLeg, "leftLeg", "leftKnee", p, "+x"), i(l, o.leftKnee, "leftKnee", "leftFoot", p, "+x"), i(l, o.leftFoot, "leftFoot", "leftToes", p, "+x"), i(l, o.rightArm, "rightArm", "rightElbow", f, "-x"), i(l, o.rightElbow, "rightElbow", "rightWrist", f, "-x"), i(l, o.rightLeg, "rightLeg", "rightKnee", p, "+x"), i(l, o.rightKnee, "rightKnee", "rightFoot", p, "+x"), i(l, o.rightFoot, "rightFoot", "rightToes", p, "+x");
|
|
206
208
|
}
|
|
207
209
|
};
|
|
208
210
|
}
|
|
209
211
|
}
|
|
210
|
-
const
|
|
211
|
-
async function
|
|
212
|
-
const e = await
|
|
212
|
+
const Y = new m.Vector2(), Z = new m.Vector2();
|
|
213
|
+
async function He(c, s) {
|
|
214
|
+
const e = await z.createFromOptions(c, {
|
|
213
215
|
baseOptions: {
|
|
214
216
|
modelAssetPath: s.modelPath ?? "hand_landmarker.task",
|
|
215
217
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
|
|
@@ -217,13 +219,20 @@ async function ge(d, s) {
|
|
|
217
219
|
},
|
|
218
220
|
runningMode: "VIDEO",
|
|
219
221
|
numHands: 2
|
|
220
|
-
}), t = (o,
|
|
222
|
+
}), t = (o, n, r) => {
|
|
223
|
+
try {
|
|
224
|
+
Y.copy(o()), Z.copy(n());
|
|
225
|
+
} catch (i) {
|
|
226
|
+
return console.warn("No pose data... will just be optimitic and say yes to everything.", i), !0;
|
|
227
|
+
}
|
|
228
|
+
return Y.distanceTo(r) < Z.distanceTo(r);
|
|
229
|
+
};
|
|
221
230
|
return {
|
|
222
|
-
left: new
|
|
223
|
-
right: new
|
|
231
|
+
left: new re(e, "Left", t.bind(null, s.leftWrist, s.rightWrist), s.drawLandmarks),
|
|
232
|
+
right: new re(e, "Right", t.bind(null, s.rightWrist, s.leftWrist), s.drawLandmarks)
|
|
224
233
|
};
|
|
225
234
|
}
|
|
226
|
-
const
|
|
235
|
+
const je = {
|
|
227
236
|
wrist: 0,
|
|
228
237
|
palm: [9, 13],
|
|
229
238
|
thumb1: 1,
|
|
@@ -246,37 +255,38 @@ const Se = {
|
|
|
246
255
|
pinky2: 18,
|
|
247
256
|
pinky3: 19,
|
|
248
257
|
pinky4: 20
|
|
249
|
-
},
|
|
258
|
+
}, N = {
|
|
250
259
|
thumb: ["thumb1", "thumb2", "thumb3", "thumb4"],
|
|
251
260
|
index: ["index1", "index2", "index3", "index4"],
|
|
252
261
|
middle: ["middle1", "middle2", "middle3", "middle4"],
|
|
253
262
|
ring: ["ring1", "ring2", "ring3", "ring4"],
|
|
254
263
|
pinky: ["pinky1", "pinky2", "pinky3", "pinky4"]
|
|
255
|
-
},
|
|
256
|
-
new
|
|
257
|
-
new
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
+
}, qe = new m.Vector3(), J = new m.Vector3(), ee = new m.Vector3(), te = new m.Vector3(), oe = new m.Vector3(), se = new m.Vector3();
|
|
265
|
+
new m.Vector3();
|
|
266
|
+
new m.Vector3();
|
|
267
|
+
new m.Vector3();
|
|
268
|
+
const ne = new m.Vector3(), ie = Math.PI / 2, Ke = new m.Vector3(0, -1, 0);
|
|
269
|
+
class re extends I {
|
|
270
|
+
constructor(e, t, o, n = !0) {
|
|
271
|
+
super(je, z.HAND_CONNECTIONS);
|
|
272
|
+
x(this, "sign");
|
|
273
|
+
x(this, "isLeft");
|
|
264
274
|
/**
|
|
265
275
|
* the axis used to look at the pole
|
|
266
276
|
*/
|
|
267
|
-
|
|
268
|
-
this.handLandmarker = e, this.side = t, this.isMyWrist = o, this.sign = this.side == "Left" ? -1 : 1, this.isLeft = this.side == "Left", this.lookAtPoleAxis = this.sign < 0 ? "+x" : "-x", this.root.scale.setScalar(
|
|
277
|
+
x(this, "lookAtPoleAxis");
|
|
278
|
+
this.handLandmarker = e, this.side = t, this.isMyWrist = o, this.drawLandmarks = n, this.sign = this.side == "Left" ? -1 : 1, this.isLeft = this.side == "Left", this.lookAtPoleAxis = this.sign < 0 ? "+x" : "-x", this.root.scale.setScalar(7), this.root.scale.y *= -1, this.root.scale.z *= -1;
|
|
269
279
|
}
|
|
270
280
|
predict(e, t) {
|
|
271
281
|
const o = this.handLandmarker.detectForVideo(e, performance.now());
|
|
272
282
|
if (o.landmarks.length)
|
|
273
|
-
for (let
|
|
274
|
-
const r = o.landmarks[
|
|
275
|
-
if (this.isMyWrist(
|
|
276
|
-
this.updateLandmarks(o.worldLandmarks[
|
|
283
|
+
for (let n = 0; n < o.landmarks.length; n++) {
|
|
284
|
+
const r = o.landmarks[n], i = r[this.points.wrist];
|
|
285
|
+
if (this.isMyWrist(i)) {
|
|
286
|
+
this.updateLandmarks(o.worldLandmarks[n]), this.drawLandmarks && (t.drawConnectors(r, z.HAND_CONNECTIONS, {
|
|
277
287
|
color: this.side == "Left" ? "#00FF00" : "#0000FF",
|
|
278
288
|
lineWidth: 4
|
|
279
|
-
}), t.drawLandmarks(r, { color: this.side == "Left" ? "#00FF00" : "#0000FF", lineWidth: 3, radius: 1 });
|
|
289
|
+
}), t.drawLandmarks(r, { color: this.side == "Left" ? "#00FF00" : "#0000FF", lineWidth: 3, radius: 1 }));
|
|
280
290
|
break;
|
|
281
291
|
}
|
|
282
292
|
}
|
|
@@ -291,52 +301,56 @@ class ee extends _ {
|
|
|
291
301
|
* @see https://ai.google.dev/edge/mediapipe/solutions/vision/hand_landmarker
|
|
292
302
|
*/
|
|
293
303
|
syncHandBones(e, t, o) {
|
|
294
|
-
const
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
).normalize(), r =
|
|
298
|
-
if (!(r.dot(
|
|
304
|
+
const n = qe.crossVectors(
|
|
305
|
+
J.copy(this.marks.index1.worldPosition).sub(this.marks.wrist.worldPosition),
|
|
306
|
+
ee.copy(this.marks.pinky1.worldPosition).sub(this.marks.wrist.worldPosition)
|
|
307
|
+
).normalize(), r = J.copy(this.marks.palm.worldPosition).sub(this.marks.wrist.worldPosition).normalize(), i = ee.copy(this.marks.pinky1.worldPosition).sub(this.marks.index1.worldPosition).normalize();
|
|
308
|
+
if (!(r.dot(Ke) > 0.8)) {
|
|
299
309
|
if (t.wrist) {
|
|
300
|
-
const a =
|
|
301
|
-
|
|
310
|
+
const a = A(te, t.wrist, o).add(r).applyMatrix4(o.matrixWorld), l = A(oe, t.wrist, o).sub(i).applyMatrix4(o.matrixWorld), p = this.getGhost(t.wrist);
|
|
311
|
+
M(p, a, l, "-y"), p.rotateX(ie), p.lerp(t.wrist, e);
|
|
302
312
|
}
|
|
303
|
-
this.syncFinger(e, o,
|
|
313
|
+
this.syncFinger(e, o, n, r, i, t, N.index, "middle1"), this.syncFinger(e, o, n, r, i, t, N.middle, "ring1"), this.syncFinger(e, o, n, r, i, t, N.ring, "pinky1"), this.syncFinger(e, o, n, r, i, t, N.pinky, "ring1", !0), this.syncFinger(e, o, n, r, i, t, N.thumb, "index1");
|
|
304
314
|
}
|
|
305
315
|
}
|
|
306
|
-
syncFinger(e, t, o,
|
|
316
|
+
syncFinger(e, t, o, n, r, i, a, l, p = !1) {
|
|
307
317
|
for (let f = 0; f < a.length - 1; f++) {
|
|
308
|
-
const R =
|
|
318
|
+
const R = i[a[f]];
|
|
309
319
|
if (!R) continue;
|
|
310
|
-
const
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
320
|
+
const d = this.getGhost(R), u = te.copy(this.marks[a[f + 1]].worldPosition).sub(this.marks[a[f]].worldPosition).normalize(), w = A(oe, R, t);
|
|
321
|
+
if (f == 0) {
|
|
322
|
+
const h = se.copy(this.marks[l].worldPosition).sub(this.marks[a[0]].worldPosition).normalize();
|
|
323
|
+
p && h.negate(), ne.copy(h), M(
|
|
324
|
+
d,
|
|
325
|
+
u.add(w).applyMatrix4(t.matrixWorld),
|
|
326
|
+
h.add(w).applyMatrix4(t.matrixWorld),
|
|
327
|
+
this.lookAtPoleAxis
|
|
328
|
+
);
|
|
329
|
+
} else
|
|
330
|
+
M(
|
|
331
|
+
d,
|
|
332
|
+
u.add(w).applyMatrix4(t.matrixWorld),
|
|
333
|
+
se.copy(ne).add(w).applyMatrix4(t.matrixWorld),
|
|
334
|
+
this.lookAtPoleAxis
|
|
335
|
+
);
|
|
336
|
+
d.rotateX(ie), d.lerp(R, e);
|
|
324
337
|
}
|
|
325
338
|
}
|
|
326
339
|
bind(e, t) {
|
|
327
|
-
const o = {},
|
|
328
|
-
const a =
|
|
329
|
-
|
|
340
|
+
const o = {}, n = (r, i) => {
|
|
341
|
+
const a = v(e, r);
|
|
342
|
+
if (a)
|
|
343
|
+
return o[i] = a, i;
|
|
330
344
|
};
|
|
331
|
-
return
|
|
345
|
+
return n(this.isLeft ? t.handL : t.handR, "wrist"), n(this.isLeft ? t.index1L : t.index1R, "index1"), n(this.isLeft ? t.index2L : t.index2R, "index2"), n(this.isLeft ? t.index3L : t.index3R, "index3"), n(this.isLeft ? t.middle1L : t.middle1R, "middle1"), n(this.isLeft ? t.middle2L : t.middle2R, "middle2"), n(this.isLeft ? t.middle3L : t.middle3R, "middle3"), n(this.isLeft ? t.ring1L : t.ring1R, "ring1"), n(this.isLeft ? t.ring2L : t.ring2R, "ring2"), n(this.isLeft ? t.ring3L : t.ring3R, "ring3"), n(this.isLeft ? t.pinky1L : t.pinky1R, "pinky1"), n(this.isLeft ? t.pinky2L : t.pinky2R, "pinky2"), n(this.isLeft ? t.pinky3L : t.pinky3R, "pinky3"), n(this.isLeft ? t.thumb1L : t.thumb1R, "thumb1"), n(this.isLeft ? t.thumb2L : t.thumb2R, "thumb2"), n(this.isLeft ? t.thumb3L : t.thumb3R, "thumb3"), {
|
|
332
346
|
update: (r) => {
|
|
333
347
|
this.syncHandBones(r, o, e);
|
|
334
348
|
}
|
|
335
349
|
};
|
|
336
350
|
}
|
|
337
351
|
}
|
|
338
|
-
async function
|
|
339
|
-
const e = await
|
|
352
|
+
async function Xe(c, s) {
|
|
353
|
+
const e = await B.createFromOptions(c, {
|
|
340
354
|
baseOptions: {
|
|
341
355
|
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "face_landmarker.task",
|
|
342
356
|
delegate: "GPU"
|
|
@@ -345,9 +359,9 @@ async function Te(d, s) {
|
|
|
345
359
|
runningMode: "VIDEO",
|
|
346
360
|
numFaces: 1
|
|
347
361
|
});
|
|
348
|
-
return new
|
|
362
|
+
return new tt(e, { ...s });
|
|
349
363
|
}
|
|
350
|
-
const
|
|
364
|
+
const Qe = {
|
|
351
365
|
eyeL: 473,
|
|
352
366
|
eyeR: 468,
|
|
353
367
|
eyeStartL: 463,
|
|
@@ -360,71 +374,116 @@ const De = {
|
|
|
360
374
|
noseBone: 6,
|
|
361
375
|
chin: 152,
|
|
362
376
|
forehead: 10
|
|
363
|
-
},
|
|
377
|
+
}, Ye = new P(), Ze = new P(), le = new P(), Je = new P(), et = new P();
|
|
364
378
|
new P();
|
|
365
|
-
class
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
this
|
|
379
|
+
class tt extends I {
|
|
380
|
+
constructor(e, t) {
|
|
381
|
+
super(Qe, B.FACE_LANDMARKS_TESSELATION);
|
|
382
|
+
x(this, "blendshapeCategories");
|
|
383
|
+
x(this, "blendshapeMap", /* @__PURE__ */ new Map());
|
|
384
|
+
x(this, "smoothed", {});
|
|
385
|
+
x(this, "smoothing", 3e-4);
|
|
386
|
+
// lower = smoother but more lag, higher = more responsive
|
|
387
|
+
x(this, "_faceLandmarks", []);
|
|
388
|
+
this.faceLandmarker = e, this.cfg = t, this.root.scale.y *= -1, this.root.scale.z *= -1, this.root.scale.multiplyScalar(3);
|
|
374
389
|
}
|
|
375
390
|
predict(e, t) {
|
|
376
|
-
var
|
|
391
|
+
var n, r, i;
|
|
377
392
|
const o = this.faceLandmarker.detectForVideo(e, performance.now());
|
|
378
|
-
o.faceLandmarks[0] && (t.drawConnectors(o.faceLandmarks[0],
|
|
393
|
+
o.faceLandmarks[0] && (this.cfg.drawLandmarks && (t.drawConnectors(o.faceLandmarks[0], B.FACE_LANDMARKS_TESSELATION, { color: "#00fff2ff", lineWidth: 0.1 }), t.drawLandmarks(o.faceLandmarks[0], { color: "#00ff00", lineWidth: 0.1, radius: 0.4 })), this.updateLandmarks(o.faceLandmarks[0], o.faceLandmarks[0])), this._faceLandmarks = o.faceLandmarks[0], this.blendshapeCategories = (r = (n = o.faceBlendshapes) == null ? void 0 : n[0]) == null ? void 0 : r.categories, (i = this.blendshapeCategories) == null || i.forEach((a) => {
|
|
379
394
|
this.blendshapeMap.set(a.categoryName, a.score);
|
|
380
395
|
});
|
|
381
396
|
}
|
|
397
|
+
get lastKnownLandmarks() {
|
|
398
|
+
return this._faceLandmarks;
|
|
399
|
+
}
|
|
382
400
|
bindShapeKeys(e) {
|
|
383
401
|
const t = e.morphTargetDictionary;
|
|
384
402
|
return {
|
|
385
403
|
update: (o) => {
|
|
386
|
-
var
|
|
387
|
-
(
|
|
388
|
-
const { categoryName:
|
|
389
|
-
if (!(t != null && t.hasOwnProperty(
|
|
390
|
-
this.smoothed[
|
|
391
|
-
const
|
|
392
|
-
this.smoothed[
|
|
404
|
+
var n;
|
|
405
|
+
(n = this.blendshapeCategories) == null || n.forEach((r) => {
|
|
406
|
+
const { categoryName: i, score: a } = r;
|
|
407
|
+
if (!(t != null && t.hasOwnProperty(i))) return;
|
|
408
|
+
this.smoothed[i] === void 0 && (this.smoothed[i] = a);
|
|
409
|
+
const l = 1 - Math.pow(this.smoothing, o);
|
|
410
|
+
this.smoothed[i] += (a - this.smoothed[i]) * l, e.morphTargetInfluences[t[i]] = this.smoothed[i];
|
|
393
411
|
});
|
|
394
412
|
}
|
|
395
413
|
};
|
|
396
414
|
}
|
|
397
415
|
bind(e) {
|
|
398
|
-
const t = new
|
|
416
|
+
const t = new ae(e, "L"), o = new ae(e, "R"), n = v(e, "head");
|
|
399
417
|
return {
|
|
400
418
|
update: (r) => {
|
|
401
|
-
if (t.update(r, this.blendshapeMap), o.update(r, this.blendshapeMap), !
|
|
402
|
-
const
|
|
403
|
-
|
|
419
|
+
if (t.update(r, this.blendshapeMap), o.update(r, this.blendshapeMap), !n) return;
|
|
420
|
+
const i = Ye.copy(this.marks.earL.worldPosition), a = Ze.copy(this.marks.earR.worldPosition), l = le.subVectors(i, a).multiplyScalar(0.5).add(a), p = Je.subVectors(this.marks.noseTip.worldPosition, l), f = i.sub(a), R = A(et, n, e), d = f.add(R).applyMatrix4(e.matrixWorld), u = p.add(R).applyMatrix4(e.matrixWorld);
|
|
421
|
+
M(n, u, d, "+x");
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* The mesh is assumed to be a canonical_face_model because we will be manipulating it's vertices.
|
|
427
|
+
*
|
|
428
|
+
* @see https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/modules/face_geometry
|
|
429
|
+
* @param mesh basically either the original or a clone of the canonical_face_model
|
|
430
|
+
*/
|
|
431
|
+
bindGeometry(e, t) {
|
|
432
|
+
const o = new P(), n = new P(), r = e.geometry, i = r.attributes.position, a = [], l = /* @__PURE__ */ new Map(), p = [];
|
|
433
|
+
for (let h = 0; h < i.count; h++) {
|
|
434
|
+
const y = `${i.getX(h).toFixed(6)},${i.getY(h).toFixed(6)},${i.getZ(h).toFixed(6)}`;
|
|
435
|
+
l.has(y) || (l.set(y, a.length), a.push(h)), p.push(l.get(y));
|
|
436
|
+
}
|
|
437
|
+
const f = xe(new P(0.5, 0.5, 0.5));
|
|
438
|
+
r.setAttribute("landmarkIndex", new ye(new Uint16Array(p), 1));
|
|
439
|
+
const R = Re("landmarkIndex", "uint"), d = Pe(468, "vec3"), u = ve(d.element(R)).xy;
|
|
440
|
+
let w;
|
|
441
|
+
return {
|
|
442
|
+
update: (h) => {
|
|
443
|
+
var g, V;
|
|
444
|
+
if (!w) {
|
|
445
|
+
if (w = (V = (g = this.cfg) == null ? void 0 : g.videoElementRef) == null ? void 0 : V.call(g), !w) return;
|
|
446
|
+
const W = new we(w);
|
|
447
|
+
W.colorSpace = Le;
|
|
448
|
+
const U = d.element(R).sub(f).xzy.mul(Ee(1.3, -1.1, 1)), G = We(W, Fe(u.x, u.y.oneMinus()));
|
|
449
|
+
t ? t(U, G) : e.material = new ke({
|
|
450
|
+
positionNode: U,
|
|
451
|
+
colorNode: G,
|
|
452
|
+
roughness: 0.93
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
const y = this.lastKnownLandmarks;
|
|
456
|
+
if (!y) return;
|
|
457
|
+
const b = d.value.array;
|
|
458
|
+
for (let W = 0; W < y.length; W++)
|
|
459
|
+
b[W * 3] = y[W].x, b[W * 3 + 1] = y[W].y, b[W * 3 + 2] = y[W].z;
|
|
460
|
+
d.value.needsUpdate = !0;
|
|
461
|
+
const E = y[234], k = y[93], L = y[454], F = y[323];
|
|
462
|
+
o.copy(E).sub(k).divideScalar(2).add(k), n.copy(L).sub(F).divideScalar(2).add(F), f.value.subVectors(n, o).divideScalar(2).add(o), f.needsUpdate = !0;
|
|
404
463
|
}
|
|
405
464
|
};
|
|
406
465
|
}
|
|
407
466
|
}
|
|
408
|
-
class
|
|
467
|
+
class ae {
|
|
409
468
|
constructor(s, e) {
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
469
|
+
x(this, "eyeBone");
|
|
470
|
+
x(this, "eyeLookOut");
|
|
471
|
+
x(this, "eyeLookIn");
|
|
472
|
+
x(this, "eyeLookUp");
|
|
473
|
+
x(this, "eyeLookDown");
|
|
474
|
+
x(this, "sign", 1);
|
|
416
475
|
this.rig = s, this.side = e, this.eyeBone = s.getObjectByName(`eye${e}`);
|
|
417
476
|
const t = e == "L" ? "Left" : "Right";
|
|
418
477
|
this.eyeLookOut = `eyeLookOut${t}`, this.eyeLookIn = `eyeLookIn${t}`, this.eyeLookUp = `eyeLookUp${t}`, this.eyeLookDown = `eyeLookDown${t}`, this.sign = e == "L" ? -1 : 1;
|
|
419
478
|
}
|
|
420
479
|
update(s, e) {
|
|
421
480
|
if (!this.eyeBone) return;
|
|
422
|
-
|
|
423
|
-
const t = e.get(this.eyeLookOut) ?? 0, o = e.get(this.eyeLookIn) ?? 0,
|
|
424
|
-
this.eyeBone.rotation.y =
|
|
481
|
+
A(le, this.eyeBone, this.rig);
|
|
482
|
+
const t = e.get(this.eyeLookOut) ?? 0, o = e.get(this.eyeLookIn) ?? 0, n = e.get(this.eyeLookUp) ?? 0, r = e.get(this.eyeLookDown) ?? 0, i = o - t, a = r - n;
|
|
483
|
+
this.eyeBone.rotation.y = i * this.sign / 2, this.eyeBone.rotation.x = a / 2;
|
|
425
484
|
}
|
|
426
485
|
}
|
|
427
|
-
const
|
|
486
|
+
const he = {
|
|
428
487
|
faceMesh: "face",
|
|
429
488
|
head: "head",
|
|
430
489
|
hips: "hips",
|
|
@@ -472,7 +531,7 @@ const ne = {
|
|
|
472
531
|
thumb1R: "thumb1R",
|
|
473
532
|
thumb2R: "thumb2R",
|
|
474
533
|
thumb3R: "thumb3R"
|
|
475
|
-
},
|
|
534
|
+
}, ot = [
|
|
476
535
|
"eyeBlinkLeft",
|
|
477
536
|
"eyeBlinkRight",
|
|
478
537
|
"eyeLookDownLeft",
|
|
@@ -526,206 +585,211 @@ const ne = {
|
|
|
526
585
|
"mouthUpperUpRight",
|
|
527
586
|
"tongueOut"
|
|
528
587
|
];
|
|
529
|
-
function
|
|
530
|
-
const t = [], o =
|
|
588
|
+
function st(c, s, e = 30) {
|
|
589
|
+
const t = [], o = c.getObjectByName(
|
|
531
590
|
s.faceMesh
|
|
532
|
-
),
|
|
591
|
+
), n = /* @__PURE__ */ new Set();
|
|
533
592
|
if (o != null && o.morphTargetDictionary)
|
|
534
593
|
for (const w in o.morphTargetDictionary)
|
|
535
|
-
|
|
594
|
+
ot.includes(w) && n.add(w);
|
|
536
595
|
const r = Object.keys(s);
|
|
537
|
-
|
|
538
|
-
if (w instanceof
|
|
539
|
-
const
|
|
540
|
-
|
|
596
|
+
c.traverse((w) => {
|
|
597
|
+
if (w instanceof O.Bone) {
|
|
598
|
+
const h = r.find((y) => w.name.indexOf(s[y]) === 0);
|
|
599
|
+
h && t.push({
|
|
541
600
|
ref: w,
|
|
542
601
|
name: w.name,
|
|
543
|
-
normalizedName:
|
|
602
|
+
normalizedName: h
|
|
544
603
|
});
|
|
545
604
|
}
|
|
546
605
|
});
|
|
547
|
-
const
|
|
548
|
-
let
|
|
606
|
+
const i = [], a = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map();
|
|
607
|
+
let p = !1, f = 0;
|
|
549
608
|
function R() {
|
|
550
|
-
a.clear(),
|
|
609
|
+
a.clear(), l.clear(), i.length = 0, f = performance.now() / 1e3, p = !0;
|
|
551
610
|
}
|
|
552
|
-
function
|
|
553
|
-
if (!
|
|
611
|
+
function d() {
|
|
612
|
+
if (!p) return;
|
|
554
613
|
const w = performance.now() / 1e3 - f;
|
|
555
|
-
|
|
556
|
-
for (const
|
|
557
|
-
a.has(
|
|
558
|
-
const
|
|
559
|
-
|
|
614
|
+
i.push(w);
|
|
615
|
+
for (const h of t) {
|
|
616
|
+
a.has(h.name) || a.set(h.name, []);
|
|
617
|
+
const y = a.get(h.name), b = h.ref.quaternion;
|
|
618
|
+
y.push(b.x, b.y, b.z, b.w);
|
|
560
619
|
}
|
|
561
|
-
for (const
|
|
562
|
-
|
|
563
|
-
const
|
|
564
|
-
|
|
620
|
+
for (const h of n) {
|
|
621
|
+
l.has(h) || l.set(h, []);
|
|
622
|
+
const y = l.get(h), b = o.morphTargetDictionary[h], E = o.morphTargetInfluences[b];
|
|
623
|
+
y.push(E);
|
|
565
624
|
}
|
|
566
625
|
}
|
|
567
|
-
function
|
|
568
|
-
|
|
569
|
-
const
|
|
626
|
+
function u(w = "RecordedClip") {
|
|
627
|
+
p = !1;
|
|
628
|
+
const h = [];
|
|
570
629
|
for (const [b, E] of a)
|
|
571
|
-
|
|
572
|
-
new
|
|
630
|
+
h.push(
|
|
631
|
+
new O.QuaternionKeyframeTrack(
|
|
573
632
|
`${b}.quaternion`,
|
|
574
|
-
|
|
633
|
+
i,
|
|
575
634
|
E
|
|
576
635
|
)
|
|
577
636
|
);
|
|
578
|
-
for (const [b, E] of
|
|
579
|
-
|
|
580
|
-
new
|
|
581
|
-
`${
|
|
582
|
-
|
|
637
|
+
for (const [b, E] of l)
|
|
638
|
+
h.push(
|
|
639
|
+
new O.NumberKeyframeTrack(
|
|
640
|
+
`${he.faceMesh}.morphTargetInfluences[${b}]`,
|
|
641
|
+
i,
|
|
583
642
|
E
|
|
584
643
|
)
|
|
585
644
|
);
|
|
586
|
-
const
|
|
645
|
+
const y = new O.AnimationClip(w, -1, h);
|
|
587
646
|
return {
|
|
588
|
-
clip:
|
|
647
|
+
clip: y,
|
|
589
648
|
saveToFile: () => {
|
|
590
|
-
new
|
|
591
|
-
|
|
649
|
+
new Ae().parse(
|
|
650
|
+
c,
|
|
592
651
|
(E) => {
|
|
593
|
-
const
|
|
652
|
+
const k = new Blob([E], {
|
|
594
653
|
type: "model/gltf-binary"
|
|
595
|
-
}),
|
|
596
|
-
|
|
654
|
+
}), L = URL.createObjectURL(k), F = document.createElement("a");
|
|
655
|
+
F.href = L, F.download = w + ".glb", F.click();
|
|
597
656
|
},
|
|
598
657
|
(E) => {
|
|
599
658
|
console.error(E);
|
|
600
659
|
},
|
|
601
660
|
{
|
|
602
661
|
binary: !0,
|
|
603
|
-
animations: [
|
|
662
|
+
animations: [y]
|
|
604
663
|
}
|
|
605
664
|
);
|
|
606
665
|
}
|
|
607
666
|
};
|
|
608
667
|
}
|
|
609
|
-
return { start: R, captureFrame:
|
|
668
|
+
return { start: R, captureFrame: d, stop: u, isRecording: () => p };
|
|
610
669
|
}
|
|
611
|
-
const
|
|
612
|
-
var
|
|
613
|
-
return !!((
|
|
670
|
+
const nt = () => {
|
|
671
|
+
var c;
|
|
672
|
+
return !!((c = navigator.mediaDevices) != null && c.getUserMedia);
|
|
614
673
|
};
|
|
615
|
-
async function
|
|
674
|
+
async function ht(c) {
|
|
616
675
|
const s = {
|
|
617
676
|
debugFrame: void 0,
|
|
618
677
|
displayScale: 1,
|
|
619
678
|
ignoreLegs: !1,
|
|
620
679
|
debugVideo: void 0,
|
|
621
680
|
ignoreFace: !1,
|
|
681
|
+
onlyFace: !1,
|
|
682
|
+
drawLandmarksOverlay: !0,
|
|
622
683
|
modelPaths: {
|
|
623
684
|
vision: "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm",
|
|
624
685
|
pose: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task",
|
|
625
686
|
hand: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
|
|
626
687
|
face: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task"
|
|
627
688
|
},
|
|
628
|
-
...
|
|
689
|
+
...c
|
|
629
690
|
};
|
|
630
691
|
let e;
|
|
631
|
-
const t = await
|
|
692
|
+
const t = await ue.forVisionTasks(s.modelPaths.vision ?? "/wasm"), o = s.onlyFace ? void 0 : await Ce(t, {
|
|
632
693
|
ignoreLegs: s.ignoreLegs,
|
|
633
|
-
modelPath: s.modelPaths.pose
|
|
634
|
-
|
|
694
|
+
modelPath: s.modelPaths.pose,
|
|
695
|
+
drawLandmarks: s.drawLandmarksOverlay
|
|
696
|
+
}), n = s.onlyFace ? void 0 : await He(t, {
|
|
635
697
|
leftWrist: () => o.leftWristNormalizedPosition,
|
|
636
698
|
rightWrist: () => o.rightWristNormalizedPosition,
|
|
637
699
|
modelPath: s.modelPaths.hand,
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
a
|
|
643
|
-
|
|
644
|
-
|
|
700
|
+
drawLandmarks: s.drawLandmarksOverlay,
|
|
701
|
+
...c == null ? void 0 : c.handsTrackerOptions
|
|
702
|
+
}), r = s.ignoreFace ? void 0 : await Xe(t, { modelPath: s.modelPaths.face, videoElementRef: () => e, drawLandmarks: s.drawLandmarksOverlay }), i = document.createElement("div");
|
|
703
|
+
i.style.position = "absolute", i.style.top = "0px", i.style.left = "0px", i.style.zIndex = "21", i.classList.add("three-mediapipe-rig"), document.body.appendChild(i);
|
|
704
|
+
const a = document.createElement("canvas"), l = a.getContext("2d"), p = new fe(l);
|
|
705
|
+
a.style.zIndex = "22", a.style.position = "absolute", a.style.top = "0px", a.style.left = "0px", a.style.pointerEvents = "none", i.appendChild(a);
|
|
706
|
+
function f(d) {
|
|
707
|
+
l.save(), l.clearRect(0, 0, a.width, a.height), o == null || o.predict(d, p), n == null || n.left.predict(d, p), n == null || n.right.predict(d, p), r == null || r.predict(d, p), l.restore();
|
|
645
708
|
}
|
|
646
709
|
function R() {
|
|
647
|
-
e = document.createElement("video"),
|
|
648
|
-
let
|
|
710
|
+
e = document.createElement("video"), i.appendChild(e);
|
|
711
|
+
let d = -1;
|
|
649
712
|
e.style.zIndex = "21", e.style.position = "absolute", e.style.top = "0px", e.style.left = "0px", s.debugVideo && (e.src = s.debugVideo, e.controls = !0, e.loop = !0, e.muted = !0, e.controls = !0, e.play());
|
|
650
|
-
function
|
|
651
|
-
|
|
713
|
+
function u() {
|
|
714
|
+
d !== e.currentTime && (f(e), d = e.currentTime), window.requestAnimationFrame(u);
|
|
652
715
|
}
|
|
653
716
|
e.addEventListener("loadeddata", () => {
|
|
654
|
-
e.width = e.videoWidth * s.displayScale, e.height = e.videoHeight * s.displayScale, a.width = e.videoWidth, a.height = e.videoHeight, a.style.height = e.height + "px", a.style.width = e.width + "px", window.requestAnimationFrame(
|
|
717
|
+
e.width = e.videoWidth * s.displayScale, e.height = e.videoHeight * s.displayScale, a.width = e.videoWidth, a.height = e.videoHeight, a.style.height = e.height + "px", a.style.width = e.width + "px", window.requestAnimationFrame(u);
|
|
655
718
|
});
|
|
656
719
|
}
|
|
657
720
|
if (s.debugFrame) {
|
|
658
|
-
const
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
function
|
|
662
|
-
f(
|
|
721
|
+
const d = document.createElement("img");
|
|
722
|
+
d.src = s.debugFrame, d.style.zIndex = "21", d.style.position = "absolute", d.style.top = "0px", d.style.left = "0px", i.appendChild(d), d.addEventListener("load", () => {
|
|
723
|
+
d.width = d.naturalWidth * s.displayScale, d.height = d.naturalHeight * s.displayScale, a.width = d.naturalWidth, a.height = d.naturalWidth, a.style.width = d.width + "px", a.style.height = d.height + "px";
|
|
724
|
+
function u() {
|
|
725
|
+
f(d);
|
|
663
726
|
}
|
|
664
|
-
window.requestAnimationFrame(
|
|
727
|
+
window.requestAnimationFrame(u);
|
|
665
728
|
});
|
|
666
729
|
} else s.debugVideo && R();
|
|
667
730
|
return {
|
|
668
731
|
poseTracker: o,
|
|
669
|
-
handsTracker:
|
|
732
|
+
handsTracker: n,
|
|
670
733
|
faceTracker: r,
|
|
671
734
|
video: e,
|
|
735
|
+
canvas: a,
|
|
672
736
|
/**
|
|
673
737
|
* A div that contains the video and canvas used to display the landmarks stacked on top of each other.
|
|
674
738
|
*/
|
|
675
|
-
domElement:
|
|
739
|
+
domElement: i,
|
|
676
740
|
/**
|
|
677
741
|
* Start the webcam feed. This must be initiated by a user triggered event ( a click on a button ) due to security reasons.
|
|
678
742
|
*/
|
|
679
743
|
start: async () => {
|
|
680
|
-
let
|
|
681
|
-
if (!
|
|
744
|
+
let d = !1;
|
|
745
|
+
if (!nt())
|
|
682
746
|
throw new Error("Webcam not supported");
|
|
683
747
|
e || R();
|
|
684
|
-
let
|
|
685
|
-
function w(
|
|
686
|
-
console.warn("Camera track ended, attempting recovery..."),
|
|
748
|
+
let u;
|
|
749
|
+
function w(k) {
|
|
750
|
+
console.warn("Camera track ended, attempting recovery..."), h(k), y(k);
|
|
687
751
|
}
|
|
688
|
-
function
|
|
689
|
-
|
|
752
|
+
function h(k) {
|
|
753
|
+
u == null || u.getTracks().forEach((L) => L.stop()), u = void 0, k.srcObject = null;
|
|
690
754
|
}
|
|
691
|
-
async function
|
|
692
|
-
const
|
|
693
|
-
if (
|
|
755
|
+
async function y(k, L = 0) {
|
|
756
|
+
const g = Math.min(1e3 * 2 ** L, 16e3);
|
|
757
|
+
if (L >= 5)
|
|
694
758
|
throw new Error("Camera recovery failed after max attempts");
|
|
695
|
-
if (await new Promise((
|
|
759
|
+
if (await new Promise((V) => setTimeout(V, g)), !d)
|
|
696
760
|
try {
|
|
697
|
-
await b(
|
|
761
|
+
await b(k), console.log("Camera recovered successfully");
|
|
698
762
|
} catch {
|
|
699
|
-
|
|
763
|
+
y(k, L + 1);
|
|
700
764
|
}
|
|
701
765
|
}
|
|
702
|
-
async function b(
|
|
766
|
+
async function b(k) {
|
|
703
767
|
try {
|
|
704
|
-
|
|
705
|
-
|
|
768
|
+
u = await navigator.mediaDevices.getUserMedia({ video: !0 }), k.srcObject = u, await k.play(), u.getVideoTracks().forEach((L) => {
|
|
769
|
+
L.addEventListener("ended", () => w(k));
|
|
706
770
|
});
|
|
707
|
-
} catch (
|
|
708
|
-
E(
|
|
771
|
+
} catch (L) {
|
|
772
|
+
E(L, k);
|
|
709
773
|
}
|
|
710
774
|
}
|
|
711
|
-
function E(
|
|
712
|
-
if (
|
|
713
|
-
switch (
|
|
775
|
+
function E(k, L) {
|
|
776
|
+
if (k instanceof DOMException)
|
|
777
|
+
switch (k.name) {
|
|
714
778
|
case "NotAllowedError":
|
|
715
779
|
throw new Error("Permission denied — prompt user to allow camera");
|
|
716
780
|
case "NotFoundError":
|
|
717
|
-
console.error("No camera found — retry when device is connected"),
|
|
781
|
+
console.error("No camera found — retry when device is connected"), y(L);
|
|
718
782
|
break;
|
|
719
783
|
case "NotReadableError":
|
|
720
|
-
console.error("Camera in use by another app"),
|
|
784
|
+
console.error("Camera in use by another app"), y(L);
|
|
721
785
|
break;
|
|
722
786
|
default:
|
|
723
|
-
console.error("Camera error:",
|
|
787
|
+
console.error("Camera error:", k.message), y(L);
|
|
724
788
|
}
|
|
725
789
|
}
|
|
726
790
|
return await b(e), {
|
|
727
791
|
stop: () => {
|
|
728
|
-
|
|
792
|
+
d = !0, h(e);
|
|
729
793
|
}
|
|
730
794
|
};
|
|
731
795
|
},
|
|
@@ -734,31 +798,33 @@ async function Qe(d) {
|
|
|
734
798
|
* @param rig The rig that contains all the bones and skinned meshes of your character.
|
|
735
799
|
* @param magging The bone mapping to use for the rig.
|
|
736
800
|
*/
|
|
737
|
-
bind: (
|
|
738
|
-
|
|
739
|
-
|
|
801
|
+
bind: (d, u) => {
|
|
802
|
+
if (u = u || he, !o) throw new Error("Pose tracker not initialized");
|
|
803
|
+
if (!n) throw new Error("Hands tracker not initialized");
|
|
804
|
+
if (!r) throw new Error("Face tracker not initialized");
|
|
805
|
+
const w = o.bind(d, u), h = n.left.bind(d, u), y = n.right.bind(d, u);
|
|
740
806
|
let b;
|
|
741
|
-
const E = r == null ? void 0 : r.bind(
|
|
742
|
-
|
|
743
|
-
|
|
807
|
+
const E = r == null ? void 0 : r.bind(d);
|
|
808
|
+
d.traverse((L) => {
|
|
809
|
+
L instanceof m.Mesh && L.name.indexOf(u.faceMesh) === 0 && (L.frustumCulled = !1, b = r == null ? void 0 : r.bindShapeKeys(L));
|
|
744
810
|
});
|
|
745
|
-
const
|
|
811
|
+
const k = st(d, u);
|
|
746
812
|
return {
|
|
747
813
|
/**
|
|
748
814
|
* Will save the tracked movement of the rig to an animation clip.
|
|
749
815
|
* Only the bones moved by the bone mapping will be recorded.
|
|
750
816
|
*/
|
|
751
|
-
startRecording:
|
|
752
|
-
stopRecording:
|
|
753
|
-
isRecording: () =>
|
|
754
|
-
update: (
|
|
755
|
-
w.update(
|
|
817
|
+
startRecording: k.start,
|
|
818
|
+
stopRecording: k.stop,
|
|
819
|
+
isRecording: () => k.isRecording(),
|
|
820
|
+
update: (L) => {
|
|
821
|
+
w.update(L), h.update(L), y.update(L), b == null || b.update(L), E == null || E.update(L), k.isRecording() && k.captureFrame();
|
|
756
822
|
}
|
|
757
823
|
};
|
|
758
824
|
}
|
|
759
825
|
};
|
|
760
826
|
}
|
|
761
827
|
export {
|
|
762
|
-
|
|
828
|
+
ht as setupTracker
|
|
763
829
|
};
|
|
764
830
|
//# sourceMappingURL=three-mediapipe-rig.js.map
|