three-mediapipe-rig 0.0.4 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -5
- package/dist/three-mediapipe-rig.js +331 -277
- package/dist/three-mediapipe-rig.js.map +1 -1
- package/dist/tracking/FaceTracker.d.ts +20 -5
- package/dist/tracking/FaceTracker.d.ts.map +1 -1
- package/dist/tracking/HandTracker.d.ts +3 -1
- package/dist/tracking/HandTracker.d.ts.map +1 -1
- package/dist/tracking/PoseTracker.d.ts +1 -0
- package/dist/tracking/PoseTracker.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,43 +1,44 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var
|
|
4
|
-
import { PoseLandmarker as
|
|
5
|
-
import * as
|
|
6
|
-
import { Vector3 as P, Quaternion as
|
|
7
|
-
import * as
|
|
8
|
-
import { Bone as
|
|
9
|
-
import {
|
|
10
|
-
|
|
1
|
+
var pe = Object.defineProperty;
|
|
2
|
+
var me = (c, s, e) => s in c ? pe(c, s, { enumerable: !0, configurable: !0, writable: !0, value: e }) : c[s] = e;
|
|
3
|
+
var x = (c, s, e) => me(c, typeof s != "symbol" ? s + "" : s, e);
|
|
4
|
+
import { PoseLandmarker as de, HandLandmarker as z, FaceLandmarker as B, FilesetResolver as ue, DrawingUtils as fe } from "@mediapipe/tasks-vision";
|
|
5
|
+
import * as m from "three/webgpu";
|
|
6
|
+
import { Vector3 as P, Quaternion as ce, BufferAttribute as ye, VideoTexture as we, SRGBColorSpace as Le, MeshPhysicalNodeMaterial as ke } from "three/webgpu";
|
|
7
|
+
import * as O from "three";
|
|
8
|
+
import { Bone as be } from "three";
|
|
9
|
+
import { uniform as xe, attribute as Re, instancedArray as Pe, varying as ve, vec3 as Ee, texture as We, vec2 as Fe } from "three/tsl";
|
|
10
|
+
import { GLTFExporter as Ae } from "three/examples/jsm/Addons.js";
|
|
11
|
+
const _ = new P(), $ = new P(), ge = new P(1, 0, 0), Me = new P(-1, 0, 0), Se = new P(0, 1, 0), Ne = new P(0, -1, 0), H = new P(0, 0, 1);
|
|
11
12
|
new P(0, 0, -1);
|
|
12
|
-
const
|
|
13
|
-
function
|
|
14
|
-
|
|
15
|
-
const o = t == "+x" ?
|
|
16
|
-
|
|
17
|
-
const
|
|
18
|
-
|
|
13
|
+
const j = new P(), Ve = new P(), Oe = new P(), q = new ce(), D = new ce();
|
|
14
|
+
function M(c, s, e, t = "+x") {
|
|
15
|
+
c.lookAt(s);
|
|
16
|
+
const o = t == "+x" ? ge : t == "-x" ? Me : t == "+y" ? Se : Ne;
|
|
17
|
+
c.getWorldPosition($), c.getWorldQuaternion(D), _.subVectors(e, $).normalize(), j.copy(o).applyQuaternion(D);
|
|
18
|
+
const n = j, r = Ve.copy(H).applyQuaternion(D), i = _.clone().addScaledVector(r, -_.dot(r)).normalize(), a = Oe.crossVectors(n, i), l = Math.atan2(a.dot(r), n.dot(i));
|
|
19
|
+
q.setFromAxisAngle(H, l), c.quaternion.multiply(q);
|
|
19
20
|
}
|
|
20
|
-
const
|
|
21
|
-
new
|
|
22
|
-
class
|
|
21
|
+
const T = new m.Vector3(), S = new m.Vector3(), C = new m.Vector3();
|
|
22
|
+
new m.Vector3();
|
|
23
|
+
class I {
|
|
23
24
|
constructor(s, e) {
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
x(this, "objectGhost");
|
|
26
|
+
x(this, "root");
|
|
26
27
|
/**
|
|
27
28
|
* per landmark index, it points to it's object3D equivalent.
|
|
28
29
|
*/
|
|
29
|
-
|
|
30
|
-
this.points = s, this.debugConnections = e, this.root = new
|
|
30
|
+
x(this, "marks", {});
|
|
31
|
+
this.points = s, this.debugConnections = e, this.root = new m.Object3D(), this.objectGhost = /* @__PURE__ */ new Map();
|
|
31
32
|
for (let t in this.points)
|
|
32
|
-
this.marks[t] = new
|
|
33
|
+
this.marks[t] = new Te(), this.root.add(this.marks[t]);
|
|
33
34
|
}
|
|
34
35
|
updateLandmarks(s, e, t) {
|
|
35
36
|
for (let o in this.points) {
|
|
36
|
-
const
|
|
37
|
-
r && (
|
|
38
|
-
s[
|
|
39
|
-
s[
|
|
40
|
-
).divideScalar(2).add(s[
|
|
37
|
+
const n = this.points[o], r = this.marks[o];
|
|
38
|
+
r && (n instanceof Array ? (T.copy(s[n[0]]), r.position.copy(s[n[1]]).sub(T).divideScalar(2).add(s[n[0]]), n.length == 4 && (T.subVectors(
|
|
39
|
+
s[n[3]],
|
|
40
|
+
s[n[2]]
|
|
41
|
+
).divideScalar(2).add(s[n[2]]).sub(r.position).divideScalar(2), r.position.add(T))) : r.position.copy(s[n]));
|
|
41
42
|
}
|
|
42
43
|
t && e && t.drawConnectors(
|
|
43
44
|
e,
|
|
@@ -50,7 +51,7 @@ class _ {
|
|
|
50
51
|
getGhost(s) {
|
|
51
52
|
var e;
|
|
52
53
|
if (!this.objectGhost.has(s)) {
|
|
53
|
-
const t = new
|
|
54
|
+
const t = new _e();
|
|
54
55
|
t.position.copy(s.position), t.quaternion.copy(s.quaternion), (e = s.parent) == null || e.add(t), this.objectGhost.set(s, t);
|
|
55
56
|
}
|
|
56
57
|
return this.objectGhost.get(s);
|
|
@@ -65,47 +66,47 @@ class _ {
|
|
|
65
66
|
this.marks[s].position.set(1, 2, 3);
|
|
66
67
|
}
|
|
67
68
|
syncObjects(s, e, t) {
|
|
68
|
-
for (const [o,
|
|
69
|
-
this.marks[r].getWorldPosition(
|
|
70
|
-
const a =
|
|
71
|
-
o.getWorldPosition(
|
|
72
|
-
const
|
|
73
|
-
|
|
69
|
+
for (const [o, n, r, i] of s) {
|
|
70
|
+
this.marks[r].getWorldPosition(C), this.marks[n].getWorldPosition(S);
|
|
71
|
+
const a = C.sub(S);
|
|
72
|
+
o.getWorldPosition(S), S.add(a);
|
|
73
|
+
const l = S, p = o.getWorldPosition(C).sub(t), f = this.getGhost(o);
|
|
74
|
+
M(f, l, p, i), f.rotateX(Math.PI / 2), o.position.lerp(f.position, e * 4), o.quaternion.slerp(f.quaternion, e * 4);
|
|
74
75
|
}
|
|
75
76
|
}
|
|
76
77
|
getBone(s, e) {
|
|
77
78
|
return s.getObjectByName(e.replace(/[\.\:]/g, ""));
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
|
-
class
|
|
81
|
+
class Te extends m.Mesh {
|
|
81
82
|
constructor() {
|
|
82
|
-
super(new
|
|
83
|
-
|
|
84
|
-
this.add(new
|
|
83
|
+
super(new m.SphereGeometry(0.01, 3, 3), new m.MeshStandardMaterial({ color: 16711680, wireframe: !0 }));
|
|
84
|
+
x(this, "_worldPosition", new m.Vector3());
|
|
85
|
+
this.add(new m.AxesHelper(1e-3));
|
|
85
86
|
}
|
|
86
87
|
get worldPosition() {
|
|
87
88
|
return this.getWorldPosition(this._worldPosition), this._worldPosition;
|
|
88
89
|
}
|
|
89
90
|
}
|
|
90
|
-
class
|
|
91
|
+
class _e extends m.Object3D {
|
|
91
92
|
lerp(s, e, t = 8) {
|
|
92
93
|
s.position.lerp(this.position, e * t), s.quaternion.slerp(this.quaternion, e * t);
|
|
93
94
|
}
|
|
94
95
|
}
|
|
95
|
-
function
|
|
96
|
-
return e.worldToLocal(s.getWorldPosition(
|
|
96
|
+
function A(c, s, e) {
|
|
97
|
+
return e.worldToLocal(s.getWorldPosition(c)), c;
|
|
97
98
|
}
|
|
98
|
-
function
|
|
99
|
-
return
|
|
99
|
+
function De(c) {
|
|
100
|
+
return c.replace(/[\.\:]/g, "");
|
|
100
101
|
}
|
|
101
|
-
function
|
|
102
|
+
function v(c, s) {
|
|
102
103
|
let e;
|
|
103
|
-
return s =
|
|
104
|
-
t.name.indexOf(s) === 0 && t instanceof
|
|
105
|
-
}), e || console.log("Bone not found: ", s,
|
|
104
|
+
return s = De(s), c.traverse((t) => {
|
|
105
|
+
t.name.indexOf(s) === 0 && t instanceof be && (e = t);
|
|
106
|
+
}), e || console.log("Bone not found: ", s, c.name), e;
|
|
106
107
|
}
|
|
107
|
-
async function
|
|
108
|
-
const e = await
|
|
108
|
+
async function Ce(c, s) {
|
|
109
|
+
const e = await de.createFromOptions(c, {
|
|
109
110
|
baseOptions: {
|
|
110
111
|
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "pose_landmarker_lite.task",
|
|
111
112
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task`,
|
|
@@ -115,9 +116,9 @@ async function We(h, s) {
|
|
|
115
116
|
runningMode: "VIDEO",
|
|
116
117
|
numPoses: 1
|
|
117
118
|
});
|
|
118
|
-
return new
|
|
119
|
+
return new $e(e, s);
|
|
119
120
|
}
|
|
120
|
-
const
|
|
121
|
+
const ze = {
|
|
121
122
|
hips: [24, 23],
|
|
122
123
|
neck: [12, 11],
|
|
123
124
|
leftLeg: 23,
|
|
@@ -139,12 +140,12 @@ const Ee = {
|
|
|
139
140
|
torso: [24, 23, 12, 11],
|
|
140
141
|
leftEar: 7,
|
|
141
142
|
rightEar: 8
|
|
142
|
-
},
|
|
143
|
-
class
|
|
143
|
+
}, Be = new m.Vector3(), K = new m.Vector3(), Ie = new m.Vector3(), Ue = new m.Vector3(), Ge = new m.Vector3(), X = new m.Vector3(), Q = new m.Vector3();
|
|
144
|
+
class $e extends I {
|
|
144
145
|
constructor(e, t) {
|
|
145
|
-
super(
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
super(ze, de.POSE_CONNECTIONS);
|
|
147
|
+
x(this, "_leftWristNormalizedPosition");
|
|
148
|
+
x(this, "_rightWristNormalizedPosition");
|
|
148
149
|
this.poseLandmarker = e, this.config = t, this.root.scale.y *= -2, this.root.scale.z *= -2, this.root.scale.x *= 2;
|
|
149
150
|
}
|
|
150
151
|
/**
|
|
@@ -161,7 +162,8 @@ class ge extends _ {
|
|
|
161
162
|
}
|
|
162
163
|
predict(e, t) {
|
|
163
164
|
this.poseLandmarker.detectForVideo(e, performance.now(), (o) => {
|
|
164
|
-
|
|
165
|
+
var n;
|
|
166
|
+
o.landmarks.length != 0 && (this.updateLandmarks(o.worldLandmarks[0], ((n = this.config) == null ? void 0 : n.drawLandmarks) === !1 ? void 0 : o.landmarks[0], t), this._leftWristNormalizedPosition = o.landmarks[0][this.points.leftWrist], this._rightWristNormalizedPosition = o.landmarks[0][this.points.rightWrist]);
|
|
165
167
|
});
|
|
166
168
|
}
|
|
167
169
|
// override sync ( delta:number, objects: BoneBinding[] ) {
|
|
@@ -174,42 +176,42 @@ class ge extends _ {
|
|
|
174
176
|
bind(e, t) {
|
|
175
177
|
var a;
|
|
176
178
|
const o = {
|
|
177
|
-
hips:
|
|
178
|
-
neck:
|
|
179
|
-
leftArm:
|
|
180
|
-
leftElbow:
|
|
181
|
-
leftWrist:
|
|
182
|
-
rightArm:
|
|
183
|
-
rightElbow:
|
|
184
|
-
rightWrist:
|
|
185
|
-
head:
|
|
186
|
-
torso:
|
|
187
|
-
leftLeg:
|
|
188
|
-
leftKnee:
|
|
189
|
-
leftFoot:
|
|
190
|
-
rightLeg:
|
|
191
|
-
rightKnee:
|
|
192
|
-
rightFoot:
|
|
179
|
+
hips: v(e, t.hips),
|
|
180
|
+
neck: v(e, t.neck),
|
|
181
|
+
leftArm: v(e, t.armL),
|
|
182
|
+
leftElbow: v(e, t.forearmL),
|
|
183
|
+
leftWrist: v(e, t.handL),
|
|
184
|
+
rightArm: v(e, t.armR),
|
|
185
|
+
rightElbow: v(e, t.forearmR),
|
|
186
|
+
rightWrist: v(e, t.handR),
|
|
187
|
+
head: v(e, t.head),
|
|
188
|
+
torso: v(e, t.torso),
|
|
189
|
+
leftLeg: v(e, t.thighL),
|
|
190
|
+
leftKnee: v(e, t.shinL),
|
|
191
|
+
leftFoot: v(e, t.footL),
|
|
192
|
+
rightLeg: v(e, t.thighR),
|
|
193
|
+
rightKnee: v(e, t.shinR),
|
|
194
|
+
rightFoot: v(e, t.footR)
|
|
193
195
|
};
|
|
194
196
|
(a = this.config) != null && a.ignoreLegs && (delete o.leftLeg, delete o.leftKnee, delete o.leftFoot, delete o.leftToes, delete o.rightLeg, delete o.rightKnee, delete o.rightFoot, delete o.rightToes);
|
|
195
|
-
const
|
|
197
|
+
const n = new m.Vector3(), r = new m.Vector3(), i = (l, p, f, R, d, u) => {
|
|
196
198
|
if (!p) return;
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
|
|
199
|
+
const w = this.marks[R].getWorldPosition(n).sub(this.marks[f].getWorldPosition(r)).normalize();
|
|
200
|
+
A(X, p, e).add(w).applyMatrix4(e.matrixWorld), A(Q, p, e).add(d).applyMatrix4(e.matrixWorld);
|
|
201
|
+
const h = this.getGhost(p);
|
|
202
|
+
M(h, X, Q, u), h.rotateX(Math.PI / 2), h.lerp(p, l);
|
|
201
203
|
};
|
|
202
204
|
return {
|
|
203
|
-
update: (
|
|
204
|
-
const p = this.marks.leftLeg.getWorldPosition(
|
|
205
|
-
|
|
205
|
+
update: (l) => {
|
|
206
|
+
const p = this.marks.leftLeg.getWorldPosition(Be).sub(this.marks.rightLeg.getWorldPosition(K)).normalize(), f = this.marks.leftArm.getWorldPosition(K).sub(this.marks.rightArm.getWorldPosition(Ie)).normalize(), R = this.marks.leftEar.getWorldPosition(Ue).sub(this.marks.rightEar.getWorldPosition(Ge)).normalize();
|
|
207
|
+
i(l, o.hips, "hips", "torso", p, "+x"), i(l, o.torso, "torso", "neck", f, "+x"), i(l, o.neck, "neck", "head", R, "+x"), i(l, o.head, "neck", "head", R, "+x"), i(l, o.leftArm, "leftArm", "leftElbow", f, "-x"), i(l, o.leftElbow, "leftElbow", "leftWrist", f, "-x"), i(l, o.leftLeg, "leftLeg", "leftKnee", p, "+x"), i(l, o.leftKnee, "leftKnee", "leftFoot", p, "+x"), i(l, o.leftFoot, "leftFoot", "leftToes", p, "+x"), i(l, o.rightArm, "rightArm", "rightElbow", f, "-x"), i(l, o.rightElbow, "rightElbow", "rightWrist", f, "-x"), i(l, o.rightLeg, "rightLeg", "rightKnee", p, "+x"), i(l, o.rightKnee, "rightKnee", "rightFoot", p, "+x"), i(l, o.rightFoot, "rightFoot", "rightToes", p, "+x");
|
|
206
208
|
}
|
|
207
209
|
};
|
|
208
210
|
}
|
|
209
211
|
}
|
|
210
|
-
const
|
|
211
|
-
async function
|
|
212
|
-
const e = await
|
|
212
|
+
const Y = new m.Vector2(), Z = new m.Vector2();
|
|
213
|
+
async function He(c, s) {
|
|
214
|
+
const e = await z.createFromOptions(c, {
|
|
213
215
|
baseOptions: {
|
|
214
216
|
modelAssetPath: s.modelPath ?? "hand_landmarker.task",
|
|
215
217
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
|
|
@@ -217,20 +219,20 @@ async function Se(h, s) {
|
|
|
217
219
|
},
|
|
218
220
|
runningMode: "VIDEO",
|
|
219
221
|
numHands: 2
|
|
220
|
-
}), t = (o,
|
|
222
|
+
}), t = (o, n, r) => {
|
|
221
223
|
try {
|
|
222
|
-
|
|
223
|
-
} catch (
|
|
224
|
-
return console.warn("No pose data... will just be optimitic and say yes to everything.",
|
|
224
|
+
Y.copy(o()), Z.copy(n());
|
|
225
|
+
} catch (i) {
|
|
226
|
+
return console.warn("No pose data... will just be optimitic and say yes to everything.", i), !0;
|
|
225
227
|
}
|
|
226
|
-
return
|
|
228
|
+
return Y.distanceTo(r) < Z.distanceTo(r);
|
|
227
229
|
};
|
|
228
230
|
return {
|
|
229
|
-
left: new
|
|
230
|
-
right: new
|
|
231
|
+
left: new re(e, "Left", t.bind(null, s.leftWrist, s.rightWrist), s.drawLandmarks),
|
|
232
|
+
right: new re(e, "Right", t.bind(null, s.rightWrist, s.leftWrist), s.drawLandmarks)
|
|
231
233
|
};
|
|
232
234
|
}
|
|
233
|
-
const
|
|
235
|
+
const je = {
|
|
234
236
|
wrist: 0,
|
|
235
237
|
palm: [9, 13],
|
|
236
238
|
thumb1: 1,
|
|
@@ -253,38 +255,38 @@ const Ne = {
|
|
|
253
255
|
pinky2: 18,
|
|
254
256
|
pinky3: 19,
|
|
255
257
|
pinky4: 20
|
|
256
|
-
},
|
|
258
|
+
}, N = {
|
|
257
259
|
thumb: ["thumb1", "thumb2", "thumb3", "thumb4"],
|
|
258
260
|
index: ["index1", "index2", "index3", "index4"],
|
|
259
261
|
middle: ["middle1", "middle2", "middle3", "middle4"],
|
|
260
262
|
ring: ["ring1", "ring2", "ring3", "ring4"],
|
|
261
263
|
pinky: ["pinky1", "pinky2", "pinky3", "pinky4"]
|
|
262
|
-
},
|
|
263
|
-
new
|
|
264
|
-
new
|
|
265
|
-
new
|
|
266
|
-
const
|
|
267
|
-
class
|
|
268
|
-
constructor(e, t, o) {
|
|
269
|
-
super(
|
|
270
|
-
|
|
271
|
-
|
|
264
|
+
}, qe = new m.Vector3(), J = new m.Vector3(), ee = new m.Vector3(), te = new m.Vector3(), oe = new m.Vector3(), se = new m.Vector3();
|
|
265
|
+
new m.Vector3();
|
|
266
|
+
new m.Vector3();
|
|
267
|
+
new m.Vector3();
|
|
268
|
+
const ne = new m.Vector3(), ie = Math.PI / 2, Ke = new m.Vector3(0, -1, 0);
|
|
269
|
+
class re extends I {
|
|
270
|
+
constructor(e, t, o, n = !0) {
|
|
271
|
+
super(je, z.HAND_CONNECTIONS);
|
|
272
|
+
x(this, "sign");
|
|
273
|
+
x(this, "isLeft");
|
|
272
274
|
/**
|
|
273
275
|
* the axis used to look at the pole
|
|
274
276
|
*/
|
|
275
|
-
|
|
276
|
-
this.handLandmarker = e, this.side = t, this.isMyWrist = o, this.sign = this.side == "Left" ? -1 : 1, this.isLeft = this.side == "Left", this.lookAtPoleAxis = this.sign < 0 ? "+x" : "-x", this.root.scale.setScalar(7), this.root.scale.y *= -1, this.root.scale.z *= -1;
|
|
277
|
+
x(this, "lookAtPoleAxis");
|
|
278
|
+
this.handLandmarker = e, this.side = t, this.isMyWrist = o, this.drawLandmarks = n, this.sign = this.side == "Left" ? -1 : 1, this.isLeft = this.side == "Left", this.lookAtPoleAxis = this.sign < 0 ? "+x" : "-x", this.root.scale.setScalar(7), this.root.scale.y *= -1, this.root.scale.z *= -1;
|
|
277
279
|
}
|
|
278
280
|
predict(e, t) {
|
|
279
281
|
const o = this.handLandmarker.detectForVideo(e, performance.now());
|
|
280
282
|
if (o.landmarks.length)
|
|
281
|
-
for (let
|
|
282
|
-
const r = o.landmarks[
|
|
283
|
-
if (this.isMyWrist(
|
|
284
|
-
this.updateLandmarks(o.worldLandmarks[
|
|
283
|
+
for (let n = 0; n < o.landmarks.length; n++) {
|
|
284
|
+
const r = o.landmarks[n], i = r[this.points.wrist];
|
|
285
|
+
if (this.isMyWrist(i)) {
|
|
286
|
+
this.updateLandmarks(o.worldLandmarks[n]), this.drawLandmarks && (t.drawConnectors(r, z.HAND_CONNECTIONS, {
|
|
285
287
|
color: this.side == "Left" ? "#00FF00" : "#0000FF",
|
|
286
288
|
lineWidth: 4
|
|
287
|
-
}), t.drawLandmarks(r, { color: this.side == "Left" ? "#00FF00" : "#0000FF", lineWidth: 3, radius: 1 });
|
|
289
|
+
}), t.drawLandmarks(r, { color: this.side == "Left" ? "#00FF00" : "#0000FF", lineWidth: 3, radius: 1 }));
|
|
288
290
|
break;
|
|
289
291
|
}
|
|
290
292
|
}
|
|
@@ -299,56 +301,56 @@ class te extends _ {
|
|
|
299
301
|
* @see https://ai.google.dev/edge/mediapipe/solutions/vision/hand_landmarker
|
|
300
302
|
*/
|
|
301
303
|
syncHandBones(e, t, o) {
|
|
302
|
-
const
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
).normalize(), r =
|
|
306
|
-
if (!(r.dot(
|
|
304
|
+
const n = qe.crossVectors(
|
|
305
|
+
J.copy(this.marks.index1.worldPosition).sub(this.marks.wrist.worldPosition),
|
|
306
|
+
ee.copy(this.marks.pinky1.worldPosition).sub(this.marks.wrist.worldPosition)
|
|
307
|
+
).normalize(), r = J.copy(this.marks.palm.worldPosition).sub(this.marks.wrist.worldPosition).normalize(), i = ee.copy(this.marks.pinky1.worldPosition).sub(this.marks.index1.worldPosition).normalize();
|
|
308
|
+
if (!(r.dot(Ke) > 0.8)) {
|
|
307
309
|
if (t.wrist) {
|
|
308
|
-
const a =
|
|
309
|
-
|
|
310
|
+
const a = A(te, t.wrist, o).add(r).applyMatrix4(o.matrixWorld), l = A(oe, t.wrist, o).sub(i).applyMatrix4(o.matrixWorld), p = this.getGhost(t.wrist);
|
|
311
|
+
M(p, a, l, "-y"), p.rotateX(ie), p.lerp(t.wrist, e);
|
|
310
312
|
}
|
|
311
|
-
this.syncFinger(e, o,
|
|
313
|
+
this.syncFinger(e, o, n, r, i, t, N.index, "middle1"), this.syncFinger(e, o, n, r, i, t, N.middle, "ring1"), this.syncFinger(e, o, n, r, i, t, N.ring, "pinky1"), this.syncFinger(e, o, n, r, i, t, N.pinky, "ring1", !0), this.syncFinger(e, o, n, r, i, t, N.thumb, "index1");
|
|
312
314
|
}
|
|
313
315
|
}
|
|
314
|
-
syncFinger(e, t, o,
|
|
316
|
+
syncFinger(e, t, o, n, r, i, a, l, p = !1) {
|
|
315
317
|
for (let f = 0; f < a.length - 1; f++) {
|
|
316
|
-
const R =
|
|
318
|
+
const R = i[a[f]];
|
|
317
319
|
if (!R) continue;
|
|
318
|
-
const d = this.getGhost(R),
|
|
320
|
+
const d = this.getGhost(R), u = te.copy(this.marks[a[f + 1]].worldPosition).sub(this.marks[a[f]].worldPosition).normalize(), w = A(oe, R, t);
|
|
319
321
|
if (f == 0) {
|
|
320
|
-
const
|
|
321
|
-
p &&
|
|
322
|
+
const h = se.copy(this.marks[l].worldPosition).sub(this.marks[a[0]].worldPosition).normalize();
|
|
323
|
+
p && h.negate(), ne.copy(h), M(
|
|
322
324
|
d,
|
|
323
|
-
|
|
324
|
-
|
|
325
|
+
u.add(w).applyMatrix4(t.matrixWorld),
|
|
326
|
+
h.add(w).applyMatrix4(t.matrixWorld),
|
|
325
327
|
this.lookAtPoleAxis
|
|
326
328
|
);
|
|
327
329
|
} else
|
|
328
|
-
|
|
330
|
+
M(
|
|
329
331
|
d,
|
|
330
|
-
|
|
331
|
-
|
|
332
|
+
u.add(w).applyMatrix4(t.matrixWorld),
|
|
333
|
+
se.copy(ne).add(w).applyMatrix4(t.matrixWorld),
|
|
332
334
|
this.lookAtPoleAxis
|
|
333
335
|
);
|
|
334
|
-
d.rotateX(
|
|
336
|
+
d.rotateX(ie), d.lerp(R, e);
|
|
335
337
|
}
|
|
336
338
|
}
|
|
337
339
|
bind(e, t) {
|
|
338
|
-
const o = {},
|
|
339
|
-
const a =
|
|
340
|
+
const o = {}, n = (r, i) => {
|
|
341
|
+
const a = v(e, r);
|
|
340
342
|
if (a)
|
|
341
|
-
return o[
|
|
343
|
+
return o[i] = a, i;
|
|
342
344
|
};
|
|
343
|
-
return
|
|
345
|
+
return n(this.isLeft ? t.handL : t.handR, "wrist"), n(this.isLeft ? t.index1L : t.index1R, "index1"), n(this.isLeft ? t.index2L : t.index2R, "index2"), n(this.isLeft ? t.index3L : t.index3R, "index3"), n(this.isLeft ? t.middle1L : t.middle1R, "middle1"), n(this.isLeft ? t.middle2L : t.middle2R, "middle2"), n(this.isLeft ? t.middle3L : t.middle3R, "middle3"), n(this.isLeft ? t.ring1L : t.ring1R, "ring1"), n(this.isLeft ? t.ring2L : t.ring2R, "ring2"), n(this.isLeft ? t.ring3L : t.ring3R, "ring3"), n(this.isLeft ? t.pinky1L : t.pinky1R, "pinky1"), n(this.isLeft ? t.pinky2L : t.pinky2R, "pinky2"), n(this.isLeft ? t.pinky3L : t.pinky3R, "pinky3"), n(this.isLeft ? t.thumb1L : t.thumb1R, "thumb1"), n(this.isLeft ? t.thumb2L : t.thumb2R, "thumb2"), n(this.isLeft ? t.thumb3L : t.thumb3R, "thumb3"), {
|
|
344
346
|
update: (r) => {
|
|
345
347
|
this.syncHandBones(r, o, e);
|
|
346
348
|
}
|
|
347
349
|
};
|
|
348
350
|
}
|
|
349
351
|
}
|
|
350
|
-
async function
|
|
351
|
-
const e = await
|
|
352
|
+
async function Xe(c, s) {
|
|
353
|
+
const e = await B.createFromOptions(c, {
|
|
352
354
|
baseOptions: {
|
|
353
355
|
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "face_landmarker.task",
|
|
354
356
|
delegate: "GPU"
|
|
@@ -357,9 +359,9 @@ async function Te(h, s) {
|
|
|
357
359
|
runningMode: "VIDEO",
|
|
358
360
|
numFaces: 1
|
|
359
361
|
});
|
|
360
|
-
return new
|
|
362
|
+
return new tt(e, { ...s });
|
|
361
363
|
}
|
|
362
|
-
const
|
|
364
|
+
const Qe = {
|
|
363
365
|
eyeL: 473,
|
|
364
366
|
eyeR: 468,
|
|
365
367
|
eyeStartL: 463,
|
|
@@ -372,71 +374,116 @@ const De = {
|
|
|
372
374
|
noseBone: 6,
|
|
373
375
|
chin: 152,
|
|
374
376
|
forehead: 10
|
|
375
|
-
},
|
|
377
|
+
}, Ye = new P(), Ze = new P(), le = new P(), Je = new P(), et = new P();
|
|
376
378
|
new P();
|
|
377
|
-
class
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
this
|
|
379
|
+
class tt extends I {
|
|
380
|
+
constructor(e, t) {
|
|
381
|
+
super(Qe, B.FACE_LANDMARKS_TESSELATION);
|
|
382
|
+
x(this, "blendshapeCategories");
|
|
383
|
+
x(this, "blendshapeMap", /* @__PURE__ */ new Map());
|
|
384
|
+
x(this, "smoothed", {});
|
|
385
|
+
x(this, "smoothing", 3e-4);
|
|
386
|
+
// lower = smoother but more lag, higher = more responsive
|
|
387
|
+
x(this, "_faceLandmarks", []);
|
|
388
|
+
this.faceLandmarker = e, this.cfg = t, this.root.scale.y *= -1, this.root.scale.z *= -1, this.root.scale.multiplyScalar(3);
|
|
386
389
|
}
|
|
387
390
|
predict(e, t) {
|
|
388
|
-
var
|
|
391
|
+
var n, r, i;
|
|
389
392
|
const o = this.faceLandmarker.detectForVideo(e, performance.now());
|
|
390
|
-
o.faceLandmarks[0] && (t.drawConnectors(o.faceLandmarks[0],
|
|
393
|
+
o.faceLandmarks[0] && (this.cfg.drawLandmarks && (t.drawConnectors(o.faceLandmarks[0], B.FACE_LANDMARKS_TESSELATION, { color: "#00fff2ff", lineWidth: 0.1 }), t.drawLandmarks(o.faceLandmarks[0], { color: "#00ff00", lineWidth: 0.1, radius: 0.4 })), this.updateLandmarks(o.faceLandmarks[0], o.faceLandmarks[0])), this._faceLandmarks = o.faceLandmarks[0], this.blendshapeCategories = (r = (n = o.faceBlendshapes) == null ? void 0 : n[0]) == null ? void 0 : r.categories, (i = this.blendshapeCategories) == null || i.forEach((a) => {
|
|
391
394
|
this.blendshapeMap.set(a.categoryName, a.score);
|
|
392
395
|
});
|
|
393
396
|
}
|
|
397
|
+
get lastKnownLandmarks() {
|
|
398
|
+
return this._faceLandmarks;
|
|
399
|
+
}
|
|
394
400
|
bindShapeKeys(e) {
|
|
395
401
|
const t = e.morphTargetDictionary;
|
|
396
402
|
return {
|
|
397
403
|
update: (o) => {
|
|
398
|
-
var
|
|
399
|
-
(
|
|
400
|
-
const { categoryName:
|
|
401
|
-
if (!(t != null && t.hasOwnProperty(
|
|
402
|
-
this.smoothed[
|
|
403
|
-
const
|
|
404
|
-
this.smoothed[
|
|
404
|
+
var n;
|
|
405
|
+
(n = this.blendshapeCategories) == null || n.forEach((r) => {
|
|
406
|
+
const { categoryName: i, score: a } = r;
|
|
407
|
+
if (!(t != null && t.hasOwnProperty(i))) return;
|
|
408
|
+
this.smoothed[i] === void 0 && (this.smoothed[i] = a);
|
|
409
|
+
const l = 1 - Math.pow(this.smoothing, o);
|
|
410
|
+
this.smoothed[i] += (a - this.smoothed[i]) * l, e.morphTargetInfluences[t[i]] = this.smoothed[i];
|
|
405
411
|
});
|
|
406
412
|
}
|
|
407
413
|
};
|
|
408
414
|
}
|
|
409
415
|
bind(e) {
|
|
410
|
-
const t = new
|
|
416
|
+
const t = new ae(e, "L"), o = new ae(e, "R"), n = v(e, "head");
|
|
411
417
|
return {
|
|
412
418
|
update: (r) => {
|
|
413
|
-
if (t.update(r, this.blendshapeMap), o.update(r, this.blendshapeMap), !
|
|
414
|
-
const
|
|
415
|
-
|
|
419
|
+
if (t.update(r, this.blendshapeMap), o.update(r, this.blendshapeMap), !n) return;
|
|
420
|
+
const i = Ye.copy(this.marks.earL.worldPosition), a = Ze.copy(this.marks.earR.worldPosition), l = le.subVectors(i, a).multiplyScalar(0.5).add(a), p = Je.subVectors(this.marks.noseTip.worldPosition, l), f = i.sub(a), R = A(et, n, e), d = f.add(R).applyMatrix4(e.matrixWorld), u = p.add(R).applyMatrix4(e.matrixWorld);
|
|
421
|
+
M(n, u, d, "+x");
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* The mesh is assumed to be a canonical_face_model because we will be manipulating it's vertices.
|
|
427
|
+
*
|
|
428
|
+
* @see https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/modules/face_geometry
|
|
429
|
+
* @param mesh basically either the original or a clone of the canonical_face_model
|
|
430
|
+
*/
|
|
431
|
+
bindGeometry(e, t) {
|
|
432
|
+
const o = new P(), n = new P(), r = e.geometry, i = r.attributes.position, a = [], l = /* @__PURE__ */ new Map(), p = [];
|
|
433
|
+
for (let h = 0; h < i.count; h++) {
|
|
434
|
+
const y = `${i.getX(h).toFixed(6)},${i.getY(h).toFixed(6)},${i.getZ(h).toFixed(6)}`;
|
|
435
|
+
l.has(y) || (l.set(y, a.length), a.push(h)), p.push(l.get(y));
|
|
436
|
+
}
|
|
437
|
+
const f = xe(new P(0.5, 0.5, 0.5));
|
|
438
|
+
r.setAttribute("landmarkIndex", new ye(new Uint16Array(p), 1));
|
|
439
|
+
const R = Re("landmarkIndex", "uint"), d = Pe(468, "vec3"), u = ve(d.element(R)).xy;
|
|
440
|
+
let w;
|
|
441
|
+
return {
|
|
442
|
+
update: (h) => {
|
|
443
|
+
var g, V;
|
|
444
|
+
if (!w) {
|
|
445
|
+
if (w = (V = (g = this.cfg) == null ? void 0 : g.videoElementRef) == null ? void 0 : V.call(g), !w) return;
|
|
446
|
+
const W = new we(w);
|
|
447
|
+
W.colorSpace = Le;
|
|
448
|
+
const U = d.element(R).sub(f).xzy.mul(Ee(1.3, -1.1, 1)), G = We(W, Fe(u.x, u.y.oneMinus()));
|
|
449
|
+
t ? t(U, G) : e.material = new ke({
|
|
450
|
+
positionNode: U,
|
|
451
|
+
colorNode: G,
|
|
452
|
+
roughness: 0.93
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
const y = this.lastKnownLandmarks;
|
|
456
|
+
if (!y) return;
|
|
457
|
+
const b = d.value.array;
|
|
458
|
+
for (let W = 0; W < y.length; W++)
|
|
459
|
+
b[W * 3] = y[W].x, b[W * 3 + 1] = y[W].y, b[W * 3 + 2] = y[W].z;
|
|
460
|
+
d.value.needsUpdate = !0;
|
|
461
|
+
const E = y[234], k = y[93], L = y[454], F = y[323];
|
|
462
|
+
o.copy(E).sub(k).divideScalar(2).add(k), n.copy(L).sub(F).divideScalar(2).add(F), f.value.subVectors(n, o).divideScalar(2).add(o), f.needsUpdate = !0;
|
|
416
463
|
}
|
|
417
464
|
};
|
|
418
465
|
}
|
|
419
466
|
}
|
|
420
|
-
class
|
|
467
|
+
class ae {
|
|
421
468
|
constructor(s, e) {
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
469
|
+
x(this, "eyeBone");
|
|
470
|
+
x(this, "eyeLookOut");
|
|
471
|
+
x(this, "eyeLookIn");
|
|
472
|
+
x(this, "eyeLookUp");
|
|
473
|
+
x(this, "eyeLookDown");
|
|
474
|
+
x(this, "sign", 1);
|
|
428
475
|
this.rig = s, this.side = e, this.eyeBone = s.getObjectByName(`eye${e}`);
|
|
429
476
|
const t = e == "L" ? "Left" : "Right";
|
|
430
477
|
this.eyeLookOut = `eyeLookOut${t}`, this.eyeLookIn = `eyeLookIn${t}`, this.eyeLookUp = `eyeLookUp${t}`, this.eyeLookDown = `eyeLookDown${t}`, this.sign = e == "L" ? -1 : 1;
|
|
431
478
|
}
|
|
432
479
|
update(s, e) {
|
|
433
480
|
if (!this.eyeBone) return;
|
|
434
|
-
|
|
435
|
-
const t = e.get(this.eyeLookOut) ?? 0, o = e.get(this.eyeLookIn) ?? 0,
|
|
436
|
-
this.eyeBone.rotation.y =
|
|
481
|
+
A(le, this.eyeBone, this.rig);
|
|
482
|
+
const t = e.get(this.eyeLookOut) ?? 0, o = e.get(this.eyeLookIn) ?? 0, n = e.get(this.eyeLookUp) ?? 0, r = e.get(this.eyeLookDown) ?? 0, i = o - t, a = r - n;
|
|
483
|
+
this.eyeBone.rotation.y = i * this.sign / 2, this.eyeBone.rotation.x = a / 2;
|
|
437
484
|
}
|
|
438
485
|
}
|
|
439
|
-
const
|
|
486
|
+
const he = {
|
|
440
487
|
faceMesh: "face",
|
|
441
488
|
head: "head",
|
|
442
489
|
hips: "hips",
|
|
@@ -484,7 +531,7 @@ const re = {
|
|
|
484
531
|
thumb1R: "thumb1R",
|
|
485
532
|
thumb2R: "thumb2R",
|
|
486
533
|
thumb3R: "thumb3R"
|
|
487
|
-
},
|
|
534
|
+
}, ot = [
|
|
488
535
|
"eyeBlinkLeft",
|
|
489
536
|
"eyeBlinkRight",
|
|
490
537
|
"eyeLookDownLeft",
|
|
@@ -538,206 +585,211 @@ const re = {
|
|
|
538
585
|
"mouthUpperUpRight",
|
|
539
586
|
"tongueOut"
|
|
540
587
|
];
|
|
541
|
-
function
|
|
542
|
-
const t = [], o =
|
|
588
|
+
function st(c, s, e = 30) {
|
|
589
|
+
const t = [], o = c.getObjectByName(
|
|
543
590
|
s.faceMesh
|
|
544
|
-
),
|
|
591
|
+
), n = /* @__PURE__ */ new Set();
|
|
545
592
|
if (o != null && o.morphTargetDictionary)
|
|
546
|
-
for (const
|
|
547
|
-
|
|
593
|
+
for (const w in o.morphTargetDictionary)
|
|
594
|
+
ot.includes(w) && n.add(w);
|
|
548
595
|
const r = Object.keys(s);
|
|
549
|
-
|
|
550
|
-
if (
|
|
551
|
-
const
|
|
552
|
-
|
|
553
|
-
ref:
|
|
554
|
-
name:
|
|
555
|
-
normalizedName:
|
|
596
|
+
c.traverse((w) => {
|
|
597
|
+
if (w instanceof O.Bone) {
|
|
598
|
+
const h = r.find((y) => w.name.indexOf(s[y]) === 0);
|
|
599
|
+
h && t.push({
|
|
600
|
+
ref: w,
|
|
601
|
+
name: w.name,
|
|
602
|
+
normalizedName: h
|
|
556
603
|
});
|
|
557
604
|
}
|
|
558
605
|
});
|
|
559
|
-
const
|
|
606
|
+
const i = [], a = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map();
|
|
560
607
|
let p = !1, f = 0;
|
|
561
608
|
function R() {
|
|
562
|
-
a.clear(),
|
|
609
|
+
a.clear(), l.clear(), i.length = 0, f = performance.now() / 1e3, p = !0;
|
|
563
610
|
}
|
|
564
611
|
function d() {
|
|
565
612
|
if (!p) return;
|
|
566
|
-
const
|
|
567
|
-
|
|
568
|
-
for (const
|
|
569
|
-
a.has(
|
|
570
|
-
const
|
|
571
|
-
|
|
613
|
+
const w = performance.now() / 1e3 - f;
|
|
614
|
+
i.push(w);
|
|
615
|
+
for (const h of t) {
|
|
616
|
+
a.has(h.name) || a.set(h.name, []);
|
|
617
|
+
const y = a.get(h.name), b = h.ref.quaternion;
|
|
618
|
+
y.push(b.x, b.y, b.z, b.w);
|
|
572
619
|
}
|
|
573
|
-
for (const
|
|
574
|
-
|
|
575
|
-
const
|
|
576
|
-
|
|
620
|
+
for (const h of n) {
|
|
621
|
+
l.has(h) || l.set(h, []);
|
|
622
|
+
const y = l.get(h), b = o.morphTargetDictionary[h], E = o.morphTargetInfluences[b];
|
|
623
|
+
y.push(E);
|
|
577
624
|
}
|
|
578
625
|
}
|
|
579
|
-
function
|
|
626
|
+
function u(w = "RecordedClip") {
|
|
580
627
|
p = !1;
|
|
581
|
-
const
|
|
628
|
+
const h = [];
|
|
582
629
|
for (const [b, E] of a)
|
|
583
|
-
|
|
584
|
-
new
|
|
630
|
+
h.push(
|
|
631
|
+
new O.QuaternionKeyframeTrack(
|
|
585
632
|
`${b}.quaternion`,
|
|
586
|
-
|
|
633
|
+
i,
|
|
587
634
|
E
|
|
588
635
|
)
|
|
589
636
|
);
|
|
590
|
-
for (const [b, E] of
|
|
591
|
-
|
|
592
|
-
new
|
|
593
|
-
`${
|
|
594
|
-
|
|
637
|
+
for (const [b, E] of l)
|
|
638
|
+
h.push(
|
|
639
|
+
new O.NumberKeyframeTrack(
|
|
640
|
+
`${he.faceMesh}.morphTargetInfluences[${b}]`,
|
|
641
|
+
i,
|
|
595
642
|
E
|
|
596
643
|
)
|
|
597
644
|
);
|
|
598
|
-
const
|
|
645
|
+
const y = new O.AnimationClip(w, -1, h);
|
|
599
646
|
return {
|
|
600
|
-
clip:
|
|
647
|
+
clip: y,
|
|
601
648
|
saveToFile: () => {
|
|
602
|
-
new
|
|
603
|
-
|
|
649
|
+
new Ae().parse(
|
|
650
|
+
c,
|
|
604
651
|
(E) => {
|
|
605
|
-
const
|
|
652
|
+
const k = new Blob([E], {
|
|
606
653
|
type: "model/gltf-binary"
|
|
607
|
-
}),
|
|
608
|
-
|
|
654
|
+
}), L = URL.createObjectURL(k), F = document.createElement("a");
|
|
655
|
+
F.href = L, F.download = w + ".glb", F.click();
|
|
609
656
|
},
|
|
610
657
|
(E) => {
|
|
611
658
|
console.error(E);
|
|
612
659
|
},
|
|
613
660
|
{
|
|
614
661
|
binary: !0,
|
|
615
|
-
animations: [
|
|
662
|
+
animations: [y]
|
|
616
663
|
}
|
|
617
664
|
);
|
|
618
665
|
}
|
|
619
666
|
};
|
|
620
667
|
}
|
|
621
|
-
return { start: R, captureFrame: d, stop:
|
|
668
|
+
return { start: R, captureFrame: d, stop: u, isRecording: () => p };
|
|
622
669
|
}
|
|
623
|
-
const
|
|
624
|
-
var
|
|
625
|
-
return !!((
|
|
670
|
+
const nt = () => {
|
|
671
|
+
var c;
|
|
672
|
+
return !!((c = navigator.mediaDevices) != null && c.getUserMedia);
|
|
626
673
|
};
|
|
627
|
-
async function
|
|
674
|
+
async function ht(c) {
|
|
628
675
|
const s = {
|
|
629
676
|
debugFrame: void 0,
|
|
630
677
|
displayScale: 1,
|
|
631
678
|
ignoreLegs: !1,
|
|
632
679
|
debugVideo: void 0,
|
|
633
680
|
ignoreFace: !1,
|
|
681
|
+
onlyFace: !1,
|
|
682
|
+
drawLandmarksOverlay: !0,
|
|
634
683
|
modelPaths: {
|
|
635
684
|
vision: "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm",
|
|
636
685
|
pose: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task",
|
|
637
686
|
hand: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
|
|
638
687
|
face: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task"
|
|
639
688
|
},
|
|
640
|
-
...
|
|
689
|
+
...c
|
|
641
690
|
};
|
|
642
691
|
let e;
|
|
643
|
-
const t = await
|
|
692
|
+
const t = await ue.forVisionTasks(s.modelPaths.vision ?? "/wasm"), o = s.onlyFace ? void 0 : await Ce(t, {
|
|
644
693
|
ignoreLegs: s.ignoreLegs,
|
|
645
|
-
modelPath: s.modelPaths.pose
|
|
646
|
-
|
|
694
|
+
modelPath: s.modelPaths.pose,
|
|
695
|
+
drawLandmarks: s.drawLandmarksOverlay
|
|
696
|
+
}), n = s.onlyFace ? void 0 : await He(t, {
|
|
647
697
|
leftWrist: () => o.leftWristNormalizedPosition,
|
|
648
698
|
rightWrist: () => o.rightWristNormalizedPosition,
|
|
649
699
|
modelPath: s.modelPaths.hand,
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
a
|
|
700
|
+
drawLandmarks: s.drawLandmarksOverlay,
|
|
701
|
+
...c == null ? void 0 : c.handsTrackerOptions
|
|
702
|
+
}), r = s.ignoreFace ? void 0 : await Xe(t, { modelPath: s.modelPaths.face, videoElementRef: () => e, drawLandmarks: s.drawLandmarksOverlay }), i = document.createElement("div");
|
|
703
|
+
i.style.position = "absolute", i.style.top = "0px", i.style.left = "0px", i.style.zIndex = "21", i.classList.add("three-mediapipe-rig"), document.body.appendChild(i);
|
|
704
|
+
const a = document.createElement("canvas"), l = a.getContext("2d"), p = new fe(l);
|
|
705
|
+
a.style.zIndex = "22", a.style.position = "absolute", a.style.top = "0px", a.style.left = "0px", a.style.pointerEvents = "none", i.appendChild(a);
|
|
655
706
|
function f(d) {
|
|
656
|
-
|
|
707
|
+
l.save(), l.clearRect(0, 0, a.width, a.height), o == null || o.predict(d, p), n == null || n.left.predict(d, p), n == null || n.right.predict(d, p), r == null || r.predict(d, p), l.restore();
|
|
657
708
|
}
|
|
658
709
|
function R() {
|
|
659
|
-
e = document.createElement("video"),
|
|
710
|
+
e = document.createElement("video"), i.appendChild(e);
|
|
660
711
|
let d = -1;
|
|
661
712
|
e.style.zIndex = "21", e.style.position = "absolute", e.style.top = "0px", e.style.left = "0px", s.debugVideo && (e.src = s.debugVideo, e.controls = !0, e.loop = !0, e.muted = !0, e.controls = !0, e.play());
|
|
662
|
-
function
|
|
663
|
-
d !== e.currentTime && (f(e), d = e.currentTime), window.requestAnimationFrame(
|
|
713
|
+
function u() {
|
|
714
|
+
d !== e.currentTime && (f(e), d = e.currentTime), window.requestAnimationFrame(u);
|
|
664
715
|
}
|
|
665
716
|
e.addEventListener("loadeddata", () => {
|
|
666
|
-
e.width = e.videoWidth * s.displayScale, e.height = e.videoHeight * s.displayScale, a.width = e.videoWidth, a.height = e.videoHeight, a.style.height = e.height + "px", a.style.width = e.width + "px", window.requestAnimationFrame(
|
|
717
|
+
e.width = e.videoWidth * s.displayScale, e.height = e.videoHeight * s.displayScale, a.width = e.videoWidth, a.height = e.videoHeight, a.style.height = e.height + "px", a.style.width = e.width + "px", window.requestAnimationFrame(u);
|
|
667
718
|
});
|
|
668
719
|
}
|
|
669
720
|
if (s.debugFrame) {
|
|
670
721
|
const d = document.createElement("img");
|
|
671
|
-
d.src = s.debugFrame, d.style.zIndex = "21", d.style.position = "absolute", d.style.top = "0px", d.style.left = "0px",
|
|
722
|
+
d.src = s.debugFrame, d.style.zIndex = "21", d.style.position = "absolute", d.style.top = "0px", d.style.left = "0px", i.appendChild(d), d.addEventListener("load", () => {
|
|
672
723
|
d.width = d.naturalWidth * s.displayScale, d.height = d.naturalHeight * s.displayScale, a.width = d.naturalWidth, a.height = d.naturalWidth, a.style.width = d.width + "px", a.style.height = d.height + "px";
|
|
673
|
-
function
|
|
724
|
+
function u() {
|
|
674
725
|
f(d);
|
|
675
726
|
}
|
|
676
|
-
window.requestAnimationFrame(
|
|
727
|
+
window.requestAnimationFrame(u);
|
|
677
728
|
});
|
|
678
729
|
} else s.debugVideo && R();
|
|
679
730
|
return {
|
|
680
731
|
poseTracker: o,
|
|
681
|
-
handsTracker:
|
|
732
|
+
handsTracker: n,
|
|
682
733
|
faceTracker: r,
|
|
683
734
|
video: e,
|
|
735
|
+
canvas: a,
|
|
684
736
|
/**
|
|
685
737
|
* A div that contains the video and canvas used to display the landmarks stacked on top of each other.
|
|
686
738
|
*/
|
|
687
|
-
domElement:
|
|
739
|
+
domElement: i,
|
|
688
740
|
/**
|
|
689
741
|
* Start the webcam feed. This must be initiated by a user triggered event ( a click on a button ) due to security reasons.
|
|
690
742
|
*/
|
|
691
743
|
start: async () => {
|
|
692
744
|
let d = !1;
|
|
693
|
-
if (!
|
|
745
|
+
if (!nt())
|
|
694
746
|
throw new Error("Webcam not supported");
|
|
695
747
|
e || R();
|
|
696
|
-
let
|
|
697
|
-
function
|
|
698
|
-
console.warn("Camera track ended, attempting recovery..."),
|
|
748
|
+
let u;
|
|
749
|
+
function w(k) {
|
|
750
|
+
console.warn("Camera track ended, attempting recovery..."), h(k), y(k);
|
|
699
751
|
}
|
|
700
|
-
function
|
|
701
|
-
|
|
752
|
+
function h(k) {
|
|
753
|
+
u == null || u.getTracks().forEach((L) => L.stop()), u = void 0, k.srcObject = null;
|
|
702
754
|
}
|
|
703
|
-
async function
|
|
704
|
-
const
|
|
705
|
-
if (
|
|
755
|
+
async function y(k, L = 0) {
|
|
756
|
+
const g = Math.min(1e3 * 2 ** L, 16e3);
|
|
757
|
+
if (L >= 5)
|
|
706
758
|
throw new Error("Camera recovery failed after max attempts");
|
|
707
|
-
if (await new Promise((
|
|
759
|
+
if (await new Promise((V) => setTimeout(V, g)), !d)
|
|
708
760
|
try {
|
|
709
|
-
await b(
|
|
761
|
+
await b(k), console.log("Camera recovered successfully");
|
|
710
762
|
} catch {
|
|
711
|
-
|
|
763
|
+
y(k, L + 1);
|
|
712
764
|
}
|
|
713
765
|
}
|
|
714
|
-
async function b(
|
|
766
|
+
async function b(k) {
|
|
715
767
|
try {
|
|
716
|
-
|
|
717
|
-
|
|
768
|
+
u = await navigator.mediaDevices.getUserMedia({ video: !0 }), k.srcObject = u, await k.play(), u.getVideoTracks().forEach((L) => {
|
|
769
|
+
L.addEventListener("ended", () => w(k));
|
|
718
770
|
});
|
|
719
|
-
} catch (
|
|
720
|
-
E(
|
|
771
|
+
} catch (L) {
|
|
772
|
+
E(L, k);
|
|
721
773
|
}
|
|
722
774
|
}
|
|
723
|
-
function E(
|
|
724
|
-
if (
|
|
725
|
-
switch (
|
|
775
|
+
function E(k, L) {
|
|
776
|
+
if (k instanceof DOMException)
|
|
777
|
+
switch (k.name) {
|
|
726
778
|
case "NotAllowedError":
|
|
727
779
|
throw new Error("Permission denied — prompt user to allow camera");
|
|
728
780
|
case "NotFoundError":
|
|
729
|
-
console.error("No camera found — retry when device is connected"),
|
|
781
|
+
console.error("No camera found — retry when device is connected"), y(L);
|
|
730
782
|
break;
|
|
731
783
|
case "NotReadableError":
|
|
732
|
-
console.error("Camera in use by another app"),
|
|
784
|
+
console.error("Camera in use by another app"), y(L);
|
|
733
785
|
break;
|
|
734
786
|
default:
|
|
735
|
-
console.error("Camera error:",
|
|
787
|
+
console.error("Camera error:", k.message), y(L);
|
|
736
788
|
}
|
|
737
789
|
}
|
|
738
790
|
return await b(e), {
|
|
739
791
|
stop: () => {
|
|
740
|
-
d = !0,
|
|
792
|
+
d = !0, h(e);
|
|
741
793
|
}
|
|
742
794
|
};
|
|
743
795
|
},
|
|
@@ -746,31 +798,33 @@ async function Qe(h) {
|
|
|
746
798
|
* @param rig The rig that contains all the bones and skinned meshes of your character.
|
|
747
799
|
* @param magging The bone mapping to use for the rig.
|
|
748
800
|
*/
|
|
749
|
-
bind: (d,
|
|
750
|
-
|
|
751
|
-
|
|
801
|
+
bind: (d, u) => {
|
|
802
|
+
if (u = u || he, !o) throw new Error("Pose tracker not initialized");
|
|
803
|
+
if (!n) throw new Error("Hands tracker not initialized");
|
|
804
|
+
if (!r) throw new Error("Face tracker not initialized");
|
|
805
|
+
const w = o.bind(d, u), h = n.left.bind(d, u), y = n.right.bind(d, u);
|
|
752
806
|
let b;
|
|
753
807
|
const E = r == null ? void 0 : r.bind(d);
|
|
754
|
-
d.traverse((
|
|
755
|
-
|
|
808
|
+
d.traverse((L) => {
|
|
809
|
+
L instanceof m.Mesh && L.name.indexOf(u.faceMesh) === 0 && (L.frustumCulled = !1, b = r == null ? void 0 : r.bindShapeKeys(L));
|
|
756
810
|
});
|
|
757
|
-
const
|
|
811
|
+
const k = st(d, u);
|
|
758
812
|
return {
|
|
759
813
|
/**
|
|
760
814
|
* Will save the tracked movement of the rig to an animation clip.
|
|
761
815
|
* Only the bones moved by the bone mapping will be recorded.
|
|
762
816
|
*/
|
|
763
|
-
startRecording:
|
|
764
|
-
stopRecording:
|
|
765
|
-
isRecording: () =>
|
|
766
|
-
update: (
|
|
767
|
-
|
|
817
|
+
startRecording: k.start,
|
|
818
|
+
stopRecording: k.stop,
|
|
819
|
+
isRecording: () => k.isRecording(),
|
|
820
|
+
update: (L) => {
|
|
821
|
+
w.update(L), h.update(L), y.update(L), b == null || b.update(L), E == null || E.update(L), k.isRecording() && k.captureFrame();
|
|
768
822
|
}
|
|
769
823
|
};
|
|
770
824
|
}
|
|
771
825
|
};
|
|
772
826
|
}
|
|
773
827
|
export {
|
|
774
|
-
|
|
828
|
+
ht as setupTracker
|
|
775
829
|
};
|
|
776
830
|
//# sourceMappingURL=three-mediapipe-rig.js.map
|