three-mediapipe-rig 0.0.4 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -5
- package/dist/three-mediapipe-rig.js +361 -298
- package/dist/three-mediapipe-rig.js.map +1 -1
- package/dist/tracking/FaceTracker.d.ts +21 -5
- package/dist/tracking/FaceTracker.d.ts.map +1 -1
- package/dist/tracking/HandTracker.d.ts +3 -1
- package/dist/tracking/HandTracker.d.ts.map +1 -1
- package/dist/tracking/PoseTracker.d.ts +1 -0
- package/dist/tracking/PoseTracker.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1,43 +1,44 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var
|
|
4
|
-
import { PoseLandmarker as
|
|
5
|
-
import * as
|
|
6
|
-
import { Vector3 as
|
|
7
|
-
import * as
|
|
8
|
-
import { Bone as
|
|
9
|
-
import {
|
|
10
|
-
|
|
11
|
-
new
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
1
|
+
var Re = Object.defineProperty;
|
|
2
|
+
var Pe = (c, s, e) => s in c ? Re(c, s, { enumerable: !0, configurable: !0, writable: !0, value: e }) : c[s] = e;
|
|
3
|
+
var P = (c, s, e) => Pe(c, typeof s != "symbol" ? s + "" : s, e);
|
|
4
|
+
import { PoseLandmarker as fe, HandLandmarker as U, FaceLandmarker as G, FilesetResolver as ve, DrawingUtils as ge } from "@mediapipe/tasks-vision";
|
|
5
|
+
import * as u from "three/webgpu";
|
|
6
|
+
import { Vector3 as b, Quaternion as we, BufferAttribute as Ee, VideoTexture as We, SRGBColorSpace as Fe, MeshPhysicalNodeMaterial as Ae } from "three/webgpu";
|
|
7
|
+
import * as T from "three";
|
|
8
|
+
import { Bone as Se } from "three";
|
|
9
|
+
import { uniform as _, attribute as Me, instancedArray as Ve, varying as Ne, vec3 as Oe, texture as Te, vec2 as _e } from "three/tsl";
|
|
10
|
+
import { GLTFExporter as Ce } from "three/examples/jsm/Addons.js";
|
|
11
|
+
const z = new b(), Y = new b(), De = new b(1, 0, 0), ze = new b(-1, 0, 0), Be = new b(0, 1, 0), Ie = new b(0, -1, 0), Z = new b(0, 0, 1);
|
|
12
|
+
new b(0, 0, -1);
|
|
13
|
+
const J = new b(), Ue = new b(), Ge = new b(), ee = new we(), B = new we();
|
|
14
|
+
function M(c, s, e, t = "+x") {
|
|
15
|
+
c.lookAt(s);
|
|
16
|
+
const o = t == "+x" ? De : t == "-x" ? ze : t == "+y" ? Be : Ie;
|
|
17
|
+
c.getWorldPosition(Y), c.getWorldQuaternion(B), z.subVectors(e, Y).normalize(), J.copy(o).applyQuaternion(B);
|
|
18
|
+
const n = J, a = Ue.copy(Z).applyQuaternion(B), i = z.clone().addScaledVector(a, -z.dot(a)).normalize(), r = Ge.crossVectors(n, i), l = Math.atan2(r.dot(a), n.dot(i));
|
|
19
|
+
ee.setFromAxisAngle(Z, l), c.quaternion.multiply(ee);
|
|
19
20
|
}
|
|
20
|
-
const
|
|
21
|
-
new
|
|
22
|
-
class
|
|
21
|
+
const C = new u.Vector3(), V = new u.Vector3(), I = new u.Vector3();
|
|
22
|
+
new u.Vector3();
|
|
23
|
+
class H {
|
|
23
24
|
constructor(s, e) {
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
P(this, "objectGhost");
|
|
26
|
+
P(this, "root");
|
|
26
27
|
/**
|
|
27
28
|
* per landmark index, it points to it's object3D equivalent.
|
|
28
29
|
*/
|
|
29
|
-
|
|
30
|
-
this.points = s, this.debugConnections = e, this.root = new
|
|
30
|
+
P(this, "marks", {});
|
|
31
|
+
this.points = s, this.debugConnections = e, this.root = new u.Object3D(), this.objectGhost = /* @__PURE__ */ new Map();
|
|
31
32
|
for (let t in this.points)
|
|
32
|
-
this.marks[t] = new
|
|
33
|
+
this.marks[t] = new He(), this.root.add(this.marks[t]);
|
|
33
34
|
}
|
|
34
35
|
updateLandmarks(s, e, t) {
|
|
35
36
|
for (let o in this.points) {
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
s[
|
|
39
|
-
s[
|
|
40
|
-
).divideScalar(2).add(s[
|
|
37
|
+
const n = this.points[o], a = this.marks[o];
|
|
38
|
+
a && (n instanceof Array ? (C.copy(s[n[0]]), a.position.copy(s[n[1]]).sub(C).divideScalar(2).add(s[n[0]]), n.length == 4 && (C.subVectors(
|
|
39
|
+
s[n[3]],
|
|
40
|
+
s[n[2]]
|
|
41
|
+
).divideScalar(2).add(s[n[2]]).sub(a.position).divideScalar(2), a.position.add(C))) : a.position.copy(s[n]));
|
|
41
42
|
}
|
|
42
43
|
t && e && t.drawConnectors(
|
|
43
44
|
e,
|
|
@@ -50,7 +51,7 @@ class _ {
|
|
|
50
51
|
getGhost(s) {
|
|
51
52
|
var e;
|
|
52
53
|
if (!this.objectGhost.has(s)) {
|
|
53
|
-
const t = new
|
|
54
|
+
const t = new $e();
|
|
54
55
|
t.position.copy(s.position), t.quaternion.copy(s.quaternion), (e = s.parent) == null || e.add(t), this.objectGhost.set(s, t);
|
|
55
56
|
}
|
|
56
57
|
return this.objectGhost.get(s);
|
|
@@ -65,47 +66,47 @@ class _ {
|
|
|
65
66
|
this.marks[s].position.set(1, 2, 3);
|
|
66
67
|
}
|
|
67
68
|
syncObjects(s, e, t) {
|
|
68
|
-
for (const [o,
|
|
69
|
-
this.marks[
|
|
70
|
-
const
|
|
71
|
-
o.getWorldPosition(
|
|
72
|
-
const
|
|
73
|
-
|
|
69
|
+
for (const [o, n, a, i] of s) {
|
|
70
|
+
this.marks[a].getWorldPosition(I), this.marks[n].getWorldPosition(V);
|
|
71
|
+
const r = I.sub(V);
|
|
72
|
+
o.getWorldPosition(V), V.add(r);
|
|
73
|
+
const l = V, h = o.getWorldPosition(I).sub(t), f = this.getGhost(o);
|
|
74
|
+
M(f, l, h, i), f.rotateX(Math.PI / 2), o.position.lerp(f.position, e * 4), o.quaternion.slerp(f.quaternion, e * 4);
|
|
74
75
|
}
|
|
75
76
|
}
|
|
76
77
|
getBone(s, e) {
|
|
77
78
|
return s.getObjectByName(e.replace(/[\.\:]/g, ""));
|
|
78
79
|
}
|
|
79
80
|
}
|
|
80
|
-
class
|
|
81
|
+
class He extends u.Mesh {
|
|
81
82
|
constructor() {
|
|
82
|
-
super(new
|
|
83
|
-
|
|
84
|
-
this.add(new
|
|
83
|
+
super(new u.SphereGeometry(0.01, 3, 3), new u.MeshStandardMaterial({ color: 16711680, wireframe: !0 }));
|
|
84
|
+
P(this, "_worldPosition", new u.Vector3());
|
|
85
|
+
this.add(new u.AxesHelper(1e-3));
|
|
85
86
|
}
|
|
86
87
|
get worldPosition() {
|
|
87
88
|
return this.getWorldPosition(this._worldPosition), this._worldPosition;
|
|
88
89
|
}
|
|
89
90
|
}
|
|
90
|
-
class
|
|
91
|
+
class $e extends u.Object3D {
|
|
91
92
|
lerp(s, e, t = 8) {
|
|
92
93
|
s.position.lerp(this.position, e * t), s.quaternion.slerp(this.quaternion, e * t);
|
|
93
94
|
}
|
|
94
95
|
}
|
|
95
|
-
function
|
|
96
|
-
return e.worldToLocal(s.getWorldPosition(
|
|
96
|
+
function S(c, s, e) {
|
|
97
|
+
return e.worldToLocal(s.getWorldPosition(c)), c;
|
|
97
98
|
}
|
|
98
|
-
function
|
|
99
|
-
return
|
|
99
|
+
function qe(c) {
|
|
100
|
+
return c.replace(/[\.\:]/g, "");
|
|
100
101
|
}
|
|
101
|
-
function
|
|
102
|
+
function g(c, s) {
|
|
102
103
|
let e;
|
|
103
|
-
return s =
|
|
104
|
-
t.name.indexOf(s) === 0 && t instanceof
|
|
105
|
-
}), e || console.log("Bone not found: ", s,
|
|
104
|
+
return s = qe(s), c.traverse((t) => {
|
|
105
|
+
t.name.indexOf(s) === 0 && t instanceof Se && (e = t);
|
|
106
|
+
}), e || console.log("Bone not found: ", s, c.name), e;
|
|
106
107
|
}
|
|
107
|
-
async function
|
|
108
|
-
const e = await
|
|
108
|
+
async function je(c, s) {
|
|
109
|
+
const e = await fe.createFromOptions(c, {
|
|
109
110
|
baseOptions: {
|
|
110
111
|
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "pose_landmarker_lite.task",
|
|
111
112
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task`,
|
|
@@ -115,9 +116,9 @@ async function We(h, s) {
|
|
|
115
116
|
runningMode: "VIDEO",
|
|
116
117
|
numPoses: 1
|
|
117
118
|
});
|
|
118
|
-
return new
|
|
119
|
+
return new Je(e, s);
|
|
119
120
|
}
|
|
120
|
-
const
|
|
121
|
+
const Ke = {
|
|
121
122
|
hips: [24, 23],
|
|
122
123
|
neck: [12, 11],
|
|
123
124
|
leftLeg: 23,
|
|
@@ -139,12 +140,12 @@ const Ee = {
|
|
|
139
140
|
torso: [24, 23, 12, 11],
|
|
140
141
|
leftEar: 7,
|
|
141
142
|
rightEar: 8
|
|
142
|
-
},
|
|
143
|
-
class
|
|
143
|
+
}, Xe = new u.Vector3(), te = new u.Vector3(), Qe = new u.Vector3(), Ye = new u.Vector3(), Ze = new u.Vector3(), oe = new u.Vector3(), se = new u.Vector3();
|
|
144
|
+
class Je extends H {
|
|
144
145
|
constructor(e, t) {
|
|
145
|
-
super(
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
super(Ke, fe.POSE_CONNECTIONS);
|
|
147
|
+
P(this, "_leftWristNormalizedPosition");
|
|
148
|
+
P(this, "_rightWristNormalizedPosition");
|
|
148
149
|
this.poseLandmarker = e, this.config = t, this.root.scale.y *= -2, this.root.scale.z *= -2, this.root.scale.x *= 2;
|
|
149
150
|
}
|
|
150
151
|
/**
|
|
@@ -161,7 +162,8 @@ class ge extends _ {
|
|
|
161
162
|
}
|
|
162
163
|
predict(e, t) {
|
|
163
164
|
this.poseLandmarker.detectForVideo(e, performance.now(), (o) => {
|
|
164
|
-
|
|
165
|
+
var n;
|
|
166
|
+
o.landmarks.length != 0 && (this.updateLandmarks(o.worldLandmarks[0], ((n = this.config) == null ? void 0 : n.drawLandmarks) === !1 ? void 0 : o.landmarks[0], t), this._leftWristNormalizedPosition = o.landmarks[0][this.points.leftWrist], this._rightWristNormalizedPosition = o.landmarks[0][this.points.rightWrist]);
|
|
165
167
|
});
|
|
166
168
|
}
|
|
167
169
|
// override sync ( delta:number, objects: BoneBinding[] ) {
|
|
@@ -172,44 +174,44 @@ class ge extends _ {
|
|
|
172
174
|
// this.syncObjects(objects, delta, torsoNormal);
|
|
173
175
|
// }
|
|
174
176
|
bind(e, t) {
|
|
175
|
-
var
|
|
177
|
+
var r;
|
|
176
178
|
const o = {
|
|
177
|
-
hips:
|
|
178
|
-
neck:
|
|
179
|
-
leftArm:
|
|
180
|
-
leftElbow:
|
|
181
|
-
leftWrist:
|
|
182
|
-
rightArm:
|
|
183
|
-
rightElbow:
|
|
184
|
-
rightWrist:
|
|
185
|
-
head:
|
|
186
|
-
torso:
|
|
187
|
-
leftLeg:
|
|
188
|
-
leftKnee:
|
|
189
|
-
leftFoot:
|
|
190
|
-
rightLeg:
|
|
191
|
-
rightKnee:
|
|
192
|
-
rightFoot:
|
|
179
|
+
hips: g(e, t.hips),
|
|
180
|
+
neck: g(e, t.neck),
|
|
181
|
+
leftArm: g(e, t.armL),
|
|
182
|
+
leftElbow: g(e, t.forearmL),
|
|
183
|
+
leftWrist: g(e, t.handL),
|
|
184
|
+
rightArm: g(e, t.armR),
|
|
185
|
+
rightElbow: g(e, t.forearmR),
|
|
186
|
+
rightWrist: g(e, t.handR),
|
|
187
|
+
head: g(e, t.head),
|
|
188
|
+
torso: g(e, t.torso),
|
|
189
|
+
leftLeg: g(e, t.thighL),
|
|
190
|
+
leftKnee: g(e, t.shinL),
|
|
191
|
+
leftFoot: g(e, t.footL),
|
|
192
|
+
rightLeg: g(e, t.thighR),
|
|
193
|
+
rightKnee: g(e, t.shinR),
|
|
194
|
+
rightFoot: g(e, t.footR)
|
|
193
195
|
};
|
|
194
|
-
(
|
|
195
|
-
const
|
|
196
|
-
if (!
|
|
197
|
-
const
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
|
|
196
|
+
(r = this.config) != null && r.ignoreLegs && (delete o.leftLeg, delete o.leftKnee, delete o.leftFoot, delete o.leftToes, delete o.rightLeg, delete o.rightKnee, delete o.rightFoot, delete o.rightToes);
|
|
197
|
+
const n = new u.Vector3(), a = new u.Vector3(), i = (l, h, f, x, d, m) => {
|
|
198
|
+
if (!h) return;
|
|
199
|
+
const y = this.marks[x].getWorldPosition(n).sub(this.marks[f].getWorldPosition(a)).normalize();
|
|
200
|
+
S(oe, h, e).add(y).applyMatrix4(e.matrixWorld), S(se, h, e).add(d).applyMatrix4(e.matrixWorld);
|
|
201
|
+
const p = this.getGhost(h);
|
|
202
|
+
M(p, oe, se, m), p.rotateX(Math.PI / 2), p.lerp(h, l);
|
|
201
203
|
};
|
|
202
204
|
return {
|
|
203
|
-
update: (
|
|
204
|
-
const
|
|
205
|
-
|
|
205
|
+
update: (l) => {
|
|
206
|
+
const h = this.marks.leftLeg.getWorldPosition(Xe).sub(this.marks.rightLeg.getWorldPosition(te)).normalize(), f = this.marks.leftArm.getWorldPosition(te).sub(this.marks.rightArm.getWorldPosition(Qe)).normalize(), x = this.marks.leftEar.getWorldPosition(Ye).sub(this.marks.rightEar.getWorldPosition(Ze)).normalize();
|
|
207
|
+
i(l, o.hips, "hips", "torso", h, "+x"), i(l, o.torso, "torso", "neck", f, "+x"), i(l, o.neck, "neck", "head", x, "+x"), i(l, o.head, "neck", "head", x, "+x"), i(l, o.leftArm, "leftArm", "leftElbow", f, "-x"), i(l, o.leftElbow, "leftElbow", "leftWrist", f, "-x"), i(l, o.leftLeg, "leftLeg", "leftKnee", h, "+x"), i(l, o.leftKnee, "leftKnee", "leftFoot", h, "+x"), i(l, o.leftFoot, "leftFoot", "leftToes", h, "+x"), i(l, o.rightArm, "rightArm", "rightElbow", f, "-x"), i(l, o.rightElbow, "rightElbow", "rightWrist", f, "-x"), i(l, o.rightLeg, "rightLeg", "rightKnee", h, "+x"), i(l, o.rightKnee, "rightKnee", "rightFoot", h, "+x"), i(l, o.rightFoot, "rightFoot", "rightToes", h, "+x");
|
|
206
208
|
}
|
|
207
209
|
};
|
|
208
210
|
}
|
|
209
211
|
}
|
|
210
|
-
const
|
|
211
|
-
async function
|
|
212
|
-
const e = await
|
|
212
|
+
const ne = new u.Vector2(), ie = new u.Vector2();
|
|
213
|
+
async function et(c, s) {
|
|
214
|
+
const e = await U.createFromOptions(c, {
|
|
213
215
|
baseOptions: {
|
|
214
216
|
modelAssetPath: s.modelPath ?? "hand_landmarker.task",
|
|
215
217
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
|
|
@@ -217,20 +219,20 @@ async function Se(h, s) {
|
|
|
217
219
|
},
|
|
218
220
|
runningMode: "VIDEO",
|
|
219
221
|
numHands: 2
|
|
220
|
-
}), t = (o,
|
|
222
|
+
}), t = (o, n, a) => {
|
|
221
223
|
try {
|
|
222
|
-
|
|
223
|
-
} catch (
|
|
224
|
-
return console.warn("No pose data... will just be optimitic and say yes to everything.",
|
|
224
|
+
ne.copy(o()), ie.copy(n());
|
|
225
|
+
} catch (i) {
|
|
226
|
+
return console.warn("No pose data... will just be optimitic and say yes to everything.", i), !0;
|
|
225
227
|
}
|
|
226
|
-
return
|
|
228
|
+
return ne.distanceTo(a) < ie.distanceTo(a);
|
|
227
229
|
};
|
|
228
230
|
return {
|
|
229
|
-
left: new
|
|
230
|
-
right: new
|
|
231
|
+
left: new me(e, "Left", t.bind(null, s.leftWrist, s.rightWrist), s.drawLandmarks),
|
|
232
|
+
right: new me(e, "Right", t.bind(null, s.rightWrist, s.leftWrist), s.drawLandmarks)
|
|
231
233
|
};
|
|
232
234
|
}
|
|
233
|
-
const
|
|
235
|
+
const tt = {
|
|
234
236
|
wrist: 0,
|
|
235
237
|
palm: [9, 13],
|
|
236
238
|
thumb1: 1,
|
|
@@ -253,38 +255,38 @@ const Ne = {
|
|
|
253
255
|
pinky2: 18,
|
|
254
256
|
pinky3: 19,
|
|
255
257
|
pinky4: 20
|
|
256
|
-
},
|
|
258
|
+
}, N = {
|
|
257
259
|
thumb: ["thumb1", "thumb2", "thumb3", "thumb4"],
|
|
258
260
|
index: ["index1", "index2", "index3", "index4"],
|
|
259
261
|
middle: ["middle1", "middle2", "middle3", "middle4"],
|
|
260
262
|
ring: ["ring1", "ring2", "ring3", "ring4"],
|
|
261
263
|
pinky: ["pinky1", "pinky2", "pinky3", "pinky4"]
|
|
262
|
-
},
|
|
263
|
-
new
|
|
264
|
-
new
|
|
265
|
-
new
|
|
266
|
-
const
|
|
267
|
-
class
|
|
268
|
-
constructor(e, t, o) {
|
|
269
|
-
super(
|
|
270
|
-
|
|
271
|
-
|
|
264
|
+
}, ot = new u.Vector3(), re = new u.Vector3(), ae = new u.Vector3(), de = new u.Vector3(), ce = new u.Vector3(), le = new u.Vector3();
|
|
265
|
+
new u.Vector3();
|
|
266
|
+
new u.Vector3();
|
|
267
|
+
new u.Vector3();
|
|
268
|
+
const he = new u.Vector3(), pe = Math.PI / 2, st = new u.Vector3(0, -1, 0);
|
|
269
|
+
class me extends H {
|
|
270
|
+
constructor(e, t, o, n = !0) {
|
|
271
|
+
super(tt, U.HAND_CONNECTIONS);
|
|
272
|
+
P(this, "sign");
|
|
273
|
+
P(this, "isLeft");
|
|
272
274
|
/**
|
|
273
275
|
* the axis used to look at the pole
|
|
274
276
|
*/
|
|
275
|
-
|
|
276
|
-
this.handLandmarker = e, this.side = t, this.isMyWrist = o, this.sign = this.side == "Left" ? -1 : 1, this.isLeft = this.side == "Left", this.lookAtPoleAxis = this.sign < 0 ? "+x" : "-x", this.root.scale.setScalar(7), this.root.scale.y *= -1, this.root.scale.z *= -1;
|
|
277
|
+
P(this, "lookAtPoleAxis");
|
|
278
|
+
this.handLandmarker = e, this.side = t, this.isMyWrist = o, this.drawLandmarks = n, this.sign = this.side == "Left" ? -1 : 1, this.isLeft = this.side == "Left", this.lookAtPoleAxis = this.sign < 0 ? "+x" : "-x", this.root.scale.setScalar(7), this.root.scale.y *= -1, this.root.scale.z *= -1;
|
|
277
279
|
}
|
|
278
280
|
predict(e, t) {
|
|
279
281
|
const o = this.handLandmarker.detectForVideo(e, performance.now());
|
|
280
282
|
if (o.landmarks.length)
|
|
281
|
-
for (let
|
|
282
|
-
const
|
|
283
|
-
if (this.isMyWrist(
|
|
284
|
-
this.updateLandmarks(o.worldLandmarks[
|
|
283
|
+
for (let n = 0; n < o.landmarks.length; n++) {
|
|
284
|
+
const a = o.landmarks[n], i = a[this.points.wrist];
|
|
285
|
+
if (this.isMyWrist(i)) {
|
|
286
|
+
this.updateLandmarks(o.worldLandmarks[n]), this.drawLandmarks && (t.drawConnectors(a, U.HAND_CONNECTIONS, {
|
|
285
287
|
color: this.side == "Left" ? "#00FF00" : "#0000FF",
|
|
286
288
|
lineWidth: 4
|
|
287
|
-
}), t.drawLandmarks(
|
|
289
|
+
}), t.drawLandmarks(a, { color: this.side == "Left" ? "#00FF00" : "#0000FF", lineWidth: 3, radius: 1 }));
|
|
288
290
|
break;
|
|
289
291
|
}
|
|
290
292
|
}
|
|
@@ -299,56 +301,56 @@ class te extends _ {
|
|
|
299
301
|
* @see https://ai.google.dev/edge/mediapipe/solutions/vision/hand_landmarker
|
|
300
302
|
*/
|
|
301
303
|
syncHandBones(e, t, o) {
|
|
302
|
-
const
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
).normalize(),
|
|
306
|
-
if (!(
|
|
304
|
+
const n = ot.crossVectors(
|
|
305
|
+
re.copy(this.marks.index1.worldPosition).sub(this.marks.wrist.worldPosition),
|
|
306
|
+
ae.copy(this.marks.pinky1.worldPosition).sub(this.marks.wrist.worldPosition)
|
|
307
|
+
).normalize(), a = re.copy(this.marks.palm.worldPosition).sub(this.marks.wrist.worldPosition).normalize(), i = ae.copy(this.marks.pinky1.worldPosition).sub(this.marks.index1.worldPosition).normalize();
|
|
308
|
+
if (!(a.dot(st) > 0.8)) {
|
|
307
309
|
if (t.wrist) {
|
|
308
|
-
const
|
|
309
|
-
|
|
310
|
+
const r = S(de, t.wrist, o).add(a).applyMatrix4(o.matrixWorld), l = S(ce, t.wrist, o).sub(i).applyMatrix4(o.matrixWorld), h = this.getGhost(t.wrist);
|
|
311
|
+
M(h, r, l, "-y"), h.rotateX(pe), h.lerp(t.wrist, e);
|
|
310
312
|
}
|
|
311
|
-
this.syncFinger(e, o,
|
|
313
|
+
this.syncFinger(e, o, n, a, i, t, N.index, "middle1"), this.syncFinger(e, o, n, a, i, t, N.middle, "ring1"), this.syncFinger(e, o, n, a, i, t, N.ring, "pinky1"), this.syncFinger(e, o, n, a, i, t, N.pinky, "ring1", !0), this.syncFinger(e, o, n, a, i, t, N.thumb, "index1");
|
|
312
314
|
}
|
|
313
315
|
}
|
|
314
|
-
syncFinger(e, t, o,
|
|
315
|
-
for (let f = 0; f <
|
|
316
|
-
const
|
|
317
|
-
if (!
|
|
318
|
-
const d = this.getGhost(
|
|
316
|
+
syncFinger(e, t, o, n, a, i, r, l, h = !1) {
|
|
317
|
+
for (let f = 0; f < r.length - 1; f++) {
|
|
318
|
+
const x = i[r[f]];
|
|
319
|
+
if (!x) continue;
|
|
320
|
+
const d = this.getGhost(x), m = de.copy(this.marks[r[f + 1]].worldPosition).sub(this.marks[r[f]].worldPosition).normalize(), y = S(ce, x, t);
|
|
319
321
|
if (f == 0) {
|
|
320
|
-
const
|
|
321
|
-
|
|
322
|
+
const p = le.copy(this.marks[l].worldPosition).sub(this.marks[r[0]].worldPosition).normalize();
|
|
323
|
+
h && p.negate(), he.copy(p), M(
|
|
322
324
|
d,
|
|
323
|
-
m.add(
|
|
324
|
-
|
|
325
|
+
m.add(y).applyMatrix4(t.matrixWorld),
|
|
326
|
+
p.add(y).applyMatrix4(t.matrixWorld),
|
|
325
327
|
this.lookAtPoleAxis
|
|
326
328
|
);
|
|
327
329
|
} else
|
|
328
|
-
|
|
330
|
+
M(
|
|
329
331
|
d,
|
|
330
|
-
m.add(
|
|
331
|
-
|
|
332
|
+
m.add(y).applyMatrix4(t.matrixWorld),
|
|
333
|
+
le.copy(he).add(y).applyMatrix4(t.matrixWorld),
|
|
332
334
|
this.lookAtPoleAxis
|
|
333
335
|
);
|
|
334
|
-
d.rotateX(
|
|
336
|
+
d.rotateX(pe), d.lerp(x, e);
|
|
335
337
|
}
|
|
336
338
|
}
|
|
337
339
|
bind(e, t) {
|
|
338
|
-
const o = {},
|
|
339
|
-
const
|
|
340
|
-
if (
|
|
341
|
-
return o[
|
|
340
|
+
const o = {}, n = (a, i) => {
|
|
341
|
+
const r = g(e, a);
|
|
342
|
+
if (r)
|
|
343
|
+
return o[i] = r, i;
|
|
342
344
|
};
|
|
343
|
-
return
|
|
344
|
-
update: (
|
|
345
|
-
this.syncHandBones(
|
|
345
|
+
return n(this.isLeft ? t.handL : t.handR, "wrist"), n(this.isLeft ? t.index1L : t.index1R, "index1"), n(this.isLeft ? t.index2L : t.index2R, "index2"), n(this.isLeft ? t.index3L : t.index3R, "index3"), n(this.isLeft ? t.middle1L : t.middle1R, "middle1"), n(this.isLeft ? t.middle2L : t.middle2R, "middle2"), n(this.isLeft ? t.middle3L : t.middle3R, "middle3"), n(this.isLeft ? t.ring1L : t.ring1R, "ring1"), n(this.isLeft ? t.ring2L : t.ring2R, "ring2"), n(this.isLeft ? t.ring3L : t.ring3R, "ring3"), n(this.isLeft ? t.pinky1L : t.pinky1R, "pinky1"), n(this.isLeft ? t.pinky2L : t.pinky2R, "pinky2"), n(this.isLeft ? t.pinky3L : t.pinky3R, "pinky3"), n(this.isLeft ? t.thumb1L : t.thumb1R, "thumb1"), n(this.isLeft ? t.thumb2L : t.thumb2R, "thumb2"), n(this.isLeft ? t.thumb3L : t.thumb3R, "thumb3"), {
|
|
346
|
+
update: (a) => {
|
|
347
|
+
this.syncHandBones(a, o, e);
|
|
346
348
|
}
|
|
347
349
|
};
|
|
348
350
|
}
|
|
349
351
|
}
|
|
350
|
-
async function
|
|
351
|
-
const e = await
|
|
352
|
+
async function nt(c, s) {
|
|
353
|
+
const e = await G.createFromOptions(c, {
|
|
352
354
|
baseOptions: {
|
|
353
355
|
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "face_landmarker.task",
|
|
354
356
|
delegate: "GPU"
|
|
@@ -357,9 +359,9 @@ async function Te(h, s) {
|
|
|
357
359
|
runningMode: "VIDEO",
|
|
358
360
|
numFaces: 1
|
|
359
361
|
});
|
|
360
|
-
return new
|
|
362
|
+
return new lt(e, { ...s });
|
|
361
363
|
}
|
|
362
|
-
const
|
|
364
|
+
const it = {
|
|
363
365
|
eyeL: 473,
|
|
364
366
|
eyeR: 468,
|
|
365
367
|
eyeStartL: 463,
|
|
@@ -372,71 +374,125 @@ const De = {
|
|
|
372
374
|
noseBone: 6,
|
|
373
375
|
chin: 152,
|
|
374
376
|
forehead: 10
|
|
375
|
-
},
|
|
376
|
-
new
|
|
377
|
-
class
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
this
|
|
377
|
+
}, rt = new b(), at = new b(), ye = new b(), dt = new b(), ct = new b();
|
|
378
|
+
new b();
|
|
379
|
+
class lt extends H {
|
|
380
|
+
constructor(e, t) {
|
|
381
|
+
super(it, G.FACE_LANDMARKS_TESSELATION);
|
|
382
|
+
P(this, "blendshapeCategories");
|
|
383
|
+
P(this, "blendshapeMap", /* @__PURE__ */ new Map());
|
|
384
|
+
P(this, "smoothed", {});
|
|
385
|
+
P(this, "smoothing", 3e-4);
|
|
386
|
+
// lower = smoother but more lag, higher = more responsive
|
|
387
|
+
P(this, "_faceLandmarks", []);
|
|
388
|
+
this.faceLandmarker = e, this.cfg = t, this.root.scale.y *= -1, this.root.scale.z *= -1, this.root.scale.multiplyScalar(3);
|
|
386
389
|
}
|
|
387
390
|
predict(e, t) {
|
|
388
|
-
var
|
|
391
|
+
var n, a, i;
|
|
389
392
|
const o = this.faceLandmarker.detectForVideo(e, performance.now());
|
|
390
|
-
o.faceLandmarks[0] && (t.drawConnectors(o.faceLandmarks[0],
|
|
391
|
-
this.blendshapeMap.set(
|
|
393
|
+
o.faceLandmarks[0] && (this.cfg.drawLandmarks && (t.drawConnectors(o.faceLandmarks[0], G.FACE_LANDMARKS_TESSELATION, { color: "#00fff2ff", lineWidth: 0.1 }), t.drawLandmarks(o.faceLandmarks[0], { color: "#00ff00", lineWidth: 0.1, radius: 0.4 })), this.updateLandmarks(o.faceLandmarks[0], o.faceLandmarks[0])), this._faceLandmarks = o.faceLandmarks[0], this.blendshapeCategories = (a = (n = o.faceBlendshapes) == null ? void 0 : n[0]) == null ? void 0 : a.categories, (i = this.blendshapeCategories) == null || i.forEach((r) => {
|
|
394
|
+
this.blendshapeMap.set(r.categoryName, r.score);
|
|
392
395
|
});
|
|
393
396
|
}
|
|
397
|
+
get lastKnownLandmarks() {
|
|
398
|
+
return this._faceLandmarks;
|
|
399
|
+
}
|
|
394
400
|
bindShapeKeys(e) {
|
|
395
401
|
const t = e.morphTargetDictionary;
|
|
396
402
|
return {
|
|
397
403
|
update: (o) => {
|
|
398
|
-
var
|
|
399
|
-
(
|
|
400
|
-
const { categoryName:
|
|
401
|
-
if (!(t != null && t.hasOwnProperty(
|
|
402
|
-
this.smoothed[
|
|
403
|
-
const
|
|
404
|
-
this.smoothed[
|
|
404
|
+
var n;
|
|
405
|
+
(n = this.blendshapeCategories) == null || n.forEach((a) => {
|
|
406
|
+
const { categoryName: i, score: r } = a;
|
|
407
|
+
if (!(t != null && t.hasOwnProperty(i))) return;
|
|
408
|
+
this.smoothed[i] === void 0 && (this.smoothed[i] = r);
|
|
409
|
+
const l = 1 - Math.pow(this.smoothing, o);
|
|
410
|
+
this.smoothed[i] += (r - this.smoothed[i]) * l, e.morphTargetInfluences[t[i]] = this.smoothed[i];
|
|
405
411
|
});
|
|
406
412
|
}
|
|
407
413
|
};
|
|
408
414
|
}
|
|
409
415
|
bind(e) {
|
|
410
|
-
const t = new
|
|
416
|
+
const t = new ue(e, "L"), o = new ue(e, "R"), n = g(e, "head");
|
|
417
|
+
return {
|
|
418
|
+
update: (a) => {
|
|
419
|
+
if (t.update(a, this.blendshapeMap), o.update(a, this.blendshapeMap), !n) return;
|
|
420
|
+
const i = rt.copy(this.marks.earL.worldPosition), r = at.copy(this.marks.earR.worldPosition), l = ye.subVectors(i, r).multiplyScalar(0.5).add(r), h = dt.subVectors(this.marks.noseTip.worldPosition, l), f = i.sub(r), x = S(ct, n, e), d = f.add(x).applyMatrix4(e.matrixWorld), m = h.add(x).applyMatrix4(e.matrixWorld);
|
|
421
|
+
M(n, m, d, "+x");
|
|
422
|
+
}
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* The mesh is assumed to be a canonical_face_model because we will be manipulating it's vertices.
|
|
427
|
+
*
|
|
428
|
+
* @see https://github.com/google-ai-edge/mediapipe/tree/master/mediapipe/modules/face_geometry
|
|
429
|
+
* @param mesh basically either the original or a clone of the canonical_face_model
|
|
430
|
+
*/
|
|
431
|
+
bindGeometry(e, t) {
|
|
432
|
+
const o = new b(), n = new b(), a = new b();
|
|
433
|
+
new b();
|
|
434
|
+
const i = e.geometry, r = i.attributes.position, l = [], h = /* @__PURE__ */ new Map(), f = [];
|
|
435
|
+
for (let F = 0; F < r.count; F++) {
|
|
436
|
+
const v = `${r.getX(F).toFixed(6)},${r.getY(F).toFixed(6)},${r.getZ(F).toFixed(6)}`;
|
|
437
|
+
h.has(v) || (h.set(v, l.length), l.push(F)), f.push(h.get(v));
|
|
438
|
+
}
|
|
439
|
+
const x = _(new b(0.5, 0.5, 0.5));
|
|
440
|
+
i.setAttribute("landmarkIndex", new Ee(new Uint16Array(f), 1));
|
|
441
|
+
const d = Me("landmarkIndex", "uint"), m = Ve(468, "vec3"), y = Ne(m.element(d)).xy;
|
|
442
|
+
let p;
|
|
443
|
+
const R = 116, k = 346, E = a.subVectors(
|
|
444
|
+
new b(r.getX(R), r.getY(R), r.getZ(R)),
|
|
445
|
+
new b(r.getX(k), r.getY(k), r.getZ(k))
|
|
446
|
+
).lengthSq(), L = _(E), w = _(1), A = _(1);
|
|
411
447
|
return {
|
|
412
|
-
update: (
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
448
|
+
update: (F) => {
|
|
449
|
+
var O, j;
|
|
450
|
+
if (!p) {
|
|
451
|
+
const W = (j = (O = this.cfg) == null ? void 0 : O.videoElementRef) == null ? void 0 : j.call(O);
|
|
452
|
+
if (!W || !W.videoWidth || !W.videoHeight) return;
|
|
453
|
+
p = W;
|
|
454
|
+
const K = new We(p);
|
|
455
|
+
K.colorSpace = Fe;
|
|
456
|
+
const xe = p.videoWidth / p.videoHeight, X = m.element(d).sub(x).xzy.mul(Oe(1, -1, 1 / xe)).mul(A), Q = Te(K, _e(y.x, y.y.oneMinus()));
|
|
457
|
+
t ? t(X, Q) : e.material = new Ae({
|
|
458
|
+
positionNode: X,
|
|
459
|
+
colorNode: Q,
|
|
460
|
+
roughness: 0.93
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
const v = this.lastKnownLandmarks;
|
|
464
|
+
if (!(v != null && v.length)) return;
|
|
465
|
+
w.value = a.subVectors(v[k], v[R]).lengthSq(), w.needsUpdate = !0;
|
|
466
|
+
const D = m.value.array;
|
|
467
|
+
for (let W = 0; W < v.length; W++)
|
|
468
|
+
D[W * 3] = v[W].x, D[W * 3 + 1] = v[W].y, D[W * 3 + 2] = v[W].z;
|
|
469
|
+
m.value.needsUpdate = !0;
|
|
470
|
+
const ke = v[234], $ = v[93], be = v[454], q = v[323];
|
|
471
|
+
o.copy(ke).sub($).divideScalar(2).add($), n.copy(be).sub(q).divideScalar(2).add(q), x.value.subVectors(n, o).divideScalar(2).add(o), x.needsUpdate = !0, A.value = Math.sqrt(L.value / w.value), A.needsUpdate = !0;
|
|
416
472
|
}
|
|
417
473
|
};
|
|
418
474
|
}
|
|
419
475
|
}
|
|
420
|
-
class
|
|
476
|
+
class ue {
|
|
421
477
|
constructor(s, e) {
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
478
|
+
P(this, "eyeBone");
|
|
479
|
+
P(this, "eyeLookOut");
|
|
480
|
+
P(this, "eyeLookIn");
|
|
481
|
+
P(this, "eyeLookUp");
|
|
482
|
+
P(this, "eyeLookDown");
|
|
483
|
+
P(this, "sign", 1);
|
|
428
484
|
this.rig = s, this.side = e, this.eyeBone = s.getObjectByName(`eye${e}`);
|
|
429
485
|
const t = e == "L" ? "Left" : "Right";
|
|
430
486
|
this.eyeLookOut = `eyeLookOut${t}`, this.eyeLookIn = `eyeLookIn${t}`, this.eyeLookUp = `eyeLookUp${t}`, this.eyeLookDown = `eyeLookDown${t}`, this.sign = e == "L" ? -1 : 1;
|
|
431
487
|
}
|
|
432
488
|
update(s, e) {
|
|
433
489
|
if (!this.eyeBone) return;
|
|
434
|
-
|
|
435
|
-
const t = e.get(this.eyeLookOut) ?? 0, o = e.get(this.eyeLookIn) ?? 0,
|
|
436
|
-
this.eyeBone.rotation.y =
|
|
490
|
+
S(ye, this.eyeBone, this.rig);
|
|
491
|
+
const t = e.get(this.eyeLookOut) ?? 0, o = e.get(this.eyeLookIn) ?? 0, n = e.get(this.eyeLookUp) ?? 0, a = e.get(this.eyeLookDown) ?? 0, i = o - t, r = a - n;
|
|
492
|
+
this.eyeBone.rotation.y = i * this.sign / 2, this.eyeBone.rotation.x = r / 2;
|
|
437
493
|
}
|
|
438
494
|
}
|
|
439
|
-
const
|
|
495
|
+
const Le = {
|
|
440
496
|
faceMesh: "face",
|
|
441
497
|
head: "head",
|
|
442
498
|
hips: "hips",
|
|
@@ -484,7 +540,7 @@ const re = {
|
|
|
484
540
|
thumb1R: "thumb1R",
|
|
485
541
|
thumb2R: "thumb2R",
|
|
486
542
|
thumb3R: "thumb3R"
|
|
487
|
-
},
|
|
543
|
+
}, ht = [
|
|
488
544
|
"eyeBlinkLeft",
|
|
489
545
|
"eyeBlinkRight",
|
|
490
546
|
"eyeLookDownLeft",
|
|
@@ -538,206 +594,211 @@ const re = {
|
|
|
538
594
|
"mouthUpperUpRight",
|
|
539
595
|
"tongueOut"
|
|
540
596
|
];
|
|
541
|
-
function
|
|
542
|
-
const t = [], o =
|
|
597
|
+
function pt(c, s, e = 30) {
|
|
598
|
+
const t = [], o = c.getObjectByName(
|
|
543
599
|
s.faceMesh
|
|
544
|
-
),
|
|
600
|
+
), n = /* @__PURE__ */ new Set();
|
|
545
601
|
if (o != null && o.morphTargetDictionary)
|
|
546
|
-
for (const
|
|
547
|
-
|
|
548
|
-
const
|
|
549
|
-
|
|
550
|
-
if (
|
|
551
|
-
const
|
|
552
|
-
|
|
553
|
-
ref:
|
|
554
|
-
name:
|
|
555
|
-
normalizedName:
|
|
602
|
+
for (const y in o.morphTargetDictionary)
|
|
603
|
+
ht.includes(y) && n.add(y);
|
|
604
|
+
const a = Object.keys(s);
|
|
605
|
+
c.traverse((y) => {
|
|
606
|
+
if (y instanceof T.Bone) {
|
|
607
|
+
const p = a.find((R) => y.name.indexOf(s[R]) === 0);
|
|
608
|
+
p && t.push({
|
|
609
|
+
ref: y,
|
|
610
|
+
name: y.name,
|
|
611
|
+
normalizedName: p
|
|
556
612
|
});
|
|
557
613
|
}
|
|
558
614
|
});
|
|
559
|
-
const
|
|
560
|
-
let
|
|
561
|
-
function
|
|
562
|
-
|
|
615
|
+
const i = [], r = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map();
|
|
616
|
+
let h = !1, f = 0;
|
|
617
|
+
function x() {
|
|
618
|
+
r.clear(), l.clear(), i.length = 0, f = performance.now() / 1e3, h = !0;
|
|
563
619
|
}
|
|
564
620
|
function d() {
|
|
565
|
-
if (!
|
|
566
|
-
const
|
|
567
|
-
|
|
568
|
-
for (const
|
|
569
|
-
|
|
570
|
-
const
|
|
571
|
-
|
|
621
|
+
if (!h) return;
|
|
622
|
+
const y = performance.now() / 1e3 - f;
|
|
623
|
+
i.push(y);
|
|
624
|
+
for (const p of t) {
|
|
625
|
+
r.has(p.name) || r.set(p.name, []);
|
|
626
|
+
const R = r.get(p.name), k = p.ref.quaternion;
|
|
627
|
+
R.push(k.x, k.y, k.z, k.w);
|
|
572
628
|
}
|
|
573
|
-
for (const
|
|
574
|
-
|
|
575
|
-
const
|
|
576
|
-
|
|
629
|
+
for (const p of n) {
|
|
630
|
+
l.has(p) || l.set(p, []);
|
|
631
|
+
const R = l.get(p), k = o.morphTargetDictionary[p], E = o.morphTargetInfluences[k];
|
|
632
|
+
R.push(E);
|
|
577
633
|
}
|
|
578
634
|
}
|
|
579
|
-
function m(
|
|
580
|
-
|
|
581
|
-
const
|
|
582
|
-
for (const [
|
|
583
|
-
|
|
584
|
-
new
|
|
585
|
-
`${
|
|
586
|
-
|
|
635
|
+
function m(y = "RecordedClip") {
|
|
636
|
+
h = !1;
|
|
637
|
+
const p = [];
|
|
638
|
+
for (const [k, E] of r)
|
|
639
|
+
p.push(
|
|
640
|
+
new T.QuaternionKeyframeTrack(
|
|
641
|
+
`${k}.quaternion`,
|
|
642
|
+
i,
|
|
587
643
|
E
|
|
588
644
|
)
|
|
589
645
|
);
|
|
590
|
-
for (const [
|
|
591
|
-
|
|
592
|
-
new
|
|
593
|
-
`${
|
|
594
|
-
|
|
646
|
+
for (const [k, E] of l)
|
|
647
|
+
p.push(
|
|
648
|
+
new T.NumberKeyframeTrack(
|
|
649
|
+
`${Le.faceMesh}.morphTargetInfluences[${k}]`,
|
|
650
|
+
i,
|
|
595
651
|
E
|
|
596
652
|
)
|
|
597
653
|
);
|
|
598
|
-
const
|
|
654
|
+
const R = new T.AnimationClip(y, -1, p);
|
|
599
655
|
return {
|
|
600
|
-
clip:
|
|
656
|
+
clip: R,
|
|
601
657
|
saveToFile: () => {
|
|
602
|
-
new
|
|
603
|
-
|
|
658
|
+
new Ce().parse(
|
|
659
|
+
c,
|
|
604
660
|
(E) => {
|
|
605
|
-
const
|
|
661
|
+
const L = new Blob([E], {
|
|
606
662
|
type: "model/gltf-binary"
|
|
607
|
-
}),
|
|
608
|
-
|
|
663
|
+
}), w = URL.createObjectURL(L), A = document.createElement("a");
|
|
664
|
+
A.href = w, A.download = y + ".glb", A.click();
|
|
609
665
|
},
|
|
610
666
|
(E) => {
|
|
611
667
|
console.error(E);
|
|
612
668
|
},
|
|
613
669
|
{
|
|
614
670
|
binary: !0,
|
|
615
|
-
animations: [
|
|
671
|
+
animations: [R]
|
|
616
672
|
}
|
|
617
673
|
);
|
|
618
674
|
}
|
|
619
675
|
};
|
|
620
676
|
}
|
|
621
|
-
return { start:
|
|
677
|
+
return { start: x, captureFrame: d, stop: m, isRecording: () => h };
|
|
622
678
|
}
|
|
623
|
-
const
|
|
624
|
-
var
|
|
625
|
-
return !!((
|
|
679
|
+
const mt = () => {
|
|
680
|
+
var c;
|
|
681
|
+
return !!((c = navigator.mediaDevices) != null && c.getUserMedia);
|
|
626
682
|
};
|
|
627
|
-
async function
|
|
683
|
+
async function bt(c) {
|
|
628
684
|
const s = {
|
|
629
685
|
debugFrame: void 0,
|
|
630
686
|
displayScale: 1,
|
|
631
687
|
ignoreLegs: !1,
|
|
632
688
|
debugVideo: void 0,
|
|
633
689
|
ignoreFace: !1,
|
|
690
|
+
onlyFace: !1,
|
|
691
|
+
drawLandmarksOverlay: !0,
|
|
634
692
|
modelPaths: {
|
|
635
693
|
vision: "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm",
|
|
636
694
|
pose: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task",
|
|
637
695
|
hand: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
|
|
638
696
|
face: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task"
|
|
639
697
|
},
|
|
640
|
-
...
|
|
698
|
+
...c
|
|
641
699
|
};
|
|
642
700
|
let e;
|
|
643
|
-
const t = await
|
|
701
|
+
const t = await ve.forVisionTasks(s.modelPaths.vision ?? "/wasm"), o = s.onlyFace ? void 0 : await je(t, {
|
|
644
702
|
ignoreLegs: s.ignoreLegs,
|
|
645
|
-
modelPath: s.modelPaths.pose
|
|
646
|
-
|
|
703
|
+
modelPath: s.modelPaths.pose,
|
|
704
|
+
drawLandmarks: s.drawLandmarksOverlay
|
|
705
|
+
}), n = s.onlyFace ? void 0 : await et(t, {
|
|
647
706
|
leftWrist: () => o.leftWristNormalizedPosition,
|
|
648
707
|
rightWrist: () => o.rightWristNormalizedPosition,
|
|
649
708
|
modelPath: s.modelPaths.hand,
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
709
|
+
drawLandmarks: s.drawLandmarksOverlay,
|
|
710
|
+
...c == null ? void 0 : c.handsTrackerOptions
|
|
711
|
+
}), a = s.ignoreFace ? void 0 : await nt(t, { modelPath: s.modelPaths.face, videoElementRef: () => e, drawLandmarks: s.drawLandmarksOverlay }), i = document.createElement("div");
|
|
712
|
+
i.style.position = "absolute", i.style.top = "0px", i.style.left = "0px", i.style.zIndex = "21", i.classList.add("three-mediapipe-rig"), document.body.appendChild(i);
|
|
713
|
+
const r = document.createElement("canvas"), l = r.getContext("2d"), h = new ge(l);
|
|
714
|
+
r.style.zIndex = "22", r.style.position = "absolute", r.style.top = "0px", r.style.left = "0px", r.style.pointerEvents = "none", i.appendChild(r);
|
|
655
715
|
function f(d) {
|
|
656
|
-
|
|
716
|
+
l.save(), l.clearRect(0, 0, r.width, r.height), o == null || o.predict(d, h), n == null || n.left.predict(d, h), n == null || n.right.predict(d, h), a == null || a.predict(d, h), l.restore();
|
|
657
717
|
}
|
|
658
|
-
function
|
|
659
|
-
e = document.createElement("video"),
|
|
718
|
+
function x() {
|
|
719
|
+
e = document.createElement("video"), i.appendChild(e);
|
|
660
720
|
let d = -1;
|
|
661
721
|
e.style.zIndex = "21", e.style.position = "absolute", e.style.top = "0px", e.style.left = "0px", s.debugVideo && (e.src = s.debugVideo, e.controls = !0, e.loop = !0, e.muted = !0, e.controls = !0, e.play());
|
|
662
722
|
function m() {
|
|
663
723
|
d !== e.currentTime && (f(e), d = e.currentTime), window.requestAnimationFrame(m);
|
|
664
724
|
}
|
|
665
725
|
e.addEventListener("loadeddata", () => {
|
|
666
|
-
e.width = e.videoWidth * s.displayScale, e.height = e.videoHeight * s.displayScale,
|
|
726
|
+
e.width = e.videoWidth * s.displayScale, e.height = e.videoHeight * s.displayScale, r.width = e.videoWidth, r.height = e.videoHeight, r.style.height = e.height + "px", r.style.width = e.width + "px", window.requestAnimationFrame(m);
|
|
667
727
|
});
|
|
668
728
|
}
|
|
669
729
|
if (s.debugFrame) {
|
|
670
730
|
const d = document.createElement("img");
|
|
671
|
-
d.src = s.debugFrame, d.style.zIndex = "21", d.style.position = "absolute", d.style.top = "0px", d.style.left = "0px",
|
|
672
|
-
d.width = d.naturalWidth * s.displayScale, d.height = d.naturalHeight * s.displayScale,
|
|
731
|
+
d.src = s.debugFrame, d.style.zIndex = "21", d.style.position = "absolute", d.style.top = "0px", d.style.left = "0px", i.appendChild(d), d.addEventListener("load", () => {
|
|
732
|
+
d.width = d.naturalWidth * s.displayScale, d.height = d.naturalHeight * s.displayScale, r.width = d.naturalWidth, r.height = d.naturalWidth, r.style.width = d.width + "px", r.style.height = d.height + "px";
|
|
673
733
|
function m() {
|
|
674
734
|
f(d);
|
|
675
735
|
}
|
|
676
736
|
window.requestAnimationFrame(m);
|
|
677
737
|
});
|
|
678
|
-
} else s.debugVideo &&
|
|
738
|
+
} else s.debugVideo && x();
|
|
679
739
|
return {
|
|
680
740
|
poseTracker: o,
|
|
681
|
-
handsTracker:
|
|
682
|
-
faceTracker:
|
|
741
|
+
handsTracker: n,
|
|
742
|
+
faceTracker: a,
|
|
683
743
|
video: e,
|
|
744
|
+
canvas: r,
|
|
684
745
|
/**
|
|
685
746
|
* A div that contains the video and canvas used to display the landmarks stacked on top of each other.
|
|
686
747
|
*/
|
|
687
|
-
domElement:
|
|
748
|
+
domElement: i,
|
|
688
749
|
/**
|
|
689
750
|
* Start the webcam feed. This must be initiated by a user triggered event ( a click on a button ) due to security reasons.
|
|
690
751
|
*/
|
|
691
752
|
start: async () => {
|
|
692
753
|
let d = !1;
|
|
693
|
-
if (!
|
|
754
|
+
if (!mt())
|
|
694
755
|
throw new Error("Webcam not supported");
|
|
695
|
-
e ||
|
|
756
|
+
e || x();
|
|
696
757
|
let m;
|
|
697
|
-
function L
|
|
698
|
-
console.warn("Camera track ended, attempting recovery..."),
|
|
758
|
+
function y(L) {
|
|
759
|
+
console.warn("Camera track ended, attempting recovery..."), p(L), R(L);
|
|
699
760
|
}
|
|
700
|
-
function
|
|
701
|
-
m == null || m.getTracks().forEach((
|
|
761
|
+
function p(L) {
|
|
762
|
+
m == null || m.getTracks().forEach((w) => w.stop()), m = void 0, L.srcObject = null;
|
|
702
763
|
}
|
|
703
|
-
async function
|
|
704
|
-
const
|
|
705
|
-
if (
|
|
764
|
+
async function R(L, w = 0) {
|
|
765
|
+
const F = Math.min(1e3 * 2 ** w, 16e3);
|
|
766
|
+
if (w >= 5)
|
|
706
767
|
throw new Error("Camera recovery failed after max attempts");
|
|
707
|
-
if (await new Promise((
|
|
768
|
+
if (await new Promise((v) => setTimeout(v, F)), !d)
|
|
708
769
|
try {
|
|
709
|
-
await
|
|
770
|
+
await k(L), console.log("Camera recovered successfully");
|
|
710
771
|
} catch {
|
|
711
|
-
|
|
772
|
+
R(L, w + 1);
|
|
712
773
|
}
|
|
713
774
|
}
|
|
714
|
-
async function
|
|
775
|
+
async function k(L) {
|
|
715
776
|
try {
|
|
716
|
-
m = await navigator.mediaDevices.getUserMedia({ video: !0 }),
|
|
717
|
-
|
|
777
|
+
m = await navigator.mediaDevices.getUserMedia({ video: !0 }), L.srcObject = m, await L.play(), m.getVideoTracks().forEach((w) => {
|
|
778
|
+
w.addEventListener("ended", () => y(L));
|
|
718
779
|
});
|
|
719
|
-
} catch (
|
|
720
|
-
E(
|
|
780
|
+
} catch (w) {
|
|
781
|
+
E(w, L);
|
|
721
782
|
}
|
|
722
783
|
}
|
|
723
|
-
function E(
|
|
724
|
-
if (
|
|
725
|
-
switch (
|
|
784
|
+
function E(L, w) {
|
|
785
|
+
if (L instanceof DOMException)
|
|
786
|
+
switch (L.name) {
|
|
726
787
|
case "NotAllowedError":
|
|
727
788
|
throw new Error("Permission denied — prompt user to allow camera");
|
|
728
789
|
case "NotFoundError":
|
|
729
|
-
console.error("No camera found — retry when device is connected"),
|
|
790
|
+
console.error("No camera found — retry when device is connected"), R(w);
|
|
730
791
|
break;
|
|
731
792
|
case "NotReadableError":
|
|
732
|
-
console.error("Camera in use by another app"),
|
|
793
|
+
console.error("Camera in use by another app"), R(w);
|
|
733
794
|
break;
|
|
734
795
|
default:
|
|
735
|
-
console.error("Camera error:",
|
|
796
|
+
console.error("Camera error:", L.message), R(w);
|
|
736
797
|
}
|
|
737
798
|
}
|
|
738
|
-
return await
|
|
799
|
+
return await k(e), {
|
|
739
800
|
stop: () => {
|
|
740
|
-
d = !0,
|
|
801
|
+
d = !0, p(e);
|
|
741
802
|
}
|
|
742
803
|
};
|
|
743
804
|
},
|
|
@@ -747,30 +808,32 @@ async function Qe(h) {
|
|
|
747
808
|
* @param magging The bone mapping to use for the rig.
|
|
748
809
|
*/
|
|
749
810
|
bind: (d, m) => {
|
|
750
|
-
m = m ||
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
const
|
|
754
|
-
|
|
755
|
-
|
|
811
|
+
if (m = m || Le, !o) throw new Error("Pose tracker not initialized");
|
|
812
|
+
if (!n) throw new Error("Hands tracker not initialized");
|
|
813
|
+
if (!a) throw new Error("Face tracker not initialized");
|
|
814
|
+
const y = o.bind(d, m), p = n.left.bind(d, m), R = n.right.bind(d, m);
|
|
815
|
+
let k;
|
|
816
|
+
const E = a == null ? void 0 : a.bind(d);
|
|
817
|
+
d.traverse((w) => {
|
|
818
|
+
w instanceof u.Mesh && w.name.indexOf(m.faceMesh) === 0 && (w.frustumCulled = !1, k = a == null ? void 0 : a.bindShapeKeys(w));
|
|
756
819
|
});
|
|
757
|
-
const
|
|
820
|
+
const L = pt(d, m);
|
|
758
821
|
return {
|
|
759
822
|
/**
|
|
760
823
|
* Will save the tracked movement of the rig to an animation clip.
|
|
761
824
|
* Only the bones moved by the bone mapping will be recorded.
|
|
762
825
|
*/
|
|
763
|
-
startRecording:
|
|
764
|
-
stopRecording:
|
|
765
|
-
isRecording: () =>
|
|
766
|
-
update: (
|
|
767
|
-
|
|
826
|
+
startRecording: L.start,
|
|
827
|
+
stopRecording: L.stop,
|
|
828
|
+
isRecording: () => L.isRecording(),
|
|
829
|
+
update: (w) => {
|
|
830
|
+
y.update(w), p.update(w), R.update(w), k == null || k.update(w), E == null || E.update(w), L.isRecording() && L.captureFrame();
|
|
768
831
|
}
|
|
769
832
|
};
|
|
770
833
|
}
|
|
771
834
|
};
|
|
772
835
|
}
|
|
773
836
|
export {
|
|
774
|
-
|
|
837
|
+
bt as setupTracker
|
|
775
838
|
};
|
|
776
839
|
//# sourceMappingURL=three-mediapipe-rig.js.map
|