three-mediapipe-rig 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/meshcap/atlas-builder.d.ts +9 -0
- package/dist/meshcap/atlas-builder.d.ts.map +1 -1
- package/dist/meshcap/constants.d.ts +8 -1
- package/dist/meshcap/constants.d.ts.map +1 -1
- package/dist/meshcap/material.d.ts +13 -1
- package/dist/meshcap/material.d.ts.map +1 -1
- package/dist/meshcap/parse-mcap-file.d.ts +2 -1
- package/dist/meshcap/parse-mcap-file.d.ts.map +1 -1
- package/dist/meshcap/types.d.ts +25 -10
- package/dist/meshcap/types.d.ts.map +1 -1
- package/dist/meshcap/write-mcap-file.d.ts +13 -0
- package/dist/meshcap/write-mcap-file.d.ts.map +1 -1
- package/dist/meshcap.js +476 -372
- package/dist/rigger.js +418 -381
- package/dist/tracking/FaceTracker.d.ts +9 -2
- package/dist/tracking/FaceTracker.d.ts.map +1 -1
- package/dist/tracking/PoseTracker.d.ts +3 -0
- package/dist/tracking/PoseTracker.d.ts.map +1 -1
- package/dist/tracking/TrackerManager.d.ts +2 -1
- package/dist/tracking/TrackerManager.d.ts.map +1 -1
- package/dist/tracking/recoding/recorder.d.ts +1 -1
- package/dist/tracking/recoding/recorder.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/rigger.js
CHANGED
|
@@ -1,45 +1,45 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var R = (
|
|
4
|
-
import { PoseLandmarker as
|
|
1
|
+
var Ae = Object.defineProperty;
|
|
2
|
+
var Me = (c, s, e) => s in c ? Ae(c, s, { enumerable: !0, configurable: !0, writable: !0, value: e }) : c[s] = e;
|
|
3
|
+
var R = (c, s, e) => Me(c, typeof s != "symbol" ? s + "" : s, e);
|
|
4
|
+
import { PoseLandmarker as ke, HandLandmarker as G, FaceLandmarker as q, FilesetResolver as Se, DrawingUtils as Ve } from "@mediapipe/tasks-vision";
|
|
5
5
|
import * as p from "three/webgpu";
|
|
6
|
-
import { Vector3 as
|
|
7
|
-
import * as
|
|
8
|
-
import { Bone as
|
|
9
|
-
import { uniform as
|
|
10
|
-
import { c as
|
|
11
|
-
import { GLTFExporter as
|
|
12
|
-
const U = new
|
|
13
|
-
new
|
|
14
|
-
const
|
|
15
|
-
function
|
|
16
|
-
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
const n =
|
|
20
|
-
|
|
6
|
+
import { Vector3 as x, Quaternion as xe, Matrix4 as Ne, VideoTexture as Z, SRGBColorSpace as J, MeshPhysicalNodeMaterial as Oe } from "three/webgpu";
|
|
7
|
+
import * as D from "three";
|
|
8
|
+
import { Bone as Te } from "three";
|
|
9
|
+
import { uniform as z, attribute as _e, instancedArray as Ce, varying as De, vec3 as ze, float as Be, select as Ue, positionLocal as Ie, texture as He, vec2 as Ge } from "three/tsl";
|
|
10
|
+
import { c as qe, F as ee } from "./face-tracker-utils-xt9__vBF.js";
|
|
11
|
+
import { GLTFExporter as je } from "three/examples/jsm/Addons.js";
|
|
12
|
+
const U = new x(), te = new x(), Ke = new x(1, 0, 0), $e = new x(-1, 0, 0), Xe = new x(0, 1, 0), Qe = new x(0, -1, 0), oe = new x(0, 0, 1);
|
|
13
|
+
new x(0, 0, -1);
|
|
14
|
+
const se = new x(), Ye = new x(), Ze = new x(), ne = new xe(), I = new xe();
|
|
15
|
+
function N(c, s, e, t = "+x") {
|
|
16
|
+
c.lookAt(s);
|
|
17
|
+
const o = t == "+x" ? Ke : t == "-x" ? $e : t == "+y" ? Xe : Qe;
|
|
18
|
+
c.getWorldPosition(te), c.getWorldQuaternion(I), U.subVectors(e, te).normalize(), se.copy(o).applyQuaternion(I);
|
|
19
|
+
const n = se, a = Ye.copy(oe).applyQuaternion(I), r = U.clone().addScaledVector(a, -U.dot(a)).normalize(), i = Ze.crossVectors(n, r), l = Math.atan2(i.dot(a), n.dot(r));
|
|
20
|
+
ne.setFromAxisAngle(oe, l), c.quaternion.multiply(ne);
|
|
21
21
|
}
|
|
22
|
-
const
|
|
22
|
+
const B = new p.Vector3(), _ = new p.Vector3(), H = new p.Vector3();
|
|
23
23
|
new p.Vector3();
|
|
24
|
-
class
|
|
25
|
-
constructor(
|
|
24
|
+
class j {
|
|
25
|
+
constructor(s, e) {
|
|
26
26
|
R(this, "objectGhost");
|
|
27
27
|
R(this, "root");
|
|
28
28
|
/**
|
|
29
29
|
* per landmark index, it points to it's object3D equivalent.
|
|
30
30
|
*/
|
|
31
31
|
R(this, "marks", {});
|
|
32
|
-
this.points =
|
|
32
|
+
this.points = s, this.debugConnections = e, this.root = new p.Object3D(), this.objectGhost = /* @__PURE__ */ new Map();
|
|
33
33
|
for (let t in this.points)
|
|
34
|
-
this.marks[t] = new
|
|
35
|
-
}
|
|
36
|
-
updateLandmarks(
|
|
37
|
-
for (let
|
|
38
|
-
const n = this.points[
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
).divideScalar(2).add(
|
|
34
|
+
this.marks[t] = new Je(), this.root.add(this.marks[t]);
|
|
35
|
+
}
|
|
36
|
+
updateLandmarks(s, e, t) {
|
|
37
|
+
for (let o in this.points) {
|
|
38
|
+
const n = this.points[o], a = this.marks[o];
|
|
39
|
+
a && (n instanceof Array ? (B.copy(s[n[0]]), a.position.copy(s[n[1]]).sub(B).divideScalar(2).add(s[n[0]]), n.length == 4 && (B.subVectors(
|
|
40
|
+
s[n[3]],
|
|
41
|
+
s[n[2]]
|
|
42
|
+
).divideScalar(2).add(s[n[2]]).sub(a.position).divideScalar(2), a.position.add(B))) : a.position.copy(s[n]));
|
|
43
43
|
}
|
|
44
44
|
t && e && t.drawConnectors(
|
|
45
45
|
e,
|
|
@@ -49,37 +49,37 @@ class q {
|
|
|
49
49
|
}
|
|
50
50
|
);
|
|
51
51
|
}
|
|
52
|
-
getGhost(
|
|
52
|
+
getGhost(s) {
|
|
53
53
|
var e;
|
|
54
|
-
if (!this.objectGhost.has(
|
|
55
|
-
const t = new
|
|
56
|
-
t.position.copy(
|
|
54
|
+
if (!this.objectGhost.has(s)) {
|
|
55
|
+
const t = new et();
|
|
56
|
+
t.position.copy(s.position), t.quaternion.copy(s.quaternion), (e = s.parent) == null || e.add(t), this.objectGhost.set(s, t);
|
|
57
57
|
}
|
|
58
|
-
return this.objectGhost.get(
|
|
58
|
+
return this.objectGhost.get(s);
|
|
59
59
|
}
|
|
60
|
-
predict(
|
|
60
|
+
predict(s, e) {
|
|
61
61
|
throw new Error("Method 'predict' must be implemented.");
|
|
62
62
|
}
|
|
63
|
-
sync(
|
|
63
|
+
sync(s, e) {
|
|
64
64
|
throw new Error("Method 'sync' must be implemented.");
|
|
65
65
|
}
|
|
66
|
-
test(
|
|
67
|
-
this.marks[
|
|
66
|
+
test(s) {
|
|
67
|
+
this.marks[s].position.set(1, 2, 3);
|
|
68
68
|
}
|
|
69
|
-
syncObjects(
|
|
70
|
-
for (const [
|
|
71
|
-
this.marks[
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
const
|
|
75
|
-
|
|
69
|
+
syncObjects(s, e, t) {
|
|
70
|
+
for (const [o, n, a, r] of s) {
|
|
71
|
+
this.marks[a].getWorldPosition(H), this.marks[n].getWorldPosition(_);
|
|
72
|
+
const i = H.sub(_);
|
|
73
|
+
o.getWorldPosition(_), _.add(i);
|
|
74
|
+
const l = _, m = o.getWorldPosition(H).sub(t), h = this.getGhost(o);
|
|
75
|
+
N(h, l, m, r), h.rotateX(Math.PI / 2), o.position.lerp(h.position, e * 4), o.quaternion.slerp(h.quaternion, e * 4);
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
|
-
getBone(
|
|
79
|
-
return
|
|
78
|
+
getBone(s, e) {
|
|
79
|
+
return s.getObjectByName(e.replace(/[\.\:]/g, ""));
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
|
-
class
|
|
82
|
+
class Je extends p.Mesh {
|
|
83
83
|
constructor() {
|
|
84
84
|
super(new p.SphereGeometry(0.01, 3, 3), new p.MeshStandardMaterial({ color: 16711680, wireframe: !0 }));
|
|
85
85
|
R(this, "_worldPosition", new p.Vector3());
|
|
@@ -89,27 +89,27 @@ class qe extends p.Mesh {
|
|
|
89
89
|
return this.getWorldPosition(this._worldPosition), this._worldPosition;
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
|
-
class
|
|
93
|
-
lerp(
|
|
94
|
-
|
|
92
|
+
class et extends p.Object3D {
|
|
93
|
+
lerp(s, e, t = 8) {
|
|
94
|
+
s.position.lerp(this.position, e * t), s.quaternion.slerp(this.quaternion, e * t);
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
|
-
function
|
|
98
|
-
return e.worldToLocal(
|
|
97
|
+
function S(c, s, e) {
|
|
98
|
+
return e.worldToLocal(s.getWorldPosition(c)), c;
|
|
99
99
|
}
|
|
100
|
-
function
|
|
101
|
-
return
|
|
100
|
+
function tt(c) {
|
|
101
|
+
return c.replace(/[\.\:]/g, "");
|
|
102
102
|
}
|
|
103
|
-
function
|
|
103
|
+
function F(c, s) {
|
|
104
104
|
let e;
|
|
105
|
-
return
|
|
106
|
-
t.name.indexOf(
|
|
107
|
-
}), e || console.log("Bone not found: ",
|
|
105
|
+
return s = tt(s), c.traverse((t) => {
|
|
106
|
+
t.name.indexOf(s) === 0 && t instanceof Te && (e = t);
|
|
107
|
+
}), e || console.log("Bone not found: ", s, c.name), e;
|
|
108
108
|
}
|
|
109
|
-
async function
|
|
110
|
-
const e = await
|
|
109
|
+
async function ot(c, s) {
|
|
110
|
+
const e = await ke.createFromOptions(c, {
|
|
111
111
|
baseOptions: {
|
|
112
|
-
modelAssetPath: (
|
|
112
|
+
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "pose_landmarker_lite.task",
|
|
113
113
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task`,
|
|
114
114
|
//modelAssetPath: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_heavy/float16/latest/pose_landmarker_heavy.task",
|
|
115
115
|
delegate: "GPU"
|
|
@@ -117,9 +117,9 @@ async function Xe(h, o) {
|
|
|
117
117
|
runningMode: "VIDEO",
|
|
118
118
|
numPoses: 1
|
|
119
119
|
});
|
|
120
|
-
return new
|
|
120
|
+
return new dt(e, s);
|
|
121
121
|
}
|
|
122
|
-
const
|
|
122
|
+
const st = {
|
|
123
123
|
hips: [24, 23],
|
|
124
124
|
neck: [12, 11],
|
|
125
125
|
leftLeg: 23,
|
|
@@ -137,17 +137,20 @@ const Qe = {
|
|
|
137
137
|
rightElbow: 14,
|
|
138
138
|
rightWrist: 16,
|
|
139
139
|
head: [8, 7],
|
|
140
|
+
forehead: [5, 2],
|
|
140
141
|
mouth: [10, 9],
|
|
141
142
|
torso: [24, 23, 12, 11],
|
|
142
143
|
leftEar: 7,
|
|
143
|
-
rightEar: 8
|
|
144
|
-
|
|
145
|
-
|
|
144
|
+
rightEar: 8,
|
|
145
|
+
nose: 0
|
|
146
|
+
}, nt = new p.Vector3(), ie = new p.Vector3(), it = new p.Vector3(), rt = new p.Vector3(), at = new p.Vector3(), re = new p.Vector3(), ae = new p.Vector3();
|
|
147
|
+
class dt extends j {
|
|
146
148
|
constructor(e, t) {
|
|
147
|
-
super(
|
|
149
|
+
super(st, ke.POSE_CONNECTIONS);
|
|
148
150
|
R(this, "_leftWristNormalizedPosition");
|
|
149
151
|
R(this, "_rightWristNormalizedPosition");
|
|
150
|
-
this
|
|
152
|
+
R(this, "ignoreLegs", !1);
|
|
153
|
+
this.poseLandmarker = e, this.config = t, this.root.scale.y *= -2, this.root.scale.z *= -2, this.root.scale.x *= 2, this.ignoreLegs = (t == null ? void 0 : t.ignoreLegs) ?? !1;
|
|
151
154
|
}
|
|
152
155
|
/**
|
|
153
156
|
* Position of the left wrist in normalized coordinates (0..1)
|
|
@@ -162,9 +165,9 @@ class tt extends q {
|
|
|
162
165
|
return this._rightWristNormalizedPosition;
|
|
163
166
|
}
|
|
164
167
|
predict(e, t) {
|
|
165
|
-
this.poseLandmarker.detectForVideo(e, performance.now(), (
|
|
168
|
+
this.poseLandmarker.detectForVideo(e, performance.now(), (o) => {
|
|
166
169
|
var n;
|
|
167
|
-
|
|
170
|
+
o.landmarks.length != 0 && (this.updateLandmarks(o.worldLandmarks[0], ((n = this.config) == null ? void 0 : n.drawLandmarks) === !1 ? void 0 : o.landmarks[0], t), this._leftWristNormalizedPosition = o.landmarks[0][this.points.leftWrist], this._rightWristNormalizedPosition = o.landmarks[0][this.points.rightWrist]);
|
|
168
171
|
});
|
|
169
172
|
}
|
|
170
173
|
// override sync ( delta:number, objects: BoneBinding[] ) {
|
|
@@ -175,65 +178,65 @@ class tt extends q {
|
|
|
175
178
|
// this.syncObjects(objects, delta, torsoNormal);
|
|
176
179
|
// }
|
|
177
180
|
bind(e, t) {
|
|
178
|
-
var
|
|
179
|
-
const
|
|
180
|
-
hips:
|
|
181
|
-
neck:
|
|
182
|
-
leftArm:
|
|
183
|
-
leftElbow:
|
|
184
|
-
leftWrist:
|
|
185
|
-
rightArm:
|
|
186
|
-
rightElbow:
|
|
187
|
-
rightWrist:
|
|
188
|
-
head:
|
|
189
|
-
torso:
|
|
190
|
-
leftLeg:
|
|
191
|
-
leftKnee:
|
|
192
|
-
leftFoot:
|
|
193
|
-
rightLeg:
|
|
194
|
-
rightKnee:
|
|
195
|
-
rightFoot:
|
|
181
|
+
var i;
|
|
182
|
+
const o = {
|
|
183
|
+
hips: F(e, t.hips),
|
|
184
|
+
neck: F(e, t.neck),
|
|
185
|
+
leftArm: F(e, t.armL),
|
|
186
|
+
leftElbow: F(e, t.forearmL),
|
|
187
|
+
leftWrist: F(e, t.handL),
|
|
188
|
+
rightArm: F(e, t.armR),
|
|
189
|
+
rightElbow: F(e, t.forearmR),
|
|
190
|
+
rightWrist: F(e, t.handR),
|
|
191
|
+
head: F(e, t.head),
|
|
192
|
+
torso: F(e, t.torso),
|
|
193
|
+
leftLeg: F(e, t.thighL),
|
|
194
|
+
leftKnee: F(e, t.shinL),
|
|
195
|
+
leftFoot: F(e, t.footL),
|
|
196
|
+
rightLeg: F(e, t.thighR),
|
|
197
|
+
rightKnee: F(e, t.shinR),
|
|
198
|
+
rightFoot: F(e, t.footR)
|
|
196
199
|
};
|
|
197
|
-
(
|
|
198
|
-
const n = new p.Vector3(),
|
|
199
|
-
if (!
|
|
200
|
-
const
|
|
201
|
-
|
|
202
|
-
const
|
|
203
|
-
|
|
200
|
+
(i = this.config) != null && i.ignoreLegs && (delete o.leftLeg, delete o.leftKnee, delete o.leftFoot, delete o.leftToes, delete o.rightLeg, delete o.rightKnee, delete o.rightFoot, delete o.rightToes);
|
|
201
|
+
const n = new p.Vector3(), a = new p.Vector3(), r = (l, m, h, b, u, E) => {
|
|
202
|
+
if (!m) return;
|
|
203
|
+
const d = this.marks[b].getWorldPosition(n).sub(this.marks[h].getWorldPosition(a)).normalize();
|
|
204
|
+
S(re, m, e).add(d).applyMatrix4(e.matrixWorld), S(ae, m, e).add(u).applyMatrix4(e.matrixWorld);
|
|
205
|
+
const y = this.getGhost(m);
|
|
206
|
+
N(y, re, ae, E), y.rotateX(Math.PI / 2), y.lerp(m, l);
|
|
204
207
|
};
|
|
205
208
|
return {
|
|
206
|
-
update: (
|
|
207
|
-
const
|
|
208
|
-
|
|
209
|
+
update: (l) => {
|
|
210
|
+
const m = this.marks.leftLeg.getWorldPosition(nt).sub(this.marks.rightLeg.getWorldPosition(ie)).normalize(), h = this.marks.leftArm.getWorldPosition(ie).sub(this.marks.rightArm.getWorldPosition(it)).normalize(), b = this.marks.leftEar.getWorldPosition(rt).sub(this.marks.rightEar.getWorldPosition(at)).normalize();
|
|
211
|
+
r(l, o.hips, "hips", "torso", m, "+x"), r(l, o.torso, "torso", "neck", h, "+x"), r(l, o.neck, "neck", "head", b, "+x"), r(l, o.head, "nose", "forehead", b, "+x"), r(l, o.leftArm, "leftArm", "leftElbow", h, "-x"), r(l, o.leftElbow, "leftElbow", "leftWrist", h, "-x"), r(l, o.rightArm, "rightArm", "rightElbow", h, "-x"), r(l, o.rightElbow, "rightElbow", "rightWrist", h, "-x"), !this.ignoreLegs && (r(l, o.leftLeg, "leftLeg", "leftKnee", m, "+x"), r(l, o.leftKnee, "leftKnee", "leftFoot", m, "+x"), r(l, o.leftFoot, "leftFoot", "leftToes", m, "+x"), r(l, o.rightLeg, "rightLeg", "rightKnee", m, "+x"), r(l, o.rightKnee, "rightKnee", "rightFoot", m, "+x"), r(l, o.rightFoot, "rightFoot", "rightToes", m, "+x"));
|
|
209
212
|
}
|
|
210
213
|
};
|
|
211
214
|
}
|
|
212
215
|
}
|
|
213
|
-
const
|
|
214
|
-
async function
|
|
215
|
-
const e = await G.createFromOptions(
|
|
216
|
+
const de = new p.Vector2(), ce = new p.Vector2();
|
|
217
|
+
async function ct(c, s) {
|
|
218
|
+
const e = await G.createFromOptions(c, {
|
|
216
219
|
baseOptions: {
|
|
217
|
-
modelAssetPath:
|
|
220
|
+
modelAssetPath: s.modelPath ?? "hand_landmarker.task",
|
|
218
221
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
|
|
219
222
|
delegate: "GPU"
|
|
220
223
|
},
|
|
221
224
|
runningMode: "VIDEO",
|
|
222
225
|
numHands: 2
|
|
223
|
-
}), t = (
|
|
226
|
+
}), t = (o, n, a) => {
|
|
224
227
|
try {
|
|
225
|
-
|
|
226
|
-
} catch (
|
|
227
|
-
return console.warn("No pose data... will just be optimitic and say yes to everything.",
|
|
228
|
+
de.copy(o()), ce.copy(n());
|
|
229
|
+
} catch (r) {
|
|
230
|
+
return console.warn("No pose data... will just be optimitic and say yes to everything.", r), !0;
|
|
228
231
|
}
|
|
229
|
-
return
|
|
232
|
+
return de.distanceTo(a) < ce.distanceTo(a);
|
|
230
233
|
};
|
|
231
234
|
return {
|
|
232
|
-
left: new
|
|
233
|
-
right: new
|
|
235
|
+
left: new ye(e, "Left", t.bind(null, s.leftWrist, s.rightWrist), s.drawLandmarks),
|
|
236
|
+
right: new ye(e, "Right", t.bind(null, s.rightWrist, s.leftWrist), s.drawLandmarks)
|
|
234
237
|
};
|
|
235
238
|
}
|
|
236
|
-
const
|
|
239
|
+
const lt = {
|
|
237
240
|
wrist: 0,
|
|
238
241
|
palm: [9, 13],
|
|
239
242
|
thumb1: 1,
|
|
@@ -256,38 +259,38 @@ const st = {
|
|
|
256
259
|
pinky2: 18,
|
|
257
260
|
pinky3: 19,
|
|
258
261
|
pinky4: 20
|
|
259
|
-
},
|
|
262
|
+
}, C = {
|
|
260
263
|
thumb: ["thumb1", "thumb2", "thumb3", "thumb4"],
|
|
261
264
|
index: ["index1", "index2", "index3", "index4"],
|
|
262
265
|
middle: ["middle1", "middle2", "middle3", "middle4"],
|
|
263
266
|
ring: ["ring1", "ring2", "ring3", "ring4"],
|
|
264
267
|
pinky: ["pinky1", "pinky2", "pinky3", "pinky4"]
|
|
265
|
-
},
|
|
268
|
+
}, ht = new p.Vector3(), le = new p.Vector3(), he = new p.Vector3(), me = new p.Vector3(), pe = new p.Vector3(), ue = new p.Vector3();
|
|
266
269
|
new p.Vector3();
|
|
267
270
|
new p.Vector3();
|
|
268
271
|
new p.Vector3();
|
|
269
|
-
const
|
|
270
|
-
class
|
|
271
|
-
constructor(e, t,
|
|
272
|
-
super(
|
|
272
|
+
const fe = new p.Vector3(), we = Math.PI / 2, mt = new p.Vector3(0, -1, 0);
|
|
273
|
+
class ye extends j {
|
|
274
|
+
constructor(e, t, o, n = !0) {
|
|
275
|
+
super(lt, G.HAND_CONNECTIONS);
|
|
273
276
|
R(this, "sign");
|
|
274
277
|
R(this, "isLeft");
|
|
275
278
|
/**
|
|
276
279
|
* the axis used to look at the pole
|
|
277
280
|
*/
|
|
278
281
|
R(this, "lookAtPoleAxis");
|
|
279
|
-
this.handLandmarker = e, this.side = t, this.isMyWrist =
|
|
282
|
+
this.handLandmarker = e, this.side = t, this.isMyWrist = o, this.drawLandmarks = n, this.sign = this.side == "Left" ? -1 : 1, this.isLeft = this.side == "Left", this.lookAtPoleAxis = this.sign < 0 ? "+x" : "-x", this.root.scale.setScalar(7), this.root.scale.y *= -1, this.root.scale.z *= -1;
|
|
280
283
|
}
|
|
281
284
|
predict(e, t) {
|
|
282
|
-
const
|
|
283
|
-
if (
|
|
284
|
-
for (let n = 0; n <
|
|
285
|
-
const
|
|
286
|
-
if (this.isMyWrist(
|
|
287
|
-
this.updateLandmarks(
|
|
285
|
+
const o = this.handLandmarker.detectForVideo(e, performance.now());
|
|
286
|
+
if (o.landmarks.length)
|
|
287
|
+
for (let n = 0; n < o.landmarks.length; n++) {
|
|
288
|
+
const a = o.landmarks[n], r = a[this.points.wrist];
|
|
289
|
+
if (this.isMyWrist(r)) {
|
|
290
|
+
this.updateLandmarks(o.worldLandmarks[n]), this.drawLandmarks && (t.drawConnectors(a, G.HAND_CONNECTIONS, {
|
|
288
291
|
color: this.side == "Left" ? "#00FF00" : "#0000FF",
|
|
289
292
|
lineWidth: 4
|
|
290
|
-
}), t.drawLandmarks(
|
|
293
|
+
}), t.drawLandmarks(a, { color: this.side == "Left" ? "#00FF00" : "#0000FF", lineWidth: 3, radius: 1 }));
|
|
291
294
|
break;
|
|
292
295
|
}
|
|
293
296
|
}
|
|
@@ -301,68 +304,69 @@ class we extends q {
|
|
|
301
304
|
* @param landmark2bones Array the same size as the umber of hand landmarks, and on each positionthe bone that belongs to that point.
|
|
302
305
|
* @see https://ai.google.dev/edge/mediapipe/solutions/vision/hand_landmarker
|
|
303
306
|
*/
|
|
304
|
-
syncHandBones(e, t,
|
|
305
|
-
const n =
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
).normalize(),
|
|
309
|
-
if (!(
|
|
307
|
+
syncHandBones(e, t, o) {
|
|
308
|
+
const n = ht.crossVectors(
|
|
309
|
+
le.copy(this.marks.index1.worldPosition).sub(this.marks.wrist.worldPosition),
|
|
310
|
+
he.copy(this.marks.pinky1.worldPosition).sub(this.marks.wrist.worldPosition)
|
|
311
|
+
).normalize(), a = le.copy(this.marks.palm.worldPosition).sub(this.marks.wrist.worldPosition).normalize(), r = he.copy(this.marks.pinky1.worldPosition).sub(this.marks.index1.worldPosition).normalize();
|
|
312
|
+
if (!(a.dot(mt) > 0.8)) {
|
|
310
313
|
if (t.wrist) {
|
|
311
|
-
const
|
|
312
|
-
|
|
314
|
+
const i = S(me, t.wrist, o).add(a).applyMatrix4(o.matrixWorld), l = S(pe, t.wrist, o).sub(r).applyMatrix4(o.matrixWorld), m = this.getGhost(t.wrist);
|
|
315
|
+
N(m, i, l, "-y"), m.rotateX(we), m.lerp(t.wrist, e);
|
|
313
316
|
}
|
|
314
|
-
this.syncFinger(e,
|
|
317
|
+
this.syncFinger(e, o, n, a, r, t, C.index, "middle1"), this.syncFinger(e, o, n, a, r, t, C.middle, "ring1"), this.syncFinger(e, o, n, a, r, t, C.ring, "pinky1"), this.syncFinger(e, o, n, a, r, t, C.pinky, "ring1", !0), this.syncFinger(e, o, n, a, r, t, C.thumb, "index1");
|
|
315
318
|
}
|
|
316
319
|
}
|
|
317
|
-
syncFinger(e, t,
|
|
318
|
-
for (let
|
|
319
|
-
const
|
|
320
|
-
if (!
|
|
321
|
-
const
|
|
322
|
-
if (
|
|
323
|
-
const
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
320
|
+
syncFinger(e, t, o, n, a, r, i, l, m = !1) {
|
|
321
|
+
for (let h = 0; h < i.length - 1; h++) {
|
|
322
|
+
const b = r[i[h]];
|
|
323
|
+
if (!b) continue;
|
|
324
|
+
const u = this.getGhost(b), E = me.copy(this.marks[i[h + 1]].worldPosition).sub(this.marks[i[h]].worldPosition).normalize(), d = S(pe, b, t);
|
|
325
|
+
if (h == 0) {
|
|
326
|
+
const y = ue.copy(this.marks[l].worldPosition).sub(this.marks[i[0]].worldPosition).normalize();
|
|
327
|
+
m && y.negate(), fe.copy(y), N(
|
|
328
|
+
u,
|
|
329
|
+
E.add(d).applyMatrix4(t.matrixWorld),
|
|
330
|
+
y.add(d).applyMatrix4(t.matrixWorld),
|
|
328
331
|
this.lookAtPoleAxis
|
|
329
332
|
);
|
|
330
333
|
} else
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
334
|
+
N(
|
|
335
|
+
u,
|
|
336
|
+
E.add(d).applyMatrix4(t.matrixWorld),
|
|
337
|
+
ue.copy(fe).add(d).applyMatrix4(t.matrixWorld),
|
|
335
338
|
this.lookAtPoleAxis
|
|
336
339
|
);
|
|
337
|
-
|
|
340
|
+
u.rotateX(we), u.lerp(b, e);
|
|
338
341
|
}
|
|
339
342
|
}
|
|
340
343
|
bind(e, t) {
|
|
341
|
-
const
|
|
342
|
-
const
|
|
343
|
-
if (
|
|
344
|
-
return
|
|
344
|
+
const o = {}, n = (a, r) => {
|
|
345
|
+
const i = F(e, a);
|
|
346
|
+
if (i)
|
|
347
|
+
return o[r] = i, r;
|
|
345
348
|
};
|
|
346
349
|
return n(this.isLeft ? t.handL : t.handR, "wrist"), n(this.isLeft ? t.index1L : t.index1R, "index1"), n(this.isLeft ? t.index2L : t.index2R, "index2"), n(this.isLeft ? t.index3L : t.index3R, "index3"), n(this.isLeft ? t.middle1L : t.middle1R, "middle1"), n(this.isLeft ? t.middle2L : t.middle2R, "middle2"), n(this.isLeft ? t.middle3L : t.middle3R, "middle3"), n(this.isLeft ? t.ring1L : t.ring1R, "ring1"), n(this.isLeft ? t.ring2L : t.ring2R, "ring2"), n(this.isLeft ? t.ring3L : t.ring3R, "ring3"), n(this.isLeft ? t.pinky1L : t.pinky1R, "pinky1"), n(this.isLeft ? t.pinky2L : t.pinky2R, "pinky2"), n(this.isLeft ? t.pinky3L : t.pinky3R, "pinky3"), n(this.isLeft ? t.thumb1L : t.thumb1R, "thumb1"), n(this.isLeft ? t.thumb2L : t.thumb2R, "thumb2"), n(this.isLeft ? t.thumb3L : t.thumb3R, "thumb3"), {
|
|
347
|
-
update: (
|
|
348
|
-
this.syncHandBones(
|
|
350
|
+
update: (a) => {
|
|
351
|
+
this.syncHandBones(a, o, e);
|
|
349
352
|
}
|
|
350
353
|
};
|
|
351
354
|
}
|
|
352
355
|
}
|
|
353
|
-
async function
|
|
354
|
-
const e = await
|
|
356
|
+
async function pt(c, s) {
|
|
357
|
+
const e = await q.createFromOptions(c, {
|
|
355
358
|
baseOptions: {
|
|
356
|
-
modelAssetPath: (
|
|
359
|
+
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "face_landmarker.task",
|
|
357
360
|
delegate: "GPU"
|
|
358
361
|
},
|
|
359
362
|
outputFaceBlendshapes: !0,
|
|
363
|
+
outputFacialTransformationMatrixes: !0,
|
|
360
364
|
runningMode: "VIDEO",
|
|
361
365
|
numFaces: 1
|
|
362
366
|
});
|
|
363
|
-
return new
|
|
367
|
+
return new kt(e, { ...s });
|
|
364
368
|
}
|
|
365
|
-
const
|
|
369
|
+
const ut = {
|
|
366
370
|
eyeL: 473,
|
|
367
371
|
eyeR: 468,
|
|
368
372
|
eyeStartL: 463,
|
|
@@ -375,51 +379,82 @@ const at = {
|
|
|
375
379
|
noseBone: 6,
|
|
376
380
|
chin: 152,
|
|
377
381
|
forehead: 10
|
|
378
|
-
},
|
|
379
|
-
new
|
|
380
|
-
class
|
|
382
|
+
}, ft = new x(), wt = new x(), be = new x(), yt = new x(), Lt = new x();
|
|
383
|
+
new x();
|
|
384
|
+
class kt extends j {
|
|
381
385
|
constructor(e, t) {
|
|
382
|
-
super(
|
|
386
|
+
super(ut, q.FACE_LANDMARKS_TESSELATION);
|
|
383
387
|
R(this, "blendshapeCategories");
|
|
384
388
|
R(this, "blendshapeMap", /* @__PURE__ */ new Map());
|
|
385
389
|
R(this, "smoothed", {});
|
|
386
390
|
R(this, "smoothing", 3e-4);
|
|
387
391
|
// lower = smoother but more lag, higher = more responsive
|
|
388
392
|
R(this, "_faceLandmarks", []);
|
|
393
|
+
R(this, "_facialTransformationMatrix");
|
|
389
394
|
this.faceLandmarker = e, this.cfg = t, this.root.scale.y *= -1, this.root.scale.z *= -1, this.root.scale.multiplyScalar(3);
|
|
390
395
|
}
|
|
391
396
|
predict(e, t) {
|
|
392
|
-
var n,
|
|
393
|
-
const
|
|
394
|
-
|
|
395
|
-
this.
|
|
397
|
+
var n, a, r;
|
|
398
|
+
const o = this.faceLandmarker.detectForVideo(e, performance.now());
|
|
399
|
+
if (o.faceLandmarks[0]) {
|
|
400
|
+
this.cfg.drawLandmarks && (t.drawConnectors(o.faceLandmarks[0], q.FACE_LANDMARKS_TESSELATION, { color: "#00fff2ff", lineWidth: 0.1 }), t.drawLandmarks(o.faceLandmarks[0], { color: "#00ff00", lineWidth: 0.1, radius: 0.4 })), this.updateLandmarks(o.faceLandmarks[0], o.faceLandmarks[0]), this._faceLandmarks = o.faceLandmarks[0], this._facialTransformationMatrix || (this._facialTransformationMatrix = new Ne());
|
|
401
|
+
const i = o.facialTransformationMatrixes[0];
|
|
402
|
+
this._facialTransformationMatrix.set(
|
|
403
|
+
i.data[0],
|
|
404
|
+
i.data[1],
|
|
405
|
+
i.data[2],
|
|
406
|
+
i.data[3],
|
|
407
|
+
i.data[4],
|
|
408
|
+
i.data[5],
|
|
409
|
+
i.data[6],
|
|
410
|
+
i.data[7],
|
|
411
|
+
i.data[8],
|
|
412
|
+
i.data[9],
|
|
413
|
+
i.data[10],
|
|
414
|
+
i.data[11],
|
|
415
|
+
i.data[12],
|
|
416
|
+
i.data[13],
|
|
417
|
+
i.data[14],
|
|
418
|
+
i.data[15]
|
|
419
|
+
);
|
|
420
|
+
}
|
|
421
|
+
this.blendshapeCategories = (a = (n = o.faceBlendshapes) == null ? void 0 : n[0]) == null ? void 0 : a.categories, (r = this.blendshapeCategories) == null || r.forEach((i) => {
|
|
422
|
+
this.blendshapeMap.set(i.categoryName, i.score);
|
|
396
423
|
});
|
|
397
424
|
}
|
|
398
425
|
get lastKnownLandmarks() {
|
|
399
426
|
return this._faceLandmarks;
|
|
400
427
|
}
|
|
428
|
+
/**
|
|
429
|
+
* this matrix is useful to know how the canonical mesh is rotated.
|
|
430
|
+
* This allows to move other objects to simulate the same transformation that the skull/face is having.
|
|
431
|
+
* Used for things like hats, glasses, etc... things you want to make look like they are attatched to the face mesh.
|
|
432
|
+
*/
|
|
433
|
+
get lastKnownFacialTransformationMatrix() {
|
|
434
|
+
return this._facialTransformationMatrix;
|
|
435
|
+
}
|
|
401
436
|
bindShapeKeys(e) {
|
|
402
437
|
const t = e.morphTargetDictionary;
|
|
403
438
|
return {
|
|
404
|
-
update: (
|
|
439
|
+
update: (o) => {
|
|
405
440
|
var n;
|
|
406
|
-
(n = this.blendshapeCategories) == null || n.forEach((
|
|
407
|
-
const { categoryName:
|
|
408
|
-
if (!(t != null && t.hasOwnProperty(
|
|
409
|
-
this.smoothed[
|
|
410
|
-
const
|
|
411
|
-
this.smoothed[
|
|
441
|
+
(n = this.blendshapeCategories) == null || n.forEach((a) => {
|
|
442
|
+
const { categoryName: r, score: i } = a;
|
|
443
|
+
if (!(t != null && t.hasOwnProperty(r))) return;
|
|
444
|
+
this.smoothed[r] === void 0 && (this.smoothed[r] = i);
|
|
445
|
+
const l = 1 - Math.pow(this.smoothing, o);
|
|
446
|
+
this.smoothed[r] += (i - this.smoothed[r]) * l, e.morphTargetInfluences[t[r]] = this.smoothed[r];
|
|
412
447
|
});
|
|
413
448
|
}
|
|
414
449
|
};
|
|
415
450
|
}
|
|
416
451
|
bind(e) {
|
|
417
|
-
const t = new
|
|
452
|
+
const t = new Le(e, "L"), o = new Le(e, "R"), n = F(e, "head");
|
|
418
453
|
return {
|
|
419
|
-
update: (
|
|
420
|
-
if (t.update(
|
|
421
|
-
const
|
|
422
|
-
|
|
454
|
+
update: (a) => {
|
|
455
|
+
if (t.update(a, this.blendshapeMap), o.update(a, this.blendshapeMap), !n) return;
|
|
456
|
+
const r = ft.copy(this.marks.earL.worldPosition), i = wt.copy(this.marks.earR.worldPosition), l = be.subVectors(r, i).multiplyScalar(0.5).add(i), m = yt.subVectors(this.marks.noseTip.worldPosition, l), h = r.sub(i), b = S(Lt, n, e), u = h.add(b).applyMatrix4(e.matrixWorld), E = m.add(b).applyMatrix4(e.matrixWorld);
|
|
457
|
+
N(n, E, u, "+x");
|
|
423
458
|
}
|
|
424
459
|
};
|
|
425
460
|
}
|
|
@@ -431,84 +466,85 @@ class mt extends q {
|
|
|
431
466
|
* @returns an object with a disposeMaterial method that should be called when the mesh is disposed of.
|
|
432
467
|
*/
|
|
433
468
|
bindGeometry(e, t) {
|
|
434
|
-
new
|
|
435
|
-
const
|
|
436
|
-
new
|
|
437
|
-
const
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
469
|
+
new x(), new x();
|
|
470
|
+
const o = new x();
|
|
471
|
+
new x();
|
|
472
|
+
const a = e.geometry.attributes.position, r = 209, i = 429, l = z(new x().addVectors(
|
|
473
|
+
new x(a.getX(r), a.getY(r), a.getZ(r)),
|
|
474
|
+
new x(a.getX(i), a.getY(i), a.getZ(i))
|
|
475
|
+
).multiplyScalar(0.5));
|
|
476
|
+
qe(e);
|
|
477
|
+
const m = _e("landmarkIndex", "uint"), h = Ce(ee, "vec3"), b = De(h.element(m)).xy;
|
|
478
|
+
let u;
|
|
479
|
+
const E = 116, d = 346, y = o.subVectors(
|
|
480
|
+
new x(a.getX(E), a.getY(E), a.getZ(E)),
|
|
481
|
+
new x(a.getX(d), a.getY(d), a.getZ(d))
|
|
482
|
+
).lengthSq(), P = z(y), k = z(1);
|
|
483
|
+
let L;
|
|
448
484
|
return {
|
|
449
485
|
/**
|
|
450
486
|
* Disposes of the material and removes events listeners on the video element.
|
|
451
487
|
*/
|
|
452
488
|
disposeMaterial: () => {
|
|
453
|
-
|
|
489
|
+
L == null || L();
|
|
454
490
|
},
|
|
455
491
|
/**
|
|
456
492
|
* asas
|
|
457
493
|
* @param delta asas
|
|
458
494
|
* @returns
|
|
459
495
|
*/
|
|
460
|
-
update: (
|
|
461
|
-
var
|
|
462
|
-
if (!
|
|
463
|
-
const
|
|
464
|
-
if (!
|
|
465
|
-
|
|
466
|
-
const
|
|
467
|
-
|
|
468
|
-
const
|
|
469
|
-
const
|
|
470
|
-
|
|
496
|
+
update: (v) => {
|
|
497
|
+
var f, A;
|
|
498
|
+
if (!u) {
|
|
499
|
+
const W = (A = (f = this.cfg) == null ? void 0 : f.videoElementRef) == null ? void 0 : A.call(f);
|
|
500
|
+
if (!W || !W.videoWidth || !W.videoHeight) return;
|
|
501
|
+
u = W;
|
|
502
|
+
const V = new Z(u);
|
|
503
|
+
V.colorSpace = J;
|
|
504
|
+
const M = z(u.videoWidth / u.videoHeight), Pe = h.element(d).sub(h.element(E)).lengthSq(), ve = P.div(Pe).sqrt(), ge = h.element(234).xy, K = h.element(93).xy, Ee = h.element(454).xy, $ = h.element(323).xy, X = ge.sub(K).div(2).add(K), We = Ee.sub($).div(2).add($).sub(X).div(2).add(X), Fe = h.element(m).sub(We).xzy.mul(ze(1, -1, Be(1).div(M))).mul(ve).add(l), Q = Ue(m.toUint().lessThanEqual(ee), Fe, Ie), O = He(V, Ge(b.x, b.y.oneMinus())), Y = () => {
|
|
505
|
+
const T = new Z(u);
|
|
506
|
+
T.colorSpace = J, O.value = T, O.needsUpdate = !0, M.value = u.videoWidth / u.videoHeight;
|
|
471
507
|
};
|
|
472
|
-
|
|
508
|
+
u.addEventListener("loadeddata", Y), t ? t(Q, O) : e.material = new Oe({
|
|
473
509
|
positionNode: Q,
|
|
474
|
-
colorNode:
|
|
510
|
+
colorNode: O,
|
|
475
511
|
roughness: 0.93
|
|
476
|
-
}),
|
|
477
|
-
var
|
|
478
|
-
|
|
512
|
+
}), L = () => {
|
|
513
|
+
var T;
|
|
514
|
+
O.value.dispose(), u == null || u.removeEventListener("loadeddata", Y), (T = e.material) == null || T.dispose();
|
|
479
515
|
};
|
|
480
516
|
}
|
|
481
517
|
const w = this.lastKnownLandmarks;
|
|
482
518
|
if (!(w != null && w.length)) return;
|
|
483
|
-
|
|
484
|
-
const
|
|
485
|
-
for (let
|
|
486
|
-
|
|
487
|
-
|
|
519
|
+
k.value = o.subVectors(w[d], w[E]).lengthSq(), k.needsUpdate = !0;
|
|
520
|
+
const g = h.value.array;
|
|
521
|
+
for (let W = 0; W < w.length; W++)
|
|
522
|
+
g[W * 3] = w[W].x, g[W * 3 + 1] = w[W].y, g[W * 3 + 2] = w[W].z;
|
|
523
|
+
h.value.needsUpdate = !0;
|
|
488
524
|
}
|
|
489
525
|
};
|
|
490
526
|
}
|
|
491
527
|
}
|
|
492
|
-
class
|
|
493
|
-
constructor(
|
|
528
|
+
class Le {
|
|
529
|
+
constructor(s, e) {
|
|
494
530
|
R(this, "eyeBone");
|
|
495
531
|
R(this, "eyeLookOut");
|
|
496
532
|
R(this, "eyeLookIn");
|
|
497
533
|
R(this, "eyeLookUp");
|
|
498
534
|
R(this, "eyeLookDown");
|
|
499
535
|
R(this, "sign", 1);
|
|
500
|
-
this.rig =
|
|
536
|
+
this.rig = s, this.side = e, this.eyeBone = s.getObjectByName(`eye${e}`);
|
|
501
537
|
const t = e == "L" ? "Left" : "Right";
|
|
502
538
|
this.eyeLookOut = `eyeLookOut${t}`, this.eyeLookIn = `eyeLookIn${t}`, this.eyeLookUp = `eyeLookUp${t}`, this.eyeLookDown = `eyeLookDown${t}`, this.sign = e == "L" ? -1 : 1;
|
|
503
539
|
}
|
|
504
|
-
update(
|
|
540
|
+
update(s, e) {
|
|
505
541
|
if (!this.eyeBone) return;
|
|
506
|
-
|
|
507
|
-
const t = e.get(this.eyeLookOut) ?? 0,
|
|
508
|
-
this.eyeBone.rotation.y =
|
|
542
|
+
S(be, this.eyeBone, this.rig);
|
|
543
|
+
const t = e.get(this.eyeLookOut) ?? 0, o = e.get(this.eyeLookIn) ?? 0, n = e.get(this.eyeLookUp) ?? 0, a = e.get(this.eyeLookDown) ?? 0, r = o - t, i = a - n;
|
|
544
|
+
this.eyeBone.rotation.y = r * this.sign / 2, this.eyeBone.rotation.x = i / 2;
|
|
509
545
|
}
|
|
510
546
|
}
|
|
511
|
-
const
|
|
547
|
+
const Re = {
|
|
512
548
|
faceMesh: "face",
|
|
513
549
|
head: "head",
|
|
514
550
|
hips: "hips",
|
|
@@ -558,7 +594,7 @@ const xe = {
|
|
|
558
594
|
thumb1R: "thumb1R",
|
|
559
595
|
thumb2R: "thumb2R",
|
|
560
596
|
thumb3R: "thumb3R"
|
|
561
|
-
},
|
|
597
|
+
}, xt = [
|
|
562
598
|
"eyeBlinkLeft",
|
|
563
599
|
"eyeBlinkRight",
|
|
564
600
|
"eyeLookDownLeft",
|
|
@@ -612,94 +648,96 @@ const xe = {
|
|
|
612
648
|
"mouthUpperUpRight",
|
|
613
649
|
"tongueOut"
|
|
614
650
|
];
|
|
615
|
-
function
|
|
616
|
-
const t = [],
|
|
617
|
-
|
|
651
|
+
function bt(c, s, e = 30) {
|
|
652
|
+
const t = [], o = c.getObjectByName(
|
|
653
|
+
s.faceMesh
|
|
618
654
|
), n = /* @__PURE__ */ new Set();
|
|
619
|
-
if (
|
|
620
|
-
for (const
|
|
621
|
-
|
|
622
|
-
const
|
|
623
|
-
|
|
624
|
-
if (
|
|
625
|
-
const
|
|
626
|
-
|
|
627
|
-
ref:
|
|
628
|
-
name:
|
|
629
|
-
normalizedName:
|
|
655
|
+
if (o != null && o.morphTargetDictionary)
|
|
656
|
+
for (const k in o.morphTargetDictionary)
|
|
657
|
+
xt.includes(k) && n.add(k);
|
|
658
|
+
const a = Object.keys(s);
|
|
659
|
+
c.traverse((k) => {
|
|
660
|
+
if (k instanceof D.Bone) {
|
|
661
|
+
const L = a.find((v) => k.name.indexOf(s[v]) === 0);
|
|
662
|
+
L && t.push({
|
|
663
|
+
ref: k,
|
|
664
|
+
name: k.name,
|
|
665
|
+
normalizedName: L
|
|
630
666
|
});
|
|
631
667
|
}
|
|
632
668
|
});
|
|
633
|
-
const
|
|
634
|
-
let
|
|
669
|
+
const r = [], i = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map();
|
|
670
|
+
let m = !1, h = 0, b = 0, u = 0, E = 0;
|
|
671
|
+
function d(k) {
|
|
672
|
+
i.clear(), l.clear(), r.length = 0, b = k ?? 0, E = b > 0 ? 1 / b : 0, h = performance.now() / 1e3, m = !0, u = 0;
|
|
673
|
+
}
|
|
635
674
|
function y() {
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
L.push(v);
|
|
675
|
+
if (!m) return;
|
|
676
|
+
const k = performance.now() / 1e3 - h;
|
|
677
|
+
if (!(b > 0 && k - u < E)) {
|
|
678
|
+
u = k, r.push(k);
|
|
679
|
+
for (const L of t) {
|
|
680
|
+
i.has(L.name) || i.set(L.name, []);
|
|
681
|
+
const v = i.get(L.name), w = L.ref.quaternion;
|
|
682
|
+
v.push(w.x, w.y, w.z, w.w);
|
|
683
|
+
}
|
|
684
|
+
for (const L of n) {
|
|
685
|
+
l.has(L) || l.set(L, []);
|
|
686
|
+
const v = l.get(L), w = o.morphTargetDictionary[L], g = o.morphTargetInfluences[w];
|
|
687
|
+
v.push(g);
|
|
688
|
+
}
|
|
651
689
|
}
|
|
652
690
|
}
|
|
653
|
-
function
|
|
654
|
-
|
|
655
|
-
const
|
|
656
|
-
for (const [w,
|
|
657
|
-
|
|
658
|
-
new
|
|
691
|
+
function P(k = "RecordedClip") {
|
|
692
|
+
m = !1;
|
|
693
|
+
const L = [];
|
|
694
|
+
for (const [w, g] of i)
|
|
695
|
+
L.push(
|
|
696
|
+
new D.QuaternionKeyframeTrack(
|
|
659
697
|
`${w}.quaternion`,
|
|
660
|
-
|
|
661
|
-
|
|
698
|
+
r,
|
|
699
|
+
g
|
|
662
700
|
)
|
|
663
701
|
);
|
|
664
|
-
for (const [w,
|
|
665
|
-
|
|
666
|
-
new
|
|
667
|
-
`${
|
|
668
|
-
|
|
669
|
-
|
|
702
|
+
for (const [w, g] of l)
|
|
703
|
+
L.push(
|
|
704
|
+
new D.NumberKeyframeTrack(
|
|
705
|
+
`${Re.faceMesh}.morphTargetInfluences[${w}]`,
|
|
706
|
+
r,
|
|
707
|
+
g
|
|
670
708
|
)
|
|
671
709
|
);
|
|
672
|
-
const
|
|
710
|
+
const v = new D.AnimationClip(k, -1, L);
|
|
673
711
|
return {
|
|
674
|
-
clip:
|
|
675
|
-
saveToFile: () => {
|
|
676
|
-
new
|
|
677
|
-
|
|
678
|
-
(
|
|
679
|
-
const
|
|
712
|
+
clip: v,
|
|
713
|
+
saveToFile: () => new Promise((w, g) => {
|
|
714
|
+
new je().parse(
|
|
715
|
+
c,
|
|
716
|
+
(A) => {
|
|
717
|
+
const W = new Blob([A], {
|
|
680
718
|
type: "model/gltf-binary"
|
|
681
|
-
}),
|
|
682
|
-
|
|
719
|
+
}), V = URL.createObjectURL(W), M = document.createElement("a");
|
|
720
|
+
M.href = V, M.download = k + ".glb", M.click(), w(!0);
|
|
683
721
|
},
|
|
684
|
-
(
|
|
685
|
-
console.error(
|
|
722
|
+
(A) => {
|
|
723
|
+
console.error(A), g(A);
|
|
686
724
|
},
|
|
687
725
|
{
|
|
688
726
|
binary: !0,
|
|
689
|
-
animations: [
|
|
727
|
+
animations: [v]
|
|
690
728
|
}
|
|
691
729
|
);
|
|
692
|
-
}
|
|
730
|
+
})
|
|
693
731
|
};
|
|
694
732
|
}
|
|
695
|
-
return { start:
|
|
733
|
+
return { start: d, captureFrame: y, stop: P, isRecording: () => m };
|
|
696
734
|
}
|
|
697
|
-
const
|
|
698
|
-
var
|
|
699
|
-
return !!((
|
|
735
|
+
const Rt = () => {
|
|
736
|
+
var c;
|
|
737
|
+
return !!((c = navigator.mediaDevices) != null && c.getUserMedia);
|
|
700
738
|
};
|
|
701
|
-
async function
|
|
702
|
-
const
|
|
739
|
+
async function St(c) {
|
|
740
|
+
const s = {
|
|
703
741
|
debugFrame: void 0,
|
|
704
742
|
displayScale: 1,
|
|
705
743
|
ignoreLegs: !1,
|
|
@@ -713,94 +751,94 @@ async function Pt(h) {
|
|
|
713
751
|
hand: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
|
|
714
752
|
face: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task"
|
|
715
753
|
},
|
|
716
|
-
...
|
|
754
|
+
...c
|
|
717
755
|
};
|
|
718
756
|
let e;
|
|
719
|
-
const t = await
|
|
720
|
-
ignoreLegs:
|
|
721
|
-
modelPath:
|
|
722
|
-
drawLandmarks:
|
|
723
|
-
}), n =
|
|
724
|
-
leftWrist: () =>
|
|
725
|
-
rightWrist: () =>
|
|
726
|
-
modelPath:
|
|
727
|
-
drawLandmarks:
|
|
728
|
-
...
|
|
729
|
-
}),
|
|
730
|
-
|
|
731
|
-
const
|
|
732
|
-
|
|
733
|
-
function
|
|
734
|
-
|
|
735
|
-
}
|
|
736
|
-
function
|
|
737
|
-
e = document.createElement("video"),
|
|
738
|
-
let
|
|
739
|
-
e.style.zIndex = "21", e.style.position = "absolute", e.style.top = "0px", e.style.left = "0px", e.style.height = "auto", e.style.maxWidth = "100%", e.style.display = "block", e.muted = !1,
|
|
740
|
-
function
|
|
741
|
-
|
|
757
|
+
const t = await Se.forVisionTasks(s.modelPaths.vision ?? "/wasm"), o = s.onlyFace ? void 0 : await ot(t, {
|
|
758
|
+
ignoreLegs: s.ignoreLegs,
|
|
759
|
+
modelPath: s.modelPaths.pose,
|
|
760
|
+
drawLandmarks: s.drawLandmarksOverlay
|
|
761
|
+
}), n = s.onlyFace ? void 0 : await ct(t, {
|
|
762
|
+
leftWrist: () => o.leftWristNormalizedPosition,
|
|
763
|
+
rightWrist: () => o.rightWristNormalizedPosition,
|
|
764
|
+
modelPath: s.modelPaths.hand,
|
|
765
|
+
drawLandmarks: s.drawLandmarksOverlay,
|
|
766
|
+
...c == null ? void 0 : c.handsTrackerOptions
|
|
767
|
+
}), a = s.ignoreFace ? void 0 : await pt(t, { modelPath: s.modelPaths.face, videoElementRef: () => e, drawLandmarks: s.drawLandmarksOverlay }), r = document.createElement("div");
|
|
768
|
+
r.style.position = "absolute", r.style.top = "0px", r.style.left = "0px", r.style.zIndex = "21", r.style.maxWidth = "600px", r.style.width = "100%", r.classList.add("three-mediapipe-rig"), document.body.appendChild(r);
|
|
769
|
+
const i = document.createElement("canvas"), l = i.getContext("2d"), m = new Ve(l);
|
|
770
|
+
i.style.zIndex = "22", i.style.position = "absolute", i.style.top = "0px", i.style.left = "0px", i.style.pointerEvents = "none", i.style.maxWidth = "100%", r.appendChild(i);
|
|
771
|
+
function h(d) {
|
|
772
|
+
l.save(), l.clearRect(0, 0, i.width, i.height), o == null || o.predict(d, m), n == null || n.left.predict(d, m), n == null || n.right.predict(d, m), a == null || a.predict(d, m), l.restore();
|
|
773
|
+
}
|
|
774
|
+
function b() {
|
|
775
|
+
e = document.createElement("video"), r.appendChild(e);
|
|
776
|
+
let d = -1;
|
|
777
|
+
e.style.zIndex = "21", e.style.position = "absolute", e.style.top = "0px", e.style.left = "0px", e.style.height = "auto", e.style.maxWidth = "100%", e.style.display = "block", e.muted = !1, s.debugVideo && (e.src = s.debugVideo, e.controls = !0, e.loop = !0, e.muted = !0, e.controls = !0, e.play());
|
|
778
|
+
function y() {
|
|
779
|
+
d !== e.currentTime && e.readyState >= 2 && (h(e), d = e.currentTime), window.requestAnimationFrame(y);
|
|
742
780
|
}
|
|
743
781
|
e.addEventListener("loadeddata", () => {
|
|
744
|
-
const
|
|
745
|
-
e.width =
|
|
782
|
+
const P = e.videoWidth, k = e.videoHeight, w = Math.min(600 / P, 600 / k, 1);
|
|
783
|
+
e.width = P * w * (s.displayScale ?? 1), e.height = k * w * (s.displayScale ?? 1), i.width = e.width, i.height = e.height, i.style.height = "auto", i.style.width = e.width + "px", i.style.maxWidth = "100%", window.requestAnimationFrame(y);
|
|
746
784
|
});
|
|
747
785
|
}
|
|
748
|
-
if (
|
|
749
|
-
const
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
function
|
|
753
|
-
|
|
786
|
+
if (s.debugFrame) {
|
|
787
|
+
const d = document.createElement("img");
|
|
788
|
+
d.src = s.debugFrame, d.style.zIndex = "21", d.style.position = "absolute", d.style.top = "0px", d.style.left = "0px", r.appendChild(d), d.addEventListener("load", () => {
|
|
789
|
+
d.width = d.naturalWidth * s.displayScale, d.height = d.naturalHeight * s.displayScale, i.width = d.naturalWidth, i.height = d.naturalWidth, i.style.width = d.width + "px", i.style.height = d.height + "px";
|
|
790
|
+
function y() {
|
|
791
|
+
h(d);
|
|
754
792
|
}
|
|
755
|
-
window.requestAnimationFrame(
|
|
793
|
+
window.requestAnimationFrame(y);
|
|
756
794
|
});
|
|
757
|
-
} else
|
|
758
|
-
let
|
|
759
|
-
function
|
|
760
|
-
e && (e.srcObject ? e.srcObject.getTracks().forEach((
|
|
795
|
+
} else s.debugVideo && b();
|
|
796
|
+
let u;
|
|
797
|
+
function E(d) {
|
|
798
|
+
e && (e.srcObject ? e.srcObject.getTracks().forEach((P) => P.enabled = d) : d ? e.play() : e.pause());
|
|
761
799
|
}
|
|
762
800
|
return {
|
|
763
|
-
poseTracker:
|
|
801
|
+
poseTracker: o,
|
|
764
802
|
handsTracker: n,
|
|
765
|
-
faceTracker:
|
|
803
|
+
faceTracker: a,
|
|
766
804
|
video: e,
|
|
767
|
-
canvas:
|
|
768
|
-
domElement:
|
|
769
|
-
start: async (
|
|
770
|
-
let
|
|
771
|
-
if (!
|
|
805
|
+
canvas: i,
|
|
806
|
+
domElement: r,
|
|
807
|
+
start: async (d = !1) => {
|
|
808
|
+
let y = !1;
|
|
809
|
+
if (!Rt())
|
|
772
810
|
throw new Error("Webcam not supported");
|
|
773
|
-
e ||
|
|
774
|
-
let
|
|
775
|
-
function
|
|
776
|
-
|
|
811
|
+
e || b();
|
|
812
|
+
let P;
|
|
813
|
+
function k(f) {
|
|
814
|
+
y || (console.warn("Camera track ended, attempting recovery..."), L(f), v(f));
|
|
777
815
|
}
|
|
778
|
-
function
|
|
779
|
-
|
|
816
|
+
function L(f) {
|
|
817
|
+
P == null || P.getVideoTracks().forEach((A) => A.stop()), P = void 0, f.srcObject = null;
|
|
780
818
|
}
|
|
781
|
-
async function
|
|
782
|
-
const
|
|
783
|
-
if (
|
|
819
|
+
async function v(f, A = 0) {
|
|
820
|
+
const V = Math.min(1e3 * 2 ** A, 16e3);
|
|
821
|
+
if (A >= 3)
|
|
784
822
|
throw new Error("Camera recovery failed after max attempts");
|
|
785
|
-
if (await new Promise((M) => setTimeout(M,
|
|
823
|
+
if (await new Promise((M) => setTimeout(M, V)), !y)
|
|
786
824
|
try {
|
|
787
|
-
await
|
|
825
|
+
await w(f);
|
|
788
826
|
} catch (M) {
|
|
789
|
-
await
|
|
827
|
+
await g(M, f, A + 1);
|
|
790
828
|
}
|
|
791
829
|
}
|
|
792
|
-
async function
|
|
793
|
-
|
|
794
|
-
|
|
830
|
+
async function w(f) {
|
|
831
|
+
P = await navigator.mediaDevices.getUserMedia({ video: !0, audio: d }), f.srcObject = P, await f.play(), P.getVideoTracks().forEach((A) => {
|
|
832
|
+
A.addEventListener("ended", () => k(f));
|
|
795
833
|
});
|
|
796
834
|
}
|
|
797
|
-
async function
|
|
835
|
+
async function g(f, A, W = 0) {
|
|
798
836
|
if (f instanceof DOMException)
|
|
799
837
|
switch (f.name) {
|
|
800
838
|
case "NotAllowedError":
|
|
801
839
|
throw new Error("Permission denied — prompt user to allow camera");
|
|
802
840
|
case "NotFoundError":
|
|
803
|
-
console.error("No camera found — retry when device is connected"), await
|
|
841
|
+
console.error("No camera found — retry when device is connected"), await v(A, W + 1);
|
|
804
842
|
break;
|
|
805
843
|
case "NotReadableError":
|
|
806
844
|
throw new Error("Camera in use by another app");
|
|
@@ -810,55 +848,54 @@ async function Pt(h) {
|
|
|
810
848
|
else
|
|
811
849
|
throw new Error("Unknown camera error: " + f);
|
|
812
850
|
}
|
|
813
|
-
return await
|
|
814
|
-
|
|
851
|
+
return await v(e), u = () => {
|
|
852
|
+
y = !0, L(e), u = void 0;
|
|
815
853
|
}, {
|
|
816
|
-
getStream: () =>
|
|
817
|
-
stop:
|
|
854
|
+
getStream: () => P,
|
|
855
|
+
stop: u
|
|
818
856
|
};
|
|
819
857
|
},
|
|
820
858
|
pause: () => {
|
|
821
|
-
|
|
859
|
+
E(!1);
|
|
822
860
|
},
|
|
823
861
|
resume: () => {
|
|
824
|
-
|
|
862
|
+
E(!0);
|
|
825
863
|
},
|
|
826
|
-
bind: (
|
|
827
|
-
if (
|
|
864
|
+
bind: (d, y) => {
|
|
865
|
+
if (y = y || Re, !o) throw new Error("Pose tracker not initialized");
|
|
828
866
|
if (!n) throw new Error("Hands tracker not initialized");
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
f instanceof p.Mesh && f.name.indexOf(c.faceMesh) === 0 && (f.frustumCulled = !1, E = d == null ? void 0 : d.bindShapeKeys(f));
|
|
867
|
+
const P = o.bind(d, y), k = n.left.bind(d, y), L = n.right.bind(d, y);
|
|
868
|
+
let v;
|
|
869
|
+
const w = a == null ? void 0 : a.bind(d);
|
|
870
|
+
d.traverse((f) => {
|
|
871
|
+
f instanceof p.Mesh && f.name.indexOf(y.faceMesh) === 0 && (f.frustumCulled = !1, v = a == null ? void 0 : a.bindShapeKeys(f));
|
|
835
872
|
});
|
|
836
|
-
const
|
|
873
|
+
const g = bt(d, y);
|
|
837
874
|
return {
|
|
838
875
|
/**
|
|
839
876
|
* Will save the tracked movement of the rig to an animation clip.
|
|
840
877
|
* Only the bones moved by the bone mapping will be recorded.
|
|
841
878
|
*/
|
|
842
|
-
startRecording:
|
|
843
|
-
stopRecording:
|
|
844
|
-
isRecording: () =>
|
|
879
|
+
startRecording: g.start,
|
|
880
|
+
stopRecording: g.stop,
|
|
881
|
+
isRecording: () => g.isRecording(),
|
|
845
882
|
update: (f) => {
|
|
846
|
-
|
|
883
|
+
P.update(f), k.update(f), L.update(f), v == null || v.update(f), w == null || w.update(f), g.isRecording() && g.captureFrame();
|
|
847
884
|
}
|
|
848
885
|
};
|
|
849
886
|
},
|
|
850
|
-
setVideoFromSource: (
|
|
851
|
-
|
|
887
|
+
setVideoFromSource: (d) => {
|
|
888
|
+
u == null || u(), e || b(), e.src = d instanceof File ? URL.createObjectURL(d) : d, e.controls = !0, e.loop = !0, e.muted = !0, e.controls = !0, e.play();
|
|
852
889
|
},
|
|
853
|
-
async setVideoFromWebcam(
|
|
854
|
-
if (e ||
|
|
890
|
+
async setVideoFromWebcam(d = !1) {
|
|
891
|
+
if (e || b(), u)
|
|
855
892
|
throw new Error("Webcam already started");
|
|
856
|
-
return await this.start(
|
|
893
|
+
return await this.start(d);
|
|
857
894
|
}
|
|
858
895
|
};
|
|
859
896
|
}
|
|
860
897
|
export {
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
898
|
+
ee as FACE_LANDMARKS_COUNT,
|
|
899
|
+
qe as createFaceLandmarksIndexAttribute,
|
|
900
|
+
St as setupTracker
|
|
864
901
|
};
|