three-mediapipe-rig 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -4
- package/dist/meshcap/atlas-builder.d.ts +9 -0
- package/dist/meshcap/atlas-builder.d.ts.map +1 -1
- package/dist/meshcap/audio.d.ts +27 -0
- package/dist/meshcap/audio.d.ts.map +1 -0
- package/dist/meshcap/constants.d.ts +8 -1
- package/dist/meshcap/constants.d.ts.map +1 -1
- package/dist/meshcap/material.d.ts +40 -6
- package/dist/meshcap/material.d.ts.map +1 -1
- package/dist/meshcap/meshcap.d.ts +1 -0
- package/dist/meshcap/meshcap.d.ts.map +1 -1
- package/dist/meshcap/parse-mcap-file.d.ts +2 -1
- package/dist/meshcap/parse-mcap-file.d.ts.map +1 -1
- package/dist/meshcap/types.d.ts +59 -9
- package/dist/meshcap/types.d.ts.map +1 -1
- package/dist/meshcap/write-mcap-file.d.ts +13 -0
- package/dist/meshcap/write-mcap-file.d.ts.map +1 -1
- package/dist/meshcap.js +527 -271
- package/dist/rigger.js +440 -392
- package/dist/tracking/FaceTracker.d.ts +9 -2
- package/dist/tracking/FaceTracker.d.ts.map +1 -1
- package/dist/tracking/PoseTracker.d.ts +3 -0
- package/dist/tracking/PoseTracker.d.ts.map +1 -1
- package/dist/tracking/TrackerManager.d.ts +16 -7
- package/dist/tracking/TrackerManager.d.ts.map +1 -1
- package/dist/tracking/recoding/recorder.d.ts +1 -1
- package/dist/tracking/recoding/recorder.d.ts.map +1 -1
- package/package.json +11 -10
package/dist/rigger.js
CHANGED
|
@@ -1,45 +1,45 @@
|
|
|
1
|
-
var
|
|
2
|
-
var
|
|
3
|
-
var R = (
|
|
4
|
-
import { PoseLandmarker as
|
|
5
|
-
import * as
|
|
6
|
-
import { Vector3 as
|
|
7
|
-
import * as
|
|
8
|
-
import { Bone as
|
|
9
|
-
import { uniform as
|
|
10
|
-
import { c as
|
|
11
|
-
import { GLTFExporter as
|
|
12
|
-
const
|
|
13
|
-
new
|
|
14
|
-
const
|
|
15
|
-
function
|
|
16
|
-
|
|
17
|
-
const
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
|
|
1
|
+
var Ae = Object.defineProperty;
|
|
2
|
+
var Me = (c, s, e) => s in c ? Ae(c, s, { enumerable: !0, configurable: !0, writable: !0, value: e }) : c[s] = e;
|
|
3
|
+
var R = (c, s, e) => Me(c, typeof s != "symbol" ? s + "" : s, e);
|
|
4
|
+
import { PoseLandmarker as ke, HandLandmarker as G, FaceLandmarker as q, FilesetResolver as Se, DrawingUtils as Ve } from "@mediapipe/tasks-vision";
|
|
5
|
+
import * as p from "three/webgpu";
|
|
6
|
+
import { Vector3 as x, Quaternion as xe, Matrix4 as Ne, VideoTexture as Z, SRGBColorSpace as J, MeshPhysicalNodeMaterial as Oe } from "three/webgpu";
|
|
7
|
+
import * as D from "three";
|
|
8
|
+
import { Bone as Te } from "three";
|
|
9
|
+
import { uniform as z, attribute as _e, instancedArray as Ce, varying as De, vec3 as ze, float as Be, select as Ue, positionLocal as Ie, texture as He, vec2 as Ge } from "three/tsl";
|
|
10
|
+
import { c as qe, F as ee } from "./face-tracker-utils-xt9__vBF.js";
|
|
11
|
+
import { GLTFExporter as je } from "three/examples/jsm/Addons.js";
|
|
12
|
+
const U = new x(), te = new x(), Ke = new x(1, 0, 0), $e = new x(-1, 0, 0), Xe = new x(0, 1, 0), Qe = new x(0, -1, 0), oe = new x(0, 0, 1);
|
|
13
|
+
new x(0, 0, -1);
|
|
14
|
+
const se = new x(), Ye = new x(), Ze = new x(), ne = new xe(), I = new xe();
|
|
15
|
+
function N(c, s, e, t = "+x") {
|
|
16
|
+
c.lookAt(s);
|
|
17
|
+
const o = t == "+x" ? Ke : t == "-x" ? $e : t == "+y" ? Xe : Qe;
|
|
18
|
+
c.getWorldPosition(te), c.getWorldQuaternion(I), U.subVectors(e, te).normalize(), se.copy(o).applyQuaternion(I);
|
|
19
|
+
const n = se, a = Ye.copy(oe).applyQuaternion(I), r = U.clone().addScaledVector(a, -U.dot(a)).normalize(), i = Ze.crossVectors(n, r), l = Math.atan2(i.dot(a), n.dot(r));
|
|
20
|
+
ne.setFromAxisAngle(oe, l), c.quaternion.multiply(ne);
|
|
21
21
|
}
|
|
22
|
-
const
|
|
23
|
-
new
|
|
24
|
-
class
|
|
25
|
-
constructor(
|
|
22
|
+
const B = new p.Vector3(), _ = new p.Vector3(), H = new p.Vector3();
|
|
23
|
+
new p.Vector3();
|
|
24
|
+
class j {
|
|
25
|
+
constructor(s, e) {
|
|
26
26
|
R(this, "objectGhost");
|
|
27
27
|
R(this, "root");
|
|
28
28
|
/**
|
|
29
29
|
* per landmark index, it points to it's object3D equivalent.
|
|
30
30
|
*/
|
|
31
31
|
R(this, "marks", {});
|
|
32
|
-
this.points =
|
|
32
|
+
this.points = s, this.debugConnections = e, this.root = new p.Object3D(), this.objectGhost = /* @__PURE__ */ new Map();
|
|
33
33
|
for (let t in this.points)
|
|
34
|
-
this.marks[t] = new
|
|
35
|
-
}
|
|
36
|
-
updateLandmarks(
|
|
37
|
-
for (let
|
|
38
|
-
const
|
|
39
|
-
a && (
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
).divideScalar(2).add(
|
|
34
|
+
this.marks[t] = new Je(), this.root.add(this.marks[t]);
|
|
35
|
+
}
|
|
36
|
+
updateLandmarks(s, e, t) {
|
|
37
|
+
for (let o in this.points) {
|
|
38
|
+
const n = this.points[o], a = this.marks[o];
|
|
39
|
+
a && (n instanceof Array ? (B.copy(s[n[0]]), a.position.copy(s[n[1]]).sub(B).divideScalar(2).add(s[n[0]]), n.length == 4 && (B.subVectors(
|
|
40
|
+
s[n[3]],
|
|
41
|
+
s[n[2]]
|
|
42
|
+
).divideScalar(2).add(s[n[2]]).sub(a.position).divideScalar(2), a.position.add(B))) : a.position.copy(s[n]));
|
|
43
43
|
}
|
|
44
44
|
t && e && t.drawConnectors(
|
|
45
45
|
e,
|
|
@@ -49,67 +49,67 @@ class H {
|
|
|
49
49
|
}
|
|
50
50
|
);
|
|
51
51
|
}
|
|
52
|
-
getGhost(
|
|
52
|
+
getGhost(s) {
|
|
53
53
|
var e;
|
|
54
|
-
if (!this.objectGhost.has(
|
|
55
|
-
const t = new
|
|
56
|
-
t.position.copy(
|
|
54
|
+
if (!this.objectGhost.has(s)) {
|
|
55
|
+
const t = new et();
|
|
56
|
+
t.position.copy(s.position), t.quaternion.copy(s.quaternion), (e = s.parent) == null || e.add(t), this.objectGhost.set(s, t);
|
|
57
57
|
}
|
|
58
|
-
return this.objectGhost.get(
|
|
58
|
+
return this.objectGhost.get(s);
|
|
59
59
|
}
|
|
60
|
-
predict(
|
|
60
|
+
predict(s, e) {
|
|
61
61
|
throw new Error("Method 'predict' must be implemented.");
|
|
62
62
|
}
|
|
63
|
-
sync(
|
|
63
|
+
sync(s, e) {
|
|
64
64
|
throw new Error("Method 'sync' must be implemented.");
|
|
65
65
|
}
|
|
66
|
-
test(
|
|
67
|
-
this.marks[
|
|
66
|
+
test(s) {
|
|
67
|
+
this.marks[s].position.set(1, 2, 3);
|
|
68
68
|
}
|
|
69
|
-
syncObjects(
|
|
70
|
-
for (const [
|
|
71
|
-
this.marks[a].getWorldPosition(
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
const
|
|
75
|
-
|
|
69
|
+
syncObjects(s, e, t) {
|
|
70
|
+
for (const [o, n, a, r] of s) {
|
|
71
|
+
this.marks[a].getWorldPosition(H), this.marks[n].getWorldPosition(_);
|
|
72
|
+
const i = H.sub(_);
|
|
73
|
+
o.getWorldPosition(_), _.add(i);
|
|
74
|
+
const l = _, m = o.getWorldPosition(H).sub(t), h = this.getGhost(o);
|
|
75
|
+
N(h, l, m, r), h.rotateX(Math.PI / 2), o.position.lerp(h.position, e * 4), o.quaternion.slerp(h.quaternion, e * 4);
|
|
76
76
|
}
|
|
77
77
|
}
|
|
78
|
-
getBone(
|
|
79
|
-
return
|
|
78
|
+
getBone(s, e) {
|
|
79
|
+
return s.getObjectByName(e.replace(/[\.\:]/g, ""));
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
|
-
class
|
|
82
|
+
class Je extends p.Mesh {
|
|
83
83
|
constructor() {
|
|
84
|
-
super(new
|
|
85
|
-
R(this, "_worldPosition", new
|
|
86
|
-
this.add(new
|
|
84
|
+
super(new p.SphereGeometry(0.01, 3, 3), new p.MeshStandardMaterial({ color: 16711680, wireframe: !0 }));
|
|
85
|
+
R(this, "_worldPosition", new p.Vector3());
|
|
86
|
+
this.add(new p.AxesHelper(1e-3));
|
|
87
87
|
}
|
|
88
88
|
get worldPosition() {
|
|
89
89
|
return this.getWorldPosition(this._worldPosition), this._worldPosition;
|
|
90
90
|
}
|
|
91
91
|
}
|
|
92
|
-
class
|
|
93
|
-
lerp(
|
|
94
|
-
|
|
92
|
+
class et extends p.Object3D {
|
|
93
|
+
lerp(s, e, t = 8) {
|
|
94
|
+
s.position.lerp(this.position, e * t), s.quaternion.slerp(this.quaternion, e * t);
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
|
-
function
|
|
98
|
-
return e.worldToLocal(
|
|
97
|
+
function S(c, s, e) {
|
|
98
|
+
return e.worldToLocal(s.getWorldPosition(c)), c;
|
|
99
99
|
}
|
|
100
|
-
function
|
|
101
|
-
return
|
|
100
|
+
function tt(c) {
|
|
101
|
+
return c.replace(/[\.\:]/g, "");
|
|
102
102
|
}
|
|
103
|
-
function
|
|
103
|
+
function F(c, s) {
|
|
104
104
|
let e;
|
|
105
|
-
return
|
|
106
|
-
t.name.indexOf(
|
|
107
|
-
}), e || console.log("Bone not found: ",
|
|
105
|
+
return s = tt(s), c.traverse((t) => {
|
|
106
|
+
t.name.indexOf(s) === 0 && t instanceof Te && (e = t);
|
|
107
|
+
}), e || console.log("Bone not found: ", s, c.name), e;
|
|
108
108
|
}
|
|
109
|
-
async function
|
|
110
|
-
const e = await
|
|
109
|
+
async function ot(c, s) {
|
|
110
|
+
const e = await ke.createFromOptions(c, {
|
|
111
111
|
baseOptions: {
|
|
112
|
-
modelAssetPath: (
|
|
112
|
+
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "pose_landmarker_lite.task",
|
|
113
113
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task`,
|
|
114
114
|
//modelAssetPath: "https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_heavy/float16/latest/pose_landmarker_heavy.task",
|
|
115
115
|
delegate: "GPU"
|
|
@@ -117,9 +117,9 @@ async function Xe(l, o) {
|
|
|
117
117
|
runningMode: "VIDEO",
|
|
118
118
|
numPoses: 1
|
|
119
119
|
});
|
|
120
|
-
return new
|
|
120
|
+
return new dt(e, s);
|
|
121
121
|
}
|
|
122
|
-
const
|
|
122
|
+
const st = {
|
|
123
123
|
hips: [24, 23],
|
|
124
124
|
neck: [12, 11],
|
|
125
125
|
leftLeg: 23,
|
|
@@ -137,17 +137,20 @@ const Qe = {
|
|
|
137
137
|
rightElbow: 14,
|
|
138
138
|
rightWrist: 16,
|
|
139
139
|
head: [8, 7],
|
|
140
|
+
forehead: [5, 2],
|
|
140
141
|
mouth: [10, 9],
|
|
141
142
|
torso: [24, 23, 12, 11],
|
|
142
143
|
leftEar: 7,
|
|
143
|
-
rightEar: 8
|
|
144
|
-
|
|
145
|
-
|
|
144
|
+
rightEar: 8,
|
|
145
|
+
nose: 0
|
|
146
|
+
}, nt = new p.Vector3(), ie = new p.Vector3(), it = new p.Vector3(), rt = new p.Vector3(), at = new p.Vector3(), re = new p.Vector3(), ae = new p.Vector3();
|
|
147
|
+
class dt extends j {
|
|
146
148
|
constructor(e, t) {
|
|
147
|
-
super(
|
|
149
|
+
super(st, ke.POSE_CONNECTIONS);
|
|
148
150
|
R(this, "_leftWristNormalizedPosition");
|
|
149
151
|
R(this, "_rightWristNormalizedPosition");
|
|
150
|
-
this
|
|
152
|
+
R(this, "ignoreLegs", !1);
|
|
153
|
+
this.poseLandmarker = e, this.config = t, this.root.scale.y *= -2, this.root.scale.z *= -2, this.root.scale.x *= 2, this.ignoreLegs = (t == null ? void 0 : t.ignoreLegs) ?? !1;
|
|
151
154
|
}
|
|
152
155
|
/**
|
|
153
156
|
* Position of the left wrist in normalized coordinates (0..1)
|
|
@@ -162,9 +165,9 @@ class tt extends H {
|
|
|
162
165
|
return this._rightWristNormalizedPosition;
|
|
163
166
|
}
|
|
164
167
|
predict(e, t) {
|
|
165
|
-
this.poseLandmarker.detectForVideo(e, performance.now(), (
|
|
166
|
-
var
|
|
167
|
-
|
|
168
|
+
this.poseLandmarker.detectForVideo(e, performance.now(), (o) => {
|
|
169
|
+
var n;
|
|
170
|
+
o.landmarks.length != 0 && (this.updateLandmarks(o.worldLandmarks[0], ((n = this.config) == null ? void 0 : n.drawLandmarks) === !1 ? void 0 : o.landmarks[0], t), this._leftWristNormalizedPosition = o.landmarks[0][this.points.leftWrist], this._rightWristNormalizedPosition = o.landmarks[0][this.points.rightWrist]);
|
|
168
171
|
});
|
|
169
172
|
}
|
|
170
173
|
// override sync ( delta:number, objects: BoneBinding[] ) {
|
|
@@ -175,65 +178,65 @@ class tt extends H {
|
|
|
175
178
|
// this.syncObjects(objects, delta, torsoNormal);
|
|
176
179
|
// }
|
|
177
180
|
bind(e, t) {
|
|
178
|
-
var
|
|
179
|
-
const
|
|
180
|
-
hips:
|
|
181
|
-
neck:
|
|
182
|
-
leftArm:
|
|
183
|
-
leftElbow:
|
|
184
|
-
leftWrist:
|
|
185
|
-
rightArm:
|
|
186
|
-
rightElbow:
|
|
187
|
-
rightWrist:
|
|
188
|
-
head:
|
|
189
|
-
torso:
|
|
190
|
-
leftLeg:
|
|
191
|
-
leftKnee:
|
|
192
|
-
leftFoot:
|
|
193
|
-
rightLeg:
|
|
194
|
-
rightKnee:
|
|
195
|
-
rightFoot:
|
|
181
|
+
var i;
|
|
182
|
+
const o = {
|
|
183
|
+
hips: F(e, t.hips),
|
|
184
|
+
neck: F(e, t.neck),
|
|
185
|
+
leftArm: F(e, t.armL),
|
|
186
|
+
leftElbow: F(e, t.forearmL),
|
|
187
|
+
leftWrist: F(e, t.handL),
|
|
188
|
+
rightArm: F(e, t.armR),
|
|
189
|
+
rightElbow: F(e, t.forearmR),
|
|
190
|
+
rightWrist: F(e, t.handR),
|
|
191
|
+
head: F(e, t.head),
|
|
192
|
+
torso: F(e, t.torso),
|
|
193
|
+
leftLeg: F(e, t.thighL),
|
|
194
|
+
leftKnee: F(e, t.shinL),
|
|
195
|
+
leftFoot: F(e, t.footL),
|
|
196
|
+
rightLeg: F(e, t.thighR),
|
|
197
|
+
rightKnee: F(e, t.shinR),
|
|
198
|
+
rightFoot: F(e, t.footR)
|
|
196
199
|
};
|
|
197
|
-
(
|
|
198
|
-
const
|
|
199
|
-
if (!
|
|
200
|
-
const
|
|
201
|
-
|
|
202
|
-
const
|
|
203
|
-
|
|
200
|
+
(i = this.config) != null && i.ignoreLegs && (delete o.leftLeg, delete o.leftKnee, delete o.leftFoot, delete o.leftToes, delete o.rightLeg, delete o.rightKnee, delete o.rightFoot, delete o.rightToes);
|
|
201
|
+
const n = new p.Vector3(), a = new p.Vector3(), r = (l, m, h, b, u, E) => {
|
|
202
|
+
if (!m) return;
|
|
203
|
+
const d = this.marks[b].getWorldPosition(n).sub(this.marks[h].getWorldPosition(a)).normalize();
|
|
204
|
+
S(re, m, e).add(d).applyMatrix4(e.matrixWorld), S(ae, m, e).add(u).applyMatrix4(e.matrixWorld);
|
|
205
|
+
const y = this.getGhost(m);
|
|
206
|
+
N(y, re, ae, E), y.rotateX(Math.PI / 2), y.lerp(m, l);
|
|
204
207
|
};
|
|
205
208
|
return {
|
|
206
|
-
update: (
|
|
207
|
-
const
|
|
208
|
-
|
|
209
|
+
update: (l) => {
|
|
210
|
+
const m = this.marks.leftLeg.getWorldPosition(nt).sub(this.marks.rightLeg.getWorldPosition(ie)).normalize(), h = this.marks.leftArm.getWorldPosition(ie).sub(this.marks.rightArm.getWorldPosition(it)).normalize(), b = this.marks.leftEar.getWorldPosition(rt).sub(this.marks.rightEar.getWorldPosition(at)).normalize();
|
|
211
|
+
r(l, o.hips, "hips", "torso", m, "+x"), r(l, o.torso, "torso", "neck", h, "+x"), r(l, o.neck, "neck", "head", b, "+x"), r(l, o.head, "nose", "forehead", b, "+x"), r(l, o.leftArm, "leftArm", "leftElbow", h, "-x"), r(l, o.leftElbow, "leftElbow", "leftWrist", h, "-x"), r(l, o.rightArm, "rightArm", "rightElbow", h, "-x"), r(l, o.rightElbow, "rightElbow", "rightWrist", h, "-x"), !this.ignoreLegs && (r(l, o.leftLeg, "leftLeg", "leftKnee", m, "+x"), r(l, o.leftKnee, "leftKnee", "leftFoot", m, "+x"), r(l, o.leftFoot, "leftFoot", "leftToes", m, "+x"), r(l, o.rightLeg, "rightLeg", "rightKnee", m, "+x"), r(l, o.rightKnee, "rightKnee", "rightFoot", m, "+x"), r(l, o.rightFoot, "rightFoot", "rightToes", m, "+x"));
|
|
209
212
|
}
|
|
210
213
|
};
|
|
211
214
|
}
|
|
212
215
|
}
|
|
213
|
-
const
|
|
214
|
-
async function
|
|
215
|
-
const e = await
|
|
216
|
+
const de = new p.Vector2(), ce = new p.Vector2();
|
|
217
|
+
async function ct(c, s) {
|
|
218
|
+
const e = await G.createFromOptions(c, {
|
|
216
219
|
baseOptions: {
|
|
217
|
-
modelAssetPath:
|
|
220
|
+
modelAssetPath: s.modelPath ?? "hand_landmarker.task",
|
|
218
221
|
//modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
|
|
219
222
|
delegate: "GPU"
|
|
220
223
|
},
|
|
221
224
|
runningMode: "VIDEO",
|
|
222
225
|
numHands: 2
|
|
223
|
-
}), t = (
|
|
226
|
+
}), t = (o, n, a) => {
|
|
224
227
|
try {
|
|
225
|
-
|
|
226
|
-
} catch (
|
|
227
|
-
return console.warn("No pose data... will just be optimitic and say yes to everything.",
|
|
228
|
+
de.copy(o()), ce.copy(n());
|
|
229
|
+
} catch (r) {
|
|
230
|
+
return console.warn("No pose data... will just be optimitic and say yes to everything.", r), !0;
|
|
228
231
|
}
|
|
229
|
-
return
|
|
232
|
+
return de.distanceTo(a) < ce.distanceTo(a);
|
|
230
233
|
};
|
|
231
234
|
return {
|
|
232
|
-
left: new
|
|
233
|
-
right: new
|
|
235
|
+
left: new ye(e, "Left", t.bind(null, s.leftWrist, s.rightWrist), s.drawLandmarks),
|
|
236
|
+
right: new ye(e, "Right", t.bind(null, s.rightWrist, s.leftWrist), s.drawLandmarks)
|
|
234
237
|
};
|
|
235
238
|
}
|
|
236
|
-
const
|
|
239
|
+
const lt = {
|
|
237
240
|
wrist: 0,
|
|
238
241
|
palm: [9, 13],
|
|
239
242
|
thumb1: 1,
|
|
@@ -256,35 +259,35 @@ const nt = {
|
|
|
256
259
|
pinky2: 18,
|
|
257
260
|
pinky3: 19,
|
|
258
261
|
pinky4: 20
|
|
259
|
-
},
|
|
262
|
+
}, C = {
|
|
260
263
|
thumb: ["thumb1", "thumb2", "thumb3", "thumb4"],
|
|
261
264
|
index: ["index1", "index2", "index3", "index4"],
|
|
262
265
|
middle: ["middle1", "middle2", "middle3", "middle4"],
|
|
263
266
|
ring: ["ring1", "ring2", "ring3", "ring4"],
|
|
264
267
|
pinky: ["pinky1", "pinky2", "pinky3", "pinky4"]
|
|
265
|
-
},
|
|
266
|
-
new
|
|
267
|
-
new
|
|
268
|
-
new
|
|
269
|
-
const
|
|
270
|
-
class
|
|
271
|
-
constructor(e, t,
|
|
272
|
-
super(
|
|
268
|
+
}, ht = new p.Vector3(), le = new p.Vector3(), he = new p.Vector3(), me = new p.Vector3(), pe = new p.Vector3(), ue = new p.Vector3();
|
|
269
|
+
new p.Vector3();
|
|
270
|
+
new p.Vector3();
|
|
271
|
+
new p.Vector3();
|
|
272
|
+
const fe = new p.Vector3(), we = Math.PI / 2, mt = new p.Vector3(0, -1, 0);
|
|
273
|
+
class ye extends j {
|
|
274
|
+
constructor(e, t, o, n = !0) {
|
|
275
|
+
super(lt, G.HAND_CONNECTIONS);
|
|
273
276
|
R(this, "sign");
|
|
274
277
|
R(this, "isLeft");
|
|
275
278
|
/**
|
|
276
279
|
* the axis used to look at the pole
|
|
277
280
|
*/
|
|
278
281
|
R(this, "lookAtPoleAxis");
|
|
279
|
-
this.handLandmarker = e, this.side = t, this.isMyWrist =
|
|
282
|
+
this.handLandmarker = e, this.side = t, this.isMyWrist = o, this.drawLandmarks = n, this.sign = this.side == "Left" ? -1 : 1, this.isLeft = this.side == "Left", this.lookAtPoleAxis = this.sign < 0 ? "+x" : "-x", this.root.scale.setScalar(7), this.root.scale.y *= -1, this.root.scale.z *= -1;
|
|
280
283
|
}
|
|
281
284
|
predict(e, t) {
|
|
282
|
-
const
|
|
283
|
-
if (
|
|
284
|
-
for (let
|
|
285
|
-
const a =
|
|
286
|
-
if (this.isMyWrist(
|
|
287
|
-
this.updateLandmarks(
|
|
285
|
+
const o = this.handLandmarker.detectForVideo(e, performance.now());
|
|
286
|
+
if (o.landmarks.length)
|
|
287
|
+
for (let n = 0; n < o.landmarks.length; n++) {
|
|
288
|
+
const a = o.landmarks[n], r = a[this.points.wrist];
|
|
289
|
+
if (this.isMyWrist(r)) {
|
|
290
|
+
this.updateLandmarks(o.worldLandmarks[n]), this.drawLandmarks && (t.drawConnectors(a, G.HAND_CONNECTIONS, {
|
|
288
291
|
color: this.side == "Left" ? "#00FF00" : "#0000FF",
|
|
289
292
|
lineWidth: 4
|
|
290
293
|
}), t.drawLandmarks(a, { color: this.side == "Left" ? "#00FF00" : "#0000FF", lineWidth: 3, radius: 1 }));
|
|
@@ -301,68 +304,69 @@ class ue extends H {
|
|
|
301
304
|
* @param landmark2bones Array the same size as the umber of hand landmarks, and on each positionthe bone that belongs to that point.
|
|
302
305
|
* @see https://ai.google.dev/edge/mediapipe/solutions/vision/hand_landmarker
|
|
303
306
|
*/
|
|
304
|
-
syncHandBones(e, t,
|
|
305
|
-
const
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
).normalize(), a =
|
|
309
|
-
if (!(a.dot(
|
|
307
|
+
syncHandBones(e, t, o) {
|
|
308
|
+
const n = ht.crossVectors(
|
|
309
|
+
le.copy(this.marks.index1.worldPosition).sub(this.marks.wrist.worldPosition),
|
|
310
|
+
he.copy(this.marks.pinky1.worldPosition).sub(this.marks.wrist.worldPosition)
|
|
311
|
+
).normalize(), a = le.copy(this.marks.palm.worldPosition).sub(this.marks.wrist.worldPosition).normalize(), r = he.copy(this.marks.pinky1.worldPosition).sub(this.marks.index1.worldPosition).normalize();
|
|
312
|
+
if (!(a.dot(mt) > 0.8)) {
|
|
310
313
|
if (t.wrist) {
|
|
311
|
-
const
|
|
312
|
-
|
|
314
|
+
const i = S(me, t.wrist, o).add(a).applyMatrix4(o.matrixWorld), l = S(pe, t.wrist, o).sub(r).applyMatrix4(o.matrixWorld), m = this.getGhost(t.wrist);
|
|
315
|
+
N(m, i, l, "-y"), m.rotateX(we), m.lerp(t.wrist, e);
|
|
313
316
|
}
|
|
314
|
-
this.syncFinger(e,
|
|
317
|
+
this.syncFinger(e, o, n, a, r, t, C.index, "middle1"), this.syncFinger(e, o, n, a, r, t, C.middle, "ring1"), this.syncFinger(e, o, n, a, r, t, C.ring, "pinky1"), this.syncFinger(e, o, n, a, r, t, C.pinky, "ring1", !0), this.syncFinger(e, o, n, a, r, t, C.thumb, "index1");
|
|
315
318
|
}
|
|
316
319
|
}
|
|
317
|
-
syncFinger(e, t,
|
|
318
|
-
for (let
|
|
319
|
-
const
|
|
320
|
-
if (!
|
|
321
|
-
const
|
|
322
|
-
if (
|
|
323
|
-
const
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
320
|
+
syncFinger(e, t, o, n, a, r, i, l, m = !1) {
|
|
321
|
+
for (let h = 0; h < i.length - 1; h++) {
|
|
322
|
+
const b = r[i[h]];
|
|
323
|
+
if (!b) continue;
|
|
324
|
+
const u = this.getGhost(b), E = me.copy(this.marks[i[h + 1]].worldPosition).sub(this.marks[i[h]].worldPosition).normalize(), d = S(pe, b, t);
|
|
325
|
+
if (h == 0) {
|
|
326
|
+
const y = ue.copy(this.marks[l].worldPosition).sub(this.marks[i[0]].worldPosition).normalize();
|
|
327
|
+
m && y.negate(), fe.copy(y), N(
|
|
328
|
+
u,
|
|
329
|
+
E.add(d).applyMatrix4(t.matrixWorld),
|
|
330
|
+
y.add(d).applyMatrix4(t.matrixWorld),
|
|
328
331
|
this.lookAtPoleAxis
|
|
329
332
|
);
|
|
330
333
|
} else
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
334
|
+
N(
|
|
335
|
+
u,
|
|
336
|
+
E.add(d).applyMatrix4(t.matrixWorld),
|
|
337
|
+
ue.copy(fe).add(d).applyMatrix4(t.matrixWorld),
|
|
335
338
|
this.lookAtPoleAxis
|
|
336
339
|
);
|
|
337
|
-
|
|
340
|
+
u.rotateX(we), u.lerp(b, e);
|
|
338
341
|
}
|
|
339
342
|
}
|
|
340
343
|
bind(e, t) {
|
|
341
|
-
const
|
|
342
|
-
const
|
|
343
|
-
if (
|
|
344
|
-
return
|
|
344
|
+
const o = {}, n = (a, r) => {
|
|
345
|
+
const i = F(e, a);
|
|
346
|
+
if (i)
|
|
347
|
+
return o[r] = i, r;
|
|
345
348
|
};
|
|
346
|
-
return
|
|
349
|
+
return n(this.isLeft ? t.handL : t.handR, "wrist"), n(this.isLeft ? t.index1L : t.index1R, "index1"), n(this.isLeft ? t.index2L : t.index2R, "index2"), n(this.isLeft ? t.index3L : t.index3R, "index3"), n(this.isLeft ? t.middle1L : t.middle1R, "middle1"), n(this.isLeft ? t.middle2L : t.middle2R, "middle2"), n(this.isLeft ? t.middle3L : t.middle3R, "middle3"), n(this.isLeft ? t.ring1L : t.ring1R, "ring1"), n(this.isLeft ? t.ring2L : t.ring2R, "ring2"), n(this.isLeft ? t.ring3L : t.ring3R, "ring3"), n(this.isLeft ? t.pinky1L : t.pinky1R, "pinky1"), n(this.isLeft ? t.pinky2L : t.pinky2R, "pinky2"), n(this.isLeft ? t.pinky3L : t.pinky3R, "pinky3"), n(this.isLeft ? t.thumb1L : t.thumb1R, "thumb1"), n(this.isLeft ? t.thumb2L : t.thumb2R, "thumb2"), n(this.isLeft ? t.thumb3L : t.thumb3R, "thumb3"), {
|
|
347
350
|
update: (a) => {
|
|
348
|
-
this.syncHandBones(a,
|
|
351
|
+
this.syncHandBones(a, o, e);
|
|
349
352
|
}
|
|
350
353
|
};
|
|
351
354
|
}
|
|
352
355
|
}
|
|
353
|
-
async function
|
|
354
|
-
const e = await
|
|
356
|
+
async function pt(c, s) {
|
|
357
|
+
const e = await q.createFromOptions(c, {
|
|
355
358
|
baseOptions: {
|
|
356
|
-
modelAssetPath: (
|
|
359
|
+
modelAssetPath: (s == null ? void 0 : s.modelPath) ?? "face_landmarker.task",
|
|
357
360
|
delegate: "GPU"
|
|
358
361
|
},
|
|
359
362
|
outputFaceBlendshapes: !0,
|
|
363
|
+
outputFacialTransformationMatrixes: !0,
|
|
360
364
|
runningMode: "VIDEO",
|
|
361
365
|
numFaces: 1
|
|
362
366
|
});
|
|
363
|
-
return new
|
|
367
|
+
return new kt(e, { ...s });
|
|
364
368
|
}
|
|
365
|
-
const
|
|
369
|
+
const ut = {
|
|
366
370
|
eyeL: 473,
|
|
367
371
|
eyeR: 468,
|
|
368
372
|
eyeStartL: 463,
|
|
@@ -375,51 +379,82 @@ const at = {
|
|
|
375
379
|
noseBone: 6,
|
|
376
380
|
chin: 152,
|
|
377
381
|
forehead: 10
|
|
378
|
-
},
|
|
379
|
-
new
|
|
380
|
-
class
|
|
382
|
+
}, ft = new x(), wt = new x(), be = new x(), yt = new x(), Lt = new x();
|
|
383
|
+
new x();
|
|
384
|
+
class kt extends j {
|
|
381
385
|
constructor(e, t) {
|
|
382
|
-
super(
|
|
386
|
+
super(ut, q.FACE_LANDMARKS_TESSELATION);
|
|
383
387
|
R(this, "blendshapeCategories");
|
|
384
388
|
R(this, "blendshapeMap", /* @__PURE__ */ new Map());
|
|
385
389
|
R(this, "smoothed", {});
|
|
386
390
|
R(this, "smoothing", 3e-4);
|
|
387
391
|
// lower = smoother but more lag, higher = more responsive
|
|
388
392
|
R(this, "_faceLandmarks", []);
|
|
393
|
+
R(this, "_facialTransformationMatrix");
|
|
389
394
|
this.faceLandmarker = e, this.cfg = t, this.root.scale.y *= -1, this.root.scale.z *= -1, this.root.scale.multiplyScalar(3);
|
|
390
395
|
}
|
|
391
396
|
predict(e, t) {
|
|
392
|
-
var
|
|
393
|
-
const
|
|
394
|
-
|
|
395
|
-
this.
|
|
397
|
+
var n, a, r;
|
|
398
|
+
const o = this.faceLandmarker.detectForVideo(e, performance.now());
|
|
399
|
+
if (o.faceLandmarks[0]) {
|
|
400
|
+
this.cfg.drawLandmarks && (t.drawConnectors(o.faceLandmarks[0], q.FACE_LANDMARKS_TESSELATION, { color: "#00fff2ff", lineWidth: 0.1 }), t.drawLandmarks(o.faceLandmarks[0], { color: "#00ff00", lineWidth: 0.1, radius: 0.4 })), this.updateLandmarks(o.faceLandmarks[0], o.faceLandmarks[0]), this._faceLandmarks = o.faceLandmarks[0], this._facialTransformationMatrix || (this._facialTransformationMatrix = new Ne());
|
|
401
|
+
const i = o.facialTransformationMatrixes[0];
|
|
402
|
+
this._facialTransformationMatrix.set(
|
|
403
|
+
i.data[0],
|
|
404
|
+
i.data[1],
|
|
405
|
+
i.data[2],
|
|
406
|
+
i.data[3],
|
|
407
|
+
i.data[4],
|
|
408
|
+
i.data[5],
|
|
409
|
+
i.data[6],
|
|
410
|
+
i.data[7],
|
|
411
|
+
i.data[8],
|
|
412
|
+
i.data[9],
|
|
413
|
+
i.data[10],
|
|
414
|
+
i.data[11],
|
|
415
|
+
i.data[12],
|
|
416
|
+
i.data[13],
|
|
417
|
+
i.data[14],
|
|
418
|
+
i.data[15]
|
|
419
|
+
);
|
|
420
|
+
}
|
|
421
|
+
this.blendshapeCategories = (a = (n = o.faceBlendshapes) == null ? void 0 : n[0]) == null ? void 0 : a.categories, (r = this.blendshapeCategories) == null || r.forEach((i) => {
|
|
422
|
+
this.blendshapeMap.set(i.categoryName, i.score);
|
|
396
423
|
});
|
|
397
424
|
}
|
|
398
425
|
get lastKnownLandmarks() {
|
|
399
426
|
return this._faceLandmarks;
|
|
400
427
|
}
|
|
428
|
+
/**
|
|
429
|
+
* this matrix is useful to know how the canonical mesh is rotated.
|
|
430
|
+
* This allows to move other objects to simulate the same transformation that the skull/face is having.
|
|
431
|
+
* Used for things like hats, glasses, etc... things you want to make look like they are attatched to the face mesh.
|
|
432
|
+
*/
|
|
433
|
+
get lastKnownFacialTransformationMatrix() {
|
|
434
|
+
return this._facialTransformationMatrix;
|
|
435
|
+
}
|
|
401
436
|
bindShapeKeys(e) {
|
|
402
437
|
const t = e.morphTargetDictionary;
|
|
403
438
|
return {
|
|
404
|
-
update: (
|
|
405
|
-
var
|
|
406
|
-
(
|
|
407
|
-
const { categoryName:
|
|
408
|
-
if (!(t != null && t.hasOwnProperty(
|
|
409
|
-
this.smoothed[
|
|
410
|
-
const
|
|
411
|
-
this.smoothed[
|
|
439
|
+
update: (o) => {
|
|
440
|
+
var n;
|
|
441
|
+
(n = this.blendshapeCategories) == null || n.forEach((a) => {
|
|
442
|
+
const { categoryName: r, score: i } = a;
|
|
443
|
+
if (!(t != null && t.hasOwnProperty(r))) return;
|
|
444
|
+
this.smoothed[r] === void 0 && (this.smoothed[r] = i);
|
|
445
|
+
const l = 1 - Math.pow(this.smoothing, o);
|
|
446
|
+
this.smoothed[r] += (i - this.smoothed[r]) * l, e.morphTargetInfluences[t[r]] = this.smoothed[r];
|
|
412
447
|
});
|
|
413
448
|
}
|
|
414
449
|
};
|
|
415
450
|
}
|
|
416
451
|
bind(e) {
|
|
417
|
-
const t = new
|
|
452
|
+
const t = new Le(e, "L"), o = new Le(e, "R"), n = F(e, "head");
|
|
418
453
|
return {
|
|
419
454
|
update: (a) => {
|
|
420
|
-
if (t.update(a, this.blendshapeMap),
|
|
421
|
-
const
|
|
422
|
-
|
|
455
|
+
if (t.update(a, this.blendshapeMap), o.update(a, this.blendshapeMap), !n) return;
|
|
456
|
+
const r = ft.copy(this.marks.earL.worldPosition), i = wt.copy(this.marks.earR.worldPosition), l = be.subVectors(r, i).multiplyScalar(0.5).add(i), m = yt.subVectors(this.marks.noseTip.worldPosition, l), h = r.sub(i), b = S(Lt, n, e), u = h.add(b).applyMatrix4(e.matrixWorld), E = m.add(b).applyMatrix4(e.matrixWorld);
|
|
457
|
+
N(n, E, u, "+x");
|
|
423
458
|
}
|
|
424
459
|
};
|
|
425
460
|
}
|
|
@@ -431,84 +466,85 @@ class mt extends H {
|
|
|
431
466
|
* @returns an object with a disposeMaterial method that should be called when the mesh is disposed of.
|
|
432
467
|
*/
|
|
433
468
|
bindGeometry(e, t) {
|
|
434
|
-
new
|
|
435
|
-
const
|
|
436
|
-
new
|
|
437
|
-
const a = e.geometry.attributes.position
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
469
|
+
new x(), new x();
|
|
470
|
+
const o = new x();
|
|
471
|
+
new x();
|
|
472
|
+
const a = e.geometry.attributes.position, r = 209, i = 429, l = z(new x().addVectors(
|
|
473
|
+
new x(a.getX(r), a.getY(r), a.getZ(r)),
|
|
474
|
+
new x(a.getX(i), a.getY(i), a.getZ(i))
|
|
475
|
+
).multiplyScalar(0.5));
|
|
476
|
+
qe(e);
|
|
477
|
+
const m = _e("landmarkIndex", "uint"), h = Ce(ee, "vec3"), b = De(h.element(m)).xy;
|
|
478
|
+
let u;
|
|
479
|
+
const E = 116, d = 346, y = o.subVectors(
|
|
480
|
+
new x(a.getX(E), a.getY(E), a.getZ(E)),
|
|
481
|
+
new x(a.getX(d), a.getY(d), a.getZ(d))
|
|
482
|
+
).lengthSq(), P = z(y), k = z(1);
|
|
483
|
+
let L;
|
|
448
484
|
return {
|
|
449
485
|
/**
|
|
450
486
|
* Disposes of the material and removes events listeners on the video element.
|
|
451
487
|
*/
|
|
452
488
|
disposeMaterial: () => {
|
|
453
|
-
|
|
489
|
+
L == null || L();
|
|
454
490
|
},
|
|
455
491
|
/**
|
|
456
492
|
* asas
|
|
457
493
|
* @param delta asas
|
|
458
494
|
* @returns
|
|
459
495
|
*/
|
|
460
|
-
update: (
|
|
461
|
-
var
|
|
462
|
-
if (!
|
|
463
|
-
const
|
|
464
|
-
if (!
|
|
465
|
-
|
|
466
|
-
const
|
|
467
|
-
|
|
468
|
-
const
|
|
469
|
-
const
|
|
470
|
-
|
|
496
|
+
update: (v) => {
|
|
497
|
+
var f, A;
|
|
498
|
+
if (!u) {
|
|
499
|
+
const W = (A = (f = this.cfg) == null ? void 0 : f.videoElementRef) == null ? void 0 : A.call(f);
|
|
500
|
+
if (!W || !W.videoWidth || !W.videoHeight) return;
|
|
501
|
+
u = W;
|
|
502
|
+
const V = new Z(u);
|
|
503
|
+
V.colorSpace = J;
|
|
504
|
+
const M = z(u.videoWidth / u.videoHeight), Pe = h.element(d).sub(h.element(E)).lengthSq(), ve = P.div(Pe).sqrt(), ge = h.element(234).xy, K = h.element(93).xy, Ee = h.element(454).xy, $ = h.element(323).xy, X = ge.sub(K).div(2).add(K), We = Ee.sub($).div(2).add($).sub(X).div(2).add(X), Fe = h.element(m).sub(We).xzy.mul(ze(1, -1, Be(1).div(M))).mul(ve).add(l), Q = Ue(m.toUint().lessThanEqual(ee), Fe, Ie), O = He(V, Ge(b.x, b.y.oneMinus())), Y = () => {
|
|
505
|
+
const T = new Z(u);
|
|
506
|
+
T.colorSpace = J, O.value = T, O.needsUpdate = !0, M.value = u.videoWidth / u.videoHeight;
|
|
471
507
|
};
|
|
472
|
-
|
|
473
|
-
positionNode:
|
|
474
|
-
colorNode:
|
|
508
|
+
u.addEventListener("loadeddata", Y), t ? t(Q, O) : e.material = new Oe({
|
|
509
|
+
positionNode: Q,
|
|
510
|
+
colorNode: O,
|
|
475
511
|
roughness: 0.93
|
|
476
|
-
}),
|
|
477
|
-
var
|
|
478
|
-
|
|
512
|
+
}), L = () => {
|
|
513
|
+
var T;
|
|
514
|
+
O.value.dispose(), u == null || u.removeEventListener("loadeddata", Y), (T = e.material) == null || T.dispose();
|
|
479
515
|
};
|
|
480
516
|
}
|
|
481
|
-
const
|
|
482
|
-
if (!(
|
|
483
|
-
|
|
484
|
-
const
|
|
485
|
-
for (let
|
|
486
|
-
|
|
487
|
-
|
|
517
|
+
const w = this.lastKnownLandmarks;
|
|
518
|
+
if (!(w != null && w.length)) return;
|
|
519
|
+
k.value = o.subVectors(w[d], w[E]).lengthSq(), k.needsUpdate = !0;
|
|
520
|
+
const g = h.value.array;
|
|
521
|
+
for (let W = 0; W < w.length; W++)
|
|
522
|
+
g[W * 3] = w[W].x, g[W * 3 + 1] = w[W].y, g[W * 3 + 2] = w[W].z;
|
|
523
|
+
h.value.needsUpdate = !0;
|
|
488
524
|
}
|
|
489
525
|
};
|
|
490
526
|
}
|
|
491
527
|
}
|
|
492
|
-
class
|
|
493
|
-
constructor(
|
|
528
|
+
class Le {
|
|
529
|
+
constructor(s, e) {
|
|
494
530
|
R(this, "eyeBone");
|
|
495
531
|
R(this, "eyeLookOut");
|
|
496
532
|
R(this, "eyeLookIn");
|
|
497
533
|
R(this, "eyeLookUp");
|
|
498
534
|
R(this, "eyeLookDown");
|
|
499
535
|
R(this, "sign", 1);
|
|
500
|
-
this.rig =
|
|
536
|
+
this.rig = s, this.side = e, this.eyeBone = s.getObjectByName(`eye${e}`);
|
|
501
537
|
const t = e == "L" ? "Left" : "Right";
|
|
502
538
|
this.eyeLookOut = `eyeLookOut${t}`, this.eyeLookIn = `eyeLookIn${t}`, this.eyeLookUp = `eyeLookUp${t}`, this.eyeLookDown = `eyeLookDown${t}`, this.sign = e == "L" ? -1 : 1;
|
|
503
539
|
}
|
|
504
|
-
update(
|
|
540
|
+
update(s, e) {
|
|
505
541
|
if (!this.eyeBone) return;
|
|
506
|
-
|
|
507
|
-
const t = e.get(this.eyeLookOut) ?? 0,
|
|
508
|
-
this.eyeBone.rotation.y =
|
|
542
|
+
S(be, this.eyeBone, this.rig);
|
|
543
|
+
const t = e.get(this.eyeLookOut) ?? 0, o = e.get(this.eyeLookIn) ?? 0, n = e.get(this.eyeLookUp) ?? 0, a = e.get(this.eyeLookDown) ?? 0, r = o - t, i = a - n;
|
|
544
|
+
this.eyeBone.rotation.y = r * this.sign / 2, this.eyeBone.rotation.x = i / 2;
|
|
509
545
|
}
|
|
510
546
|
}
|
|
511
|
-
const
|
|
547
|
+
const Re = {
|
|
512
548
|
faceMesh: "face",
|
|
513
549
|
head: "head",
|
|
514
550
|
hips: "hips",
|
|
@@ -558,7 +594,7 @@ const ke = {
|
|
|
558
594
|
thumb1R: "thumb1R",
|
|
559
595
|
thumb2R: "thumb2R",
|
|
560
596
|
thumb3R: "thumb3R"
|
|
561
|
-
},
|
|
597
|
+
}, xt = [
|
|
562
598
|
"eyeBlinkLeft",
|
|
563
599
|
"eyeBlinkRight",
|
|
564
600
|
"eyeLookDownLeft",
|
|
@@ -612,94 +648,96 @@ const ke = {
|
|
|
612
648
|
"mouthUpperUpRight",
|
|
613
649
|
"tongueOut"
|
|
614
650
|
];
|
|
615
|
-
function
|
|
616
|
-
const t = [],
|
|
617
|
-
|
|
618
|
-
),
|
|
619
|
-
if (
|
|
620
|
-
for (const
|
|
621
|
-
|
|
622
|
-
const a = Object.keys(
|
|
623
|
-
|
|
624
|
-
if (
|
|
625
|
-
const
|
|
626
|
-
|
|
627
|
-
ref:
|
|
628
|
-
name:
|
|
629
|
-
normalizedName:
|
|
651
|
+
function bt(c, s, e = 30) {
|
|
652
|
+
const t = [], o = c.getObjectByName(
|
|
653
|
+
s.faceMesh
|
|
654
|
+
), n = /* @__PURE__ */ new Set();
|
|
655
|
+
if (o != null && o.morphTargetDictionary)
|
|
656
|
+
for (const k in o.morphTargetDictionary)
|
|
657
|
+
xt.includes(k) && n.add(k);
|
|
658
|
+
const a = Object.keys(s);
|
|
659
|
+
c.traverse((k) => {
|
|
660
|
+
if (k instanceof D.Bone) {
|
|
661
|
+
const L = a.find((v) => k.name.indexOf(s[v]) === 0);
|
|
662
|
+
L && t.push({
|
|
663
|
+
ref: k,
|
|
664
|
+
name: k.name,
|
|
665
|
+
normalizedName: L
|
|
630
666
|
});
|
|
631
667
|
}
|
|
632
668
|
});
|
|
633
|
-
const
|
|
634
|
-
let
|
|
635
|
-
function k
|
|
636
|
-
|
|
637
|
-
}
|
|
638
|
-
function
|
|
639
|
-
if (!
|
|
640
|
-
const
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
669
|
+
const r = [], i = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map();
|
|
670
|
+
let m = !1, h = 0, b = 0, u = 0, E = 0;
|
|
671
|
+
function d(k) {
|
|
672
|
+
i.clear(), l.clear(), r.length = 0, b = k ?? 0, E = b > 0 ? 1 / b : 0, h = performance.now() / 1e3, m = !0, u = 0;
|
|
673
|
+
}
|
|
674
|
+
function y() {
|
|
675
|
+
if (!m) return;
|
|
676
|
+
const k = performance.now() / 1e3 - h;
|
|
677
|
+
if (!(b > 0 && k - u < E)) {
|
|
678
|
+
u = k, r.push(k);
|
|
679
|
+
for (const L of t) {
|
|
680
|
+
i.has(L.name) || i.set(L.name, []);
|
|
681
|
+
const v = i.get(L.name), w = L.ref.quaternion;
|
|
682
|
+
v.push(w.x, w.y, w.z, w.w);
|
|
683
|
+
}
|
|
684
|
+
for (const L of n) {
|
|
685
|
+
l.has(L) || l.set(L, []);
|
|
686
|
+
const v = l.get(L), w = o.morphTargetDictionary[L], g = o.morphTargetInfluences[w];
|
|
687
|
+
v.push(g);
|
|
688
|
+
}
|
|
651
689
|
}
|
|
652
690
|
}
|
|
653
|
-
function
|
|
654
|
-
|
|
655
|
-
const
|
|
656
|
-
for (const [
|
|
657
|
-
|
|
658
|
-
new
|
|
659
|
-
`${
|
|
660
|
-
|
|
661
|
-
|
|
691
|
+
function P(k = "RecordedClip") {
|
|
692
|
+
m = !1;
|
|
693
|
+
const L = [];
|
|
694
|
+
for (const [w, g] of i)
|
|
695
|
+
L.push(
|
|
696
|
+
new D.QuaternionKeyframeTrack(
|
|
697
|
+
`${w}.quaternion`,
|
|
698
|
+
r,
|
|
699
|
+
g
|
|
662
700
|
)
|
|
663
701
|
);
|
|
664
|
-
for (const [
|
|
665
|
-
|
|
666
|
-
new
|
|
667
|
-
`${
|
|
668
|
-
|
|
669
|
-
|
|
702
|
+
for (const [w, g] of l)
|
|
703
|
+
L.push(
|
|
704
|
+
new D.NumberKeyframeTrack(
|
|
705
|
+
`${Re.faceMesh}.morphTargetInfluences[${w}]`,
|
|
706
|
+
r,
|
|
707
|
+
g
|
|
670
708
|
)
|
|
671
709
|
);
|
|
672
|
-
const
|
|
710
|
+
const v = new D.AnimationClip(k, -1, L);
|
|
673
711
|
return {
|
|
674
|
-
clip:
|
|
675
|
-
saveToFile: () => {
|
|
676
|
-
new
|
|
677
|
-
|
|
678
|
-
(
|
|
679
|
-
const
|
|
712
|
+
clip: v,
|
|
713
|
+
saveToFile: () => new Promise((w, g) => {
|
|
714
|
+
new je().parse(
|
|
715
|
+
c,
|
|
716
|
+
(A) => {
|
|
717
|
+
const W = new Blob([A], {
|
|
680
718
|
type: "model/gltf-binary"
|
|
681
|
-
}),
|
|
682
|
-
|
|
719
|
+
}), V = URL.createObjectURL(W), M = document.createElement("a");
|
|
720
|
+
M.href = V, M.download = k + ".glb", M.click(), w(!0);
|
|
683
721
|
},
|
|
684
|
-
(
|
|
685
|
-
console.error(
|
|
722
|
+
(A) => {
|
|
723
|
+
console.error(A), g(A);
|
|
686
724
|
},
|
|
687
725
|
{
|
|
688
726
|
binary: !0,
|
|
689
|
-
animations: [
|
|
727
|
+
animations: [v]
|
|
690
728
|
}
|
|
691
729
|
);
|
|
692
|
-
}
|
|
730
|
+
})
|
|
693
731
|
};
|
|
694
732
|
}
|
|
695
|
-
return { start:
|
|
733
|
+
return { start: d, captureFrame: y, stop: P, isRecording: () => m };
|
|
696
734
|
}
|
|
697
|
-
const
|
|
698
|
-
var
|
|
699
|
-
return !!((
|
|
735
|
+
const Rt = () => {
|
|
736
|
+
var c;
|
|
737
|
+
return !!((c = navigator.mediaDevices) != null && c.getUserMedia);
|
|
700
738
|
};
|
|
701
|
-
async function
|
|
702
|
-
const
|
|
739
|
+
async function St(c) {
|
|
740
|
+
const s = {
|
|
703
741
|
debugFrame: void 0,
|
|
704
742
|
displayScale: 1,
|
|
705
743
|
ignoreLegs: !1,
|
|
@@ -713,141 +751,151 @@ async function Pt(l) {
|
|
|
713
751
|
hand: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
|
|
714
752
|
face: "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task"
|
|
715
753
|
},
|
|
716
|
-
...
|
|
754
|
+
...c
|
|
717
755
|
};
|
|
718
756
|
let e;
|
|
719
|
-
const t = await
|
|
720
|
-
ignoreLegs:
|
|
721
|
-
modelPath:
|
|
722
|
-
drawLandmarks:
|
|
723
|
-
}),
|
|
724
|
-
leftWrist: () =>
|
|
725
|
-
rightWrist: () =>
|
|
726
|
-
modelPath:
|
|
727
|
-
drawLandmarks:
|
|
728
|
-
...
|
|
729
|
-
}), a =
|
|
730
|
-
|
|
731
|
-
const
|
|
732
|
-
|
|
733
|
-
function
|
|
734
|
-
|
|
735
|
-
}
|
|
736
|
-
function
|
|
737
|
-
e = document.createElement("video"),
|
|
757
|
+
const t = await Se.forVisionTasks(s.modelPaths.vision ?? "/wasm"), o = s.onlyFace ? void 0 : await ot(t, {
|
|
758
|
+
ignoreLegs: s.ignoreLegs,
|
|
759
|
+
modelPath: s.modelPaths.pose,
|
|
760
|
+
drawLandmarks: s.drawLandmarksOverlay
|
|
761
|
+
}), n = s.onlyFace ? void 0 : await ct(t, {
|
|
762
|
+
leftWrist: () => o.leftWristNormalizedPosition,
|
|
763
|
+
rightWrist: () => o.rightWristNormalizedPosition,
|
|
764
|
+
modelPath: s.modelPaths.hand,
|
|
765
|
+
drawLandmarks: s.drawLandmarksOverlay,
|
|
766
|
+
...c == null ? void 0 : c.handsTrackerOptions
|
|
767
|
+
}), a = s.ignoreFace ? void 0 : await pt(t, { modelPath: s.modelPaths.face, videoElementRef: () => e, drawLandmarks: s.drawLandmarksOverlay }), r = document.createElement("div");
|
|
768
|
+
r.style.position = "absolute", r.style.top = "0px", r.style.left = "0px", r.style.zIndex = "21", r.style.maxWidth = "600px", r.style.width = "100%", r.classList.add("three-mediapipe-rig"), document.body.appendChild(r);
|
|
769
|
+
const i = document.createElement("canvas"), l = i.getContext("2d"), m = new Ve(l);
|
|
770
|
+
i.style.zIndex = "22", i.style.position = "absolute", i.style.top = "0px", i.style.left = "0px", i.style.pointerEvents = "none", i.style.maxWidth = "100%", r.appendChild(i);
|
|
771
|
+
function h(d) {
|
|
772
|
+
l.save(), l.clearRect(0, 0, i.width, i.height), o == null || o.predict(d, m), n == null || n.left.predict(d, m), n == null || n.right.predict(d, m), a == null || a.predict(d, m), l.restore();
|
|
773
|
+
}
|
|
774
|
+
function b() {
|
|
775
|
+
e = document.createElement("video"), r.appendChild(e);
|
|
738
776
|
let d = -1;
|
|
739
|
-
e.style.zIndex = "21", e.style.position = "absolute", e.style.top = "0px", e.style.left = "0px", e.style.height = "auto", e.style.maxWidth = "100%", e.style.display = "block",
|
|
740
|
-
function
|
|
741
|
-
d !== e.currentTime && e.readyState >= 2 && (
|
|
777
|
+
e.style.zIndex = "21", e.style.position = "absolute", e.style.top = "0px", e.style.left = "0px", e.style.height = "auto", e.style.maxWidth = "100%", e.style.display = "block", e.muted = !1, s.debugVideo && (e.src = s.debugVideo, e.controls = !0, e.loop = !0, e.muted = !0, e.controls = !0, e.play());
|
|
778
|
+
function y() {
|
|
779
|
+
d !== e.currentTime && e.readyState >= 2 && (h(e), d = e.currentTime), window.requestAnimationFrame(y);
|
|
742
780
|
}
|
|
743
781
|
e.addEventListener("loadeddata", () => {
|
|
744
|
-
|
|
782
|
+
const P = e.videoWidth, k = e.videoHeight, w = Math.min(600 / P, 600 / k, 1);
|
|
783
|
+
e.width = P * w * (s.displayScale ?? 1), e.height = k * w * (s.displayScale ?? 1), i.width = e.width, i.height = e.height, i.style.height = "auto", i.style.width = e.width + "px", i.style.maxWidth = "100%", window.requestAnimationFrame(y);
|
|
745
784
|
});
|
|
746
785
|
}
|
|
747
|
-
if (
|
|
786
|
+
if (s.debugFrame) {
|
|
748
787
|
const d = document.createElement("img");
|
|
749
|
-
d.src =
|
|
750
|
-
d.width = d.naturalWidth *
|
|
751
|
-
function
|
|
752
|
-
|
|
788
|
+
d.src = s.debugFrame, d.style.zIndex = "21", d.style.position = "absolute", d.style.top = "0px", d.style.left = "0px", r.appendChild(d), d.addEventListener("load", () => {
|
|
789
|
+
d.width = d.naturalWidth * s.displayScale, d.height = d.naturalHeight * s.displayScale, i.width = d.naturalWidth, i.height = d.naturalWidth, i.style.width = d.width + "px", i.style.height = d.height + "px";
|
|
790
|
+
function y() {
|
|
791
|
+
h(d);
|
|
753
792
|
}
|
|
754
|
-
window.requestAnimationFrame(
|
|
793
|
+
window.requestAnimationFrame(y);
|
|
755
794
|
});
|
|
756
|
-
} else
|
|
757
|
-
let
|
|
795
|
+
} else s.debugVideo && b();
|
|
796
|
+
let u;
|
|
797
|
+
function E(d) {
|
|
798
|
+
e && (e.srcObject ? e.srcObject.getTracks().forEach((P) => P.enabled = d) : d ? e.play() : e.pause());
|
|
799
|
+
}
|
|
758
800
|
return {
|
|
759
|
-
poseTracker:
|
|
760
|
-
handsTracker:
|
|
801
|
+
poseTracker: o,
|
|
802
|
+
handsTracker: n,
|
|
761
803
|
faceTracker: a,
|
|
762
804
|
video: e,
|
|
763
|
-
canvas:
|
|
764
|
-
domElement:
|
|
765
|
-
start: async () => {
|
|
766
|
-
let
|
|
767
|
-
if (!
|
|
805
|
+
canvas: i,
|
|
806
|
+
domElement: r,
|
|
807
|
+
start: async (d = !1) => {
|
|
808
|
+
let y = !1;
|
|
809
|
+
if (!Rt())
|
|
768
810
|
throw new Error("Webcam not supported");
|
|
769
|
-
e ||
|
|
770
|
-
let
|
|
771
|
-
function
|
|
772
|
-
|
|
811
|
+
e || b();
|
|
812
|
+
let P;
|
|
813
|
+
function k(f) {
|
|
814
|
+
y || (console.warn("Camera track ended, attempting recovery..."), L(f), v(f));
|
|
773
815
|
}
|
|
774
|
-
function
|
|
775
|
-
|
|
816
|
+
function L(f) {
|
|
817
|
+
P == null || P.getVideoTracks().forEach((A) => A.stop()), P = void 0, f.srcObject = null;
|
|
776
818
|
}
|
|
777
|
-
async function
|
|
778
|
-
const
|
|
779
|
-
if (
|
|
819
|
+
async function v(f, A = 0) {
|
|
820
|
+
const V = Math.min(1e3 * 2 ** A, 16e3);
|
|
821
|
+
if (A >= 3)
|
|
780
822
|
throw new Error("Camera recovery failed after max attempts");
|
|
781
|
-
if (await new Promise((
|
|
823
|
+
if (await new Promise((M) => setTimeout(M, V)), !y)
|
|
782
824
|
try {
|
|
783
|
-
await
|
|
784
|
-
} catch (
|
|
785
|
-
await g(
|
|
825
|
+
await w(f);
|
|
826
|
+
} catch (M) {
|
|
827
|
+
await g(M, f, A + 1);
|
|
786
828
|
}
|
|
787
829
|
}
|
|
788
|
-
async function
|
|
789
|
-
|
|
790
|
-
|
|
830
|
+
async function w(f) {
|
|
831
|
+
P = await navigator.mediaDevices.getUserMedia({ video: !0, audio: d }), f.srcObject = P, await f.play(), P.getVideoTracks().forEach((A) => {
|
|
832
|
+
A.addEventListener("ended", () => k(f));
|
|
791
833
|
});
|
|
792
834
|
}
|
|
793
|
-
async function g(
|
|
794
|
-
if (
|
|
795
|
-
switch (
|
|
835
|
+
async function g(f, A, W = 0) {
|
|
836
|
+
if (f instanceof DOMException)
|
|
837
|
+
switch (f.name) {
|
|
796
838
|
case "NotAllowedError":
|
|
797
839
|
throw new Error("Permission denied — prompt user to allow camera");
|
|
798
840
|
case "NotFoundError":
|
|
799
|
-
console.error("No camera found — retry when device is connected"), await
|
|
841
|
+
console.error("No camera found — retry when device is connected"), await v(A, W + 1);
|
|
800
842
|
break;
|
|
801
843
|
case "NotReadableError":
|
|
802
844
|
throw new Error("Camera in use by another app");
|
|
803
845
|
default:
|
|
804
|
-
throw new Error("Camera error: " +
|
|
846
|
+
throw new Error("Camera error: " + f.message);
|
|
805
847
|
}
|
|
806
848
|
else
|
|
807
|
-
throw new Error("Unknown camera error: " +
|
|
849
|
+
throw new Error("Unknown camera error: " + f);
|
|
808
850
|
}
|
|
809
|
-
return await
|
|
810
|
-
|
|
851
|
+
return await v(e), u = () => {
|
|
852
|
+
y = !0, L(e), u = void 0;
|
|
811
853
|
}, {
|
|
812
|
-
|
|
854
|
+
getStream: () => P,
|
|
855
|
+
stop: u
|
|
813
856
|
};
|
|
814
857
|
},
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
858
|
+
pause: () => {
|
|
859
|
+
E(!1);
|
|
860
|
+
},
|
|
861
|
+
resume: () => {
|
|
862
|
+
E(!0);
|
|
863
|
+
},
|
|
864
|
+
bind: (d, y) => {
|
|
865
|
+
if (y = y || Re, !o) throw new Error("Pose tracker not initialized");
|
|
866
|
+
if (!n) throw new Error("Hands tracker not initialized");
|
|
867
|
+
const P = o.bind(d, y), k = n.left.bind(d, y), L = n.right.bind(d, y);
|
|
820
868
|
let v;
|
|
821
|
-
const
|
|
822
|
-
d.traverse((
|
|
823
|
-
|
|
869
|
+
const w = a == null ? void 0 : a.bind(d);
|
|
870
|
+
d.traverse((f) => {
|
|
871
|
+
f instanceof p.Mesh && f.name.indexOf(y.faceMesh) === 0 && (f.frustumCulled = !1, v = a == null ? void 0 : a.bindShapeKeys(f));
|
|
824
872
|
});
|
|
825
|
-
const
|
|
873
|
+
const g = bt(d, y);
|
|
826
874
|
return {
|
|
827
875
|
/**
|
|
828
876
|
* Will save the tracked movement of the rig to an animation clip.
|
|
829
877
|
* Only the bones moved by the bone mapping will be recorded.
|
|
830
878
|
*/
|
|
831
|
-
startRecording:
|
|
832
|
-
stopRecording:
|
|
833
|
-
isRecording: () =>
|
|
834
|
-
update: (
|
|
835
|
-
|
|
879
|
+
startRecording: g.start,
|
|
880
|
+
stopRecording: g.stop,
|
|
881
|
+
isRecording: () => g.isRecording(),
|
|
882
|
+
update: (f) => {
|
|
883
|
+
P.update(f), k.update(f), L.update(f), v == null || v.update(f), w == null || w.update(f), g.isRecording() && g.captureFrame();
|
|
836
884
|
}
|
|
837
885
|
};
|
|
838
886
|
},
|
|
839
887
|
setVideoFromSource: (d) => {
|
|
840
|
-
|
|
888
|
+
u == null || u(), e || b(), e.src = d instanceof File ? URL.createObjectURL(d) : d, e.controls = !0, e.loop = !0, e.muted = !0, e.controls = !0, e.play();
|
|
841
889
|
},
|
|
842
|
-
async setVideoFromWebcam() {
|
|
843
|
-
if (e ||
|
|
890
|
+
async setVideoFromWebcam(d = !1) {
|
|
891
|
+
if (e || b(), u)
|
|
844
892
|
throw new Error("Webcam already started");
|
|
845
|
-
return await this.start();
|
|
893
|
+
return await this.start(d);
|
|
846
894
|
}
|
|
847
895
|
};
|
|
848
896
|
}
|
|
849
897
|
export {
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
898
|
+
ee as FACE_LANDMARKS_COUNT,
|
|
899
|
+
qe as createFaceLandmarksIndexAttribute,
|
|
900
|
+
St as setupTracker
|
|
853
901
|
};
|