three-mediapipe-rig 0.1.1 → 0.1.3

package/dist/meshcap.js

@@ -0,0 +1,468 @@
+import { inflate as xt, deflate as Mt } from "fflate";
+import { c as At, F as X } from "./face-tracker-utils-xt9__vBF.js";
+import { MeshPhysicalNodeMaterial as Ct, Vector3 as et } from "three/webgpu";
+import { instancedArray as H, texture as it, uniform as K, attribute as bt, varying as St, vec3 as It, float as kt } from "three/tsl";
+const dt = 1296253264, Rt = 2;
+async function Vt(a) {
+  if (a instanceof ArrayBuffer)
+    return a;
+  if (a instanceof File || a instanceof Blob)
+    return a.arrayBuffer();
+  if (typeof a == "string")
+    return (await fetch(a)).arrayBuffer();
+  throw new Error(`Unsupported audio source type: ${typeof a}`);
+}
+async function Lt(a, d) {
+  const i = new AudioContext(), o = await Vt(a), e = await i.decodeAudioData(o);
+  for (const l of d) {
+    if (!l.audioSprite)
+      continue;
+    const v = Math.floor(l.audioSprite.start * e.sampleRate), x = Math.floor(l.duration * e.sampleRate), f = i.createBuffer(
+      e.numberOfChannels,
+      x,
+      e.sampleRate
+    );
+    for (let r = 0; r < e.numberOfChannels; r++) {
+      const p = e.getChannelData(r).subarray(v, v + x);
+      f.getChannelData(r).set(p);
+    }
+    const s = Tt(f), t = new Blob([s], { type: "audio/wav" }), n = URL.createObjectURL(t), c = new Audio(n);
+    l.audioSprite.domElement = c, console.log("Assigned audio to clip: ", l.name, " at ", l.audioSprite.start, " for ", l.duration, " seconds");
+  }
+  return i.close(), {
+    blob: new Blob([o], { type: "audio/wav" }),
+    sprites: d.map((l) => l.audioSprite ? [l.audioSprite.start, l.duration] : void 0)
+  };
+}
+function Tt(a) {
+  const d = a.numberOfChannels, i = a.sampleRate, o = a.length * d * 2, e = new ArrayBuffer(44 + o), l = new DataView(e), v = (f, s) => [...s].forEach((t, n) => l.setUint8(f + n, t.charCodeAt(0)));
+  v(0, "RIFF"), l.setUint32(4, 36 + o, !0), v(8, "WAVE"), v(12, "fmt "), l.setUint32(16, 16, !0), l.setUint16(20, 1, !0), l.setUint16(22, d, !0), l.setUint32(24, i, !0), l.setUint32(28, i * d * 2, !0), l.setUint16(32, d * 2, !0), l.setUint16(34, 16, !0), v(36, "data"), l.setUint32(40, o, !0);
+  let x = 44;
+  for (let f = 0; f < a.length; f++)
+    for (let s = 0; s < d; s++) {
+      const t = Math.max(-1, Math.min(1, a.getChannelData(s)[f]));
+      l.setInt16(x, t * 32767, !0), x += 2;
+    }
+  return e;
+}
+const ct = /* @__PURE__ */ new WeakMap();
+function zt(a, d) {
+  const i = ct.get(a) || new AudioContext();
+  ct.set(a, i);
+  const o = i.createGain();
+  o.connect(i.destination), o.gain.value = 1;
+  let e;
+  return {
+    stopCurrent() {
+      e && (e.stop(), e = void 0);
+    },
+    setVolume(l) {
+      o.gain.value = l;
+    },
+    /** 
+     * @param clipIndex Index of the clip to play
+     * @returns The audio buffer source node that is playing the audio.
+     */
+    playSprite(l) {
+      this.stopCurrent();
+      const v = d[l];
+      if (!v.audioSprite) return;
+      const x = i.createBufferSource();
+      return x.buffer = a, x.connect(o), x.start(0, v.audioSprite.start, v.duration), e = x, console.log("PLAYING SOUNC", v.name), x;
+    }
+  };
+}
+function Bt(a, d, i, o, e) {
+  const l = !o;
+  o ?? (o = new Ct()), l && (i.material = o), i.geometry.hasAttribute("landmarkIndex") || At(i);
+  const v = [], x = [], f = [];
+  let s = 0, t = 0;
+  for (let U = 0; U < d.length; U++) {
+    const w = d[U];
+    v.push(s, t, w.frames.length, w.fps), s += w.frames.length, t += w.frames.length * X, x.push(w.aspectRatio), f.push(U == 0 ? 0 : f[U - 1] + d[U - 1].frames.length);
+  }
+  const n = H(new Float32Array(x), "float"), c = H(new Uint32Array(v), "uvec4"), r = new Float32Array(d.flatMap((U) => U.frames).reduce((U, w) => (U.push(w.frameUV.u, w.frameUV.v, w.frameUV.w, w.frameUV.h), U), [])), p = H(r, "vec4"), M = new Float32Array(d.flatMap((U) => U.frames.flatMap((w) => [w.cropUV.u, w.cropUV.v, w.cropUV.w, w.cropUV.h]))), m = H(M, "vec4"), u = new Float32Array(d.flatMap((U) => U.landmarks.flatMap((w) => w.flatMap((I) => [I.x, I.y, I.z])))), h = H(u, "vec3"), g = it(a), y = K(0), A = K(0), C = K(!0), k = n.element(y), j = H(new Uint32Array(f), "uint"), R = A, D = j.element(y).add(R), z = p.element(c.element(y).x.add(R)), b = c.element(y).y.add(R.mul(X)), J = bt("landmarkIndex", "uint").toUint(), Q = b.add(J), B = h.element(Q), V = m.element(D), Y = B.xy.sub(V.xy).div(V.zw), q = St(z.xy.add(Y.mul(z.zw))), _ = it(g, q);
+  o.colorNode = _;
+  const ft = h.element(b.add(234)).xy, nt = h.element(b.add(93)).xy, ut = h.element(b.add(454)).xy, ot = h.element(b.add(323)).xy, at = ft.sub(nt).div(2).add(nt), ht = ut.sub(ot).div(2).add(ot).sub(at).div(2).add(at), E = i.geometry.attributes.position, Z = 116, $ = 346, pt = new et().subVectors(
+    new et(E.getX(Z), E.getY(Z), E.getZ(Z)),
+    new et(E.getX($), E.getY($), E.getZ($))
+  ).lengthSq(), mt = K(pt), gt = h.element(b.add($)).sub(h.element(b.add(Z))).lengthSq(), Ut = mt.div(gt).sqrt().mul(2), wt = B.sub(ht).xzy.mul(It(1, -1, kt(1).div(k))).mul(Ut);
+  o.positionNode = wt;
+  let G = 0, L = 0, N, P, st = d[0].frames, tt = !1, O = e ? zt(e, d) : void 0;
+  const S = {
+    muted: !1,
+    clips: d,
+    atlasTexture: a,
+    material: o,
+    goto(U, w = !0, I, yt = !0) {
+      var rt;
+      let T = -1;
+      if (typeof U == "number")
+        T = U;
+      else if (T = d.findIndex((vt) => vt.name === U), T === -1)
+        throw new Error(`Clip ${U.toString()} not found`);
+      C.value = w, A.value = 0, y.value = T;
+      const W = d[T];
+      N = W, st = W.frames, G = W.duration, L = 0, P = I, tt = !0, O && (O.stopCurrent(), yt && !S.muted && W.audioSprite && (O.playSprite(T), (rt = S.playClipAudioHook) == null || rt.call(S, T, W.audioSprite.start, W.duration)));
+    },
+    update(U) {
+      var w;
+      if (tt) {
+        if (L += U, A.value = Ft(st, L, A.value), C.value)
+          L >= G && (L -= G, A.value = 0, S.muted || (O == null || O.playSprite(y.value), N != null && N.audioSprite && ((w = S.playClipAudioHook) == null || w.call(S, y.value, N.audioSprite.start, N.duration))), P == null || P(L));
+        else if (L >= G) {
+          const I = P;
+          P = void 0, I == null || I(G);
+        }
+      }
+    },
+    dispose() {
+      a.dispose(), o.dispose();
+    },
+    gotoAndLoop(U, w) {
+      this.goto(U, !0, w);
+    },
+    gotoAndPlay(U, w) {
+      this.goto(U, !1, w);
+    },
+    gotoAndStop(U, w = 0) {
+      this.goto(U, !1, void 0, !1), A.value = w, tt = !1;
+    }
+  };
+  return S;
+}
+function Ft(a, d, i) {
+  for (; i < a.length - 1 && a[i + 1].startTime <= d; )
+    i++;
+  return i;
+}
+async function _t(a) {
+  if (typeof a == "string") {
+    const i = await (await fetch(a)).arrayBuffer();
+    return lt(i);
+  } else
+    return new Promise((d, i) => {
+      const o = new FileReader();
+      o.onload = (e) => {
+        const l = e.target.result;
+        d(lt(l));
+      }, o.onerror = (e) => {
+        i(e);
+      }, o.readAsArrayBuffer(a);
+    });
+}
+async function lt(a) {
+  let d = !1;
+  const i = await new Promise((n, c) => {
+    xt(new Uint8Array(a), (r, p) => {
+      r ? c(r) : n(p);
+    });
+  }), o = new DataView(i.buffer);
+  let e = 0;
+  const l = o.getUint32(e);
+  if (e += 4, l !== dt) throw new Error("Invalid file: not a MCAP file");
+  const v = o.getUint8(e);
+  e += 1;
+  const x = o.getUint8(e);
+  e += 1;
+  const f = o.getUint16(e);
+  e += 2;
+  const s = o.getUint8(e);
+  e += 1;
+  const t = [];
+  for (let n = 0; n < x; n++) {
+    const c = o.getUint8(e);
+    e += 1;
+    const r = new Uint8Array(i.buffer, e, c), p = new TextDecoder().decode(r);
+    e += c;
+    const M = o.getUint8(e);
+    e += 1;
+    const m = o.getUint8(e);
+    e += 1;
+    const u = o.getUint8(e) / 100;
+    e += 1;
+    const h = o.getUint8(e) / 100;
+    e += 1;
+    const g = [], y = [], A = [];
+    for (let C = 0; C < M; C++) {
+      const k = o.getUint16(e) / 1e3;
+      e += 2;
+      const j = o.getUint16(e) / 1e3;
+      e += 2;
+      const R = o.getUint16(e) / 1e3;
+      e += 2;
+      const D = o.getUint16(e) / 1e3;
+      e += 2, g.push({ u: k, v: j, w: R, h: D });
+      const z = o.getUint16(e) / 1e3;
+      e += 2;
+      const b = o.getUint16(e) / 1e3;
+      e += 2;
+      const J = o.getUint16(e) / 1e3;
+      e += 2;
+      const Q = o.getUint16(e) / 1e3;
+      e += 2, A.push({ u: z, v: b, w: J, h: Q });
+      const B = [], V = C > 0 ? y[C - 1] : null;
+      for (let F = 0; F < X; F++)
+        if (V === null) {
+          const Y = o.getUint16(e) / 1e3;
+          e += 2;
+          const q = o.getUint16(e) / 1e3;
+          e += 2;
+          const _ = o.getInt16(e) / 1e3;
+          e += 2, B.push({ x: Y, y: q, z: _ });
+        } else {
+          const Y = o.getInt8(e) / 1e3;
+          e += 1;
+          const q = o.getInt8(e) / 1e3;
+          e += 1;
+          const _ = o.getInt8(e) / 1e3;
+          e += 1, B.push({
+            x: V[F].x + Y,
+            y: V[F].y + q,
+            z: V[F].z + _
+          });
+        }
+      y.push(B);
+    }
+    t.push({
+      name: p,
+      fps: m,
+      scale: u,
+      aspectRatio: h,
+      frames: A.map((C, k) => ({
+        cropUV: C,
+        frameUV: g[k],
+        startTime: k / m
+        //<-- default for versions sub 2...
+      })),
+      landmarks: y,
+      duration: M / m
+      // default for versions sub 2...
+    });
+  }
+  if (v >= 2) {
+    for (let n = 0; n < t.length; n++) {
+      let c = o.getUint16(e);
+      e += 2, t[n].duration = c / 1e3;
+      let r = o.getUint16(e);
+      e += 2, r === 1 ? console.log("Clip " + n + " has no audio") : (r /= 1e3, t[n].audioSprite = { start: r }, d = !0);
+    }
+    for (let n = 0; n < t.length; n++) {
+      const c = t[n];
+      let r = 0;
+      for (let p = 0; p < c.frames.length; p++) {
+        const M = o.getUint8(e);
+        e += 1;
+        const m = r + M / 1e3;
+        c.frames[p].startTime = m, r = m;
+      }
+    }
+  }
+  return {
+    clips: t,
+    version: v,
+    atlasSize: f,
+    atlasPadding: s,
+    unpackClips: async (n, c) => {
+      let r, p = !0, M;
+      if (n instanceof HTMLCanvasElement)
+        p = !1, r = n;
+      else {
+        let u = new Image();
+        if (typeof n == "string")
+          u.src = n;
+        else if (n instanceof File || n instanceof Blob) {
+          const g = URL.createObjectURL(n);
+          u.src = g, await new Promise((y, A) => {
+            u.onload = () => {
+              URL.revokeObjectURL(g), y();
+            }, u.onerror = A;
+          });
+        } else n instanceof HTMLImageElement && (u = n);
+        (!u.complete || !u.naturalWidth) && await new Promise((g, y) => {
+          u.onload = () => g(), u.onerror = y;
+        }), r = document.createElement("canvas"), r.width = u.width, r.height = u.height, r.getContext("2d").drawImage(u, 0, 0);
+      }
+      const m = t.map((u) => ({
+        ...u,
+        frames: u.frames.map((h) => {
+          const g = document.createElement("canvas");
+          return g.width = h.frameUV.w * r.width, g.height = h.frameUV.h * r.height, g.getContext("2d").drawImage(
+            r,
+            h.frameUV.u * r.width,
+            h.frameUV.v * r.height,
+            h.frameUV.w * r.width,
+            h.frameUV.h * r.height,
+            0,
+            0,
+            g.width,
+            g.height
+          ), {
+            canvas: g,
+            cropUV: h.cropUV,
+            startTime: h.startTime
+          };
+        })
+      }));
+      return p && r.remove(), d && (c ? M = await Lt(c, m) : console.warn("This mcap file uses an audio atlas, but no audio atlas file was provided.")), {
+        clips: m,
+        audioAtlas: M
+      };
+    },
+    createMaterialHandlerOnMesh: (n, c, r, p) => Bt(c, t, n, r, p)
+  };
+}
+async function Et(a, d = !0) {
+  const i = new TextEncoder(), o = a.clips.map((n) => i.encode(n.name)), e = 9, l = o.reduce((n, c, r) => {
+    const p = a.clips[r].frames.length, M = 1 + c.byteLength, m = 1, u = 1, h = 1, g = 1, y = 16;
+    let A = X * 6;
+    return n + M + m + u + h + g + (d ? y + A + (y + X * 3) * (p - 1) : (y + A) * p);
+  }, 0), v = a.clips.reduce((n, c) => n + 2 + 2, 0), x = a.clips.reduce((n, c) => n + c.frames.length * 1, 0), f = new ArrayBuffer(e + l + v + x), s = new DataView(f);
+  let t = 0;
+  s.setUint32(t, dt), t += 4, s.setUint8(t, Rt), t += 1, s.setUint8(t, a.clips.length), t += 1, s.setUint16(t, a.atlasSize), t += 2, s.setUint8(t, a.padding), t += 1;
+  for (let n = 0; n < a.clips.length; n++) {
+    s.setUint8(t, o[n].byteLength), t += 1, new Uint8Array(f, t, o[n].byteLength).set(o[n]), t += o[n].byteLength, s.setUint8(t, a.clips[n].frames.length), t += 1, s.setUint8(t, a.clips[n].fps), t += 1, s.setUint8(t, Math.round(a.clips[n].scale * 100)), t += 1, s.setUint8(t, Math.round(a.clips[n].aspectRatio * 100)), t += 1;
+    const c = a.clips[n];
+    for (let r = 0; r < c.frames.length; r++) {
+      const p = c.frames[r].cropUV, M = c.frames[r].frameUV;
+      s.setUint16(t, Math.round(M.u * 1e3)), t += 2, s.setUint16(t, Math.round(M.v * 1e3)), t += 2, s.setUint16(t, Math.round(M.w * 1e3)), t += 2, s.setUint16(t, Math.round(M.h * 1e3)), t += 2, s.setUint16(t, Math.round(p.u * 1e3)), t += 2, s.setUint16(t, Math.round(p.v * 1e3)), t += 2, s.setUint16(t, Math.round(p.w * 1e3)), t += 2, s.setUint16(t, Math.round(p.h * 1e3)), t += 2;
+      const m = c.landmarks[r], u = r > 0 ? c.landmarks[r - 1] : null;
+      m.reduce((h, g) => Math.max(h, Math.abs(g.z)), 0);
+      for (let h = 0; h < X; h++) {
+        const g = Math.round(m[h].x * 1e3), y = Math.round(m[h].y * 1e3), A = Math.round(m[h].z * 1e3);
+        if (d)
+          if (u === null)
+            s.setUint16(t, g), t += 2, s.setUint16(t, y), t += 2, s.setInt16(t, A), t += 2;
+          else {
+            const C = Math.round(u[h].x * 1e3), k = Math.round(u[h].y * 1e3), j = Math.round(u[h].z * 1e3), R = g - C, D = y - k, z = A - j;
+            s.setInt8(t, R), t += 1, s.setInt8(t, D), t += 1, s.setInt8(t, z), t += 1;
+          }
+        else
+          s.setUint16(t, g), t += 2, s.setUint16(t, y), t += 2, s.setInt16(t, A), t += 2;
+      }
+    }
+  }
+  for (let n = 0; n < a.clips.length; n++) {
+    const c = a.clips[n];
+    s.setUint16(t, Math.round(c.duration * 1e3)), t += 2, c.audioSprite ? s.setUint16(t, Math.round(c.audioSprite.start * 1e3)) : s.setUint16(t, 1), t += 2;
+  }
+  for (let n = 0; n < a.clips.length; n++) {
+    const c = a.clips[n];
+    let r = 0;
+    for (const p of c.frames) {
+      const M = Math.floor((p.startTime - r) * 1e3);
+      s.setUint8(t, M), t += 1, r = p.startTime;
+    }
+  }
+  return new Promise((n, c) => {
+    Mt(new Uint8Array(f), { level: 9 }, (r, p) => {
+      if (r) return c(r);
+      const M = new Blob([p], { type: "application/octet-stream" });
+      n(M);
+    });
+  });
+}
+function Nt(a) {
+  const d = Pt(a), i = [], o = [];
+  let e = 0;
+  const l = [...a].sort((f, s) => s.height - f.height);
+  for (const f of l) {
+    let s = !1;
+    for (const t of i)
+      if (f.width <= d - t.currentX && f.height <= t.height) {
+        o.push({
+          ...f,
+          x: t.currentX,
+          y: t.y
+        }), t.currentX += f.width, s = !0;
+        break;
+      }
+    if (!s) {
+      const t = {
+        y: e,
+        height: f.height,
+        currentX: f.width
+      };
+      i.push(t), o.push({
+        ...f,
+        x: 0,
+        y: e
+      }), e += f.height;
+    }
+  }
+  const v = i.length > 0 ? Math.max(...i.map((f) => f.currentX)) : 0, x = e;
+  return {
+    packed: o,
+    width: Math.max(v, d),
+    height: x
+  };
+}
+function Pt(a) {
+  if (a.length === 0) return 1;
+  const d = a.reduce((e, l) => e + l.width * l.height, 0);
+  let i = Math.ceil(Math.sqrt(d));
+  const o = Math.max(...a.map((e) => e.width));
+  return i = Math.max(i, o), i = Ot(i), i;
+}
+function Ot(a) {
+  return Math.pow(2, Math.ceil(Math.log2(a)));
+}
+const Wt = 20;
+function Gt(a, d, i = 0) {
+  const o = a.flatMap((m) => m.frames.map((u) => u.canvas)), e = Array.from(o.entries()).map(([m, u]) => ({
+    id: m,
+    // index of the frame after all the frames from all the clips have been flattened
+    width: u.width + i * 2,
+    height: u.height + i * 2
+  })), l = Nt(e), v = l.height, x = l.width, f = Math.min(1, d / x, d / v), s = Math.floor(x * f), t = Math.floor(v * f) + Wt, n = document.createElement("canvas");
+  n.width = s, n.height = t;
+  const c = n.getContext("2d");
+  c.fillStyle = "#000000", c.fillRect(0, 0, n.width, n.height);
+  const r = [];
+  for (const m of l.packed) {
+    const u = o[m.id], h = m.x + i, g = m.y + i, y = u.width, A = u.height;
+    c.drawImage(u, h * f, g * f, y * f, A * f), r[m.id] = {
+      u: h * f / s,
+      v: g * f / t,
+      w: y * f / s,
+      h: A * f / t
+    };
+  }
+  const p = [];
+  let M = 0;
+  for (const m of a) {
+    const u = {
+      ...m,
+      frames: []
+    };
+    for (const h of m.frames)
+      u.frames.push({
+        frameUV: r[M++],
+        cropUV: h.cropUV,
+        startTime: h.startTime
+      });
+    p.push(u);
+  }
+  return c.font = `${Math.max(6, Math.floor(12))}px monospace`, c.fillStyle = "#ff0000", c.fillText("Created with MeshCap : https://bandinopla.github.io/three-mediapipe-rig/?editor=meshcap", 0, t - 4), {
+    canvas: n,
+    clips: p,
+    padding: i,
+    atlasSize: d,
+    async save(m) {
+      const u = await Et(this);
+      if (m) {
+        const h = URL.createObjectURL(u), g = document.createElement("a");
+        g.href = h, g.download = "atlas.mcap", g.click(), URL.revokeObjectURL(h), g.remove();
+      }
+      return u;
+    }
+  };
+}
+export {
+  Tt as audioBufferToWav,
+  Gt as buildMeshCapAtlas,
+  zt as createAudioAtlasPlayer,
+  Bt as createMeshCapMaterial,
+  Lt as extractAudioSprites,
+  _t as loadMeshCapFile
+};

package/dist/module.d.ts

@@ -1,2 +1,3 @@
 export * from './tracking/TrackerManager';
+export * from './tracking/util/face-tracker-utils';
 //# sourceMappingURL=module.d.ts.map

package/dist/module.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"module.d.ts","sourceRoot":"","sources":["../src/module.ts"],"names":[],"mappings":"AACA,cAAc,2BAA2B,CAAA"}
+{"version":3,"file":"module.d.ts","sourceRoot":"","sources":["../src/module.ts"],"names":[],"mappings":"AACA,cAAc,2BAA2B,CAAA;AACzC,cAAc,oCAAoC,CAAA"}

package/dist/rigger.d.ts

@@ -0,0 +1,2 @@
+export * from './module'
+export {}