talking-head-studio 0.4.11 → 0.4.12

package/dist/html/visemes.js

@@ -0,0 +1,348 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildVisemeScript = buildVisemeScript;
+/**
+ * Generated avatar WebView visemes script section.
+ *
+ * Kept as a string builder so html.ts can compose the final document without
+ * owning every browser-side subsystem inline.
+ */
+function buildVisemeScript() {
+    return `let amplitudeDecay = 0;
+let jawMorphCache = null;
+let visemeMorphCache = null;
+
+// Voice-driven mood detection
+let voiceEnergySmoothed = 0;
+let voiceMoodCurrent = 'neutral';
+let voiceMoodFramesAbove = 0;
+let voiceMoodFramesBelow = 0;
+const VOICE_ENERGY_EXCITED_THRESH = 0.45;
+const VOICE_ENERGY_NEUTRAL_THRESH = 0.15;
+const VOICE_MOOD_FRAMES_REQUIRED = 12; // ~0.2s at 60fps
+function updateVoiceMood(rawAmplitude) {
+  voiceEnergySmoothed = voiceEnergySmoothed * 0.85 + rawAmplitude * 0.15;
+  if (voiceEnergySmoothed > VOICE_ENERGY_EXCITED_THRESH) {
+    voiceMoodFramesAbove++;
+    voiceMoodFramesBelow = 0;
+    if (voiceMoodCurrent !== 'excited' && voiceMoodFramesAbove >= VOICE_MOOD_FRAMES_REQUIRED) {
+      voiceMoodCurrent = 'excited';
+      rnPost(JSON.stringify({ type: 'voiceMood', mood: 'excited' }));
+    }
+  } else if (voiceEnergySmoothed < VOICE_ENERGY_NEUTRAL_THRESH) {
+    voiceMoodFramesBelow++;
+    voiceMoodFramesAbove = 0;
+    if (voiceMoodCurrent !== 'neutral' && voiceMoodFramesBelow >= VOICE_MOOD_FRAMES_REQUIRED * 3) {
+      voiceMoodCurrent = 'neutral';
+      rnPost(JSON.stringify({ type: 'voiceMood', mood: 'neutral' }));
+    }
+  }
+}
+const visemeState = {};
+
+const VISEME_MORPH_ALIASES = {
+  sil: ['viseme_sil', 'sil', 'mouthClose', 'mouth_close'],
+  PP:  ['viseme_PP',  'pp',  'viseme_pp', 'mouthPucker', 'mouth_pucker'],
+  FF:  ['viseme_FF',  'ff',  'viseme_ff', 'mouthLowerLipIn', 'mouth_lower_lip_in', 'mouthRollLower', 'mouthShrugLower'],
+  TH:  ['viseme_TH',  'th',  'viseme_th', 'tongueOut', 'tongue_out'],
+  DD:  ['viseme_DD',  'dd',  'viseme_dd', 'mouthShrugUpper', 'mouth_shrug_upper'],
+  kk:  ['viseme_kk',  'kk',  'viseme_k',  'mouthStretchLeft', 'mouth_stretch_left'],
+  CH:  ['viseme_CH',  'ch',  'viseme_ch', 'mouthSmile', 'mouth_smile', 'mouthSmileLeft', 'mouth_smile_left'],
+  SS:  ['viseme_SS',  'ss',  'viseme_ss', 'mouthStretchRight', 'mouth_stretch_right'],
+  nn:  ['viseme_nn',  'nn',  'viseme_n',  'mouthDimpleLeft', 'mouth_dimple_left'],
+  RR:  ['viseme_RR',  'rr',  'viseme_r',  'mouthDimpleRight', 'mouth_dimple_right'],
+  aa:  ['viseme_aa',  'viseme_AA', 'aa', 'jawOpen', 'jaw_open', 'jawopen', 'mouthOpen', 'mouth_open', 'mouthopen'],
+  ee:  ['viseme_ee',  'viseme_E',  'ee', 'mouthSmileLeft', 'mouth_smile_left'],
+  ih:  ['viseme_ih',  'viseme_I',  'ih', 'mouthSmileRight', 'mouth_smile_right'],
+  oh:  ['viseme_oh',  'viseme_O',  'oh', 'mouthFunnel', 'mouth_funnel'],
+  ou:  ['viseme_ou',  'viseme_U',  'ou', 'mouthRollLower', 'mouth_roll_lower'],
+};
+
+// For ARKit models, each viseme may need multiple blend shapes driven together.
+// Each entry is a list of morph names to combine for that viseme.
+// The first alias list that has ANY match on the model is used.
+const VISEME_COMPOUND_ARKIT = {
+  aa:  [['jawOpen', 'mouthLowerDownLeft', 'mouthLowerDownRight'], ['jawOpen', 'mouthOpen']],
+  oh:  [['mouthFunnel', 'jawOpen'], ['mouthFunnel']],
+  ou:  [['mouthPucker', 'mouthRollLower'], ['mouthPucker']],
+  PP:  [['mouthPucker', 'mouthClose'], ['mouthPucker']],
+  FF:  [['mouthRollLower', 'mouthLowerDownLeft', 'mouthLowerDownRight'], ['mouthShrugLower']],
+  CH:  [['mouthSmileLeft', 'mouthSmileRight', 'mouthStretchLeft', 'mouthStretchRight'], ['mouthSmileLeft', 'mouthSmileRight']],
+  ee:  [['mouthSmileLeft', 'mouthSmileRight'], ['mouthSmileLeft']],
+  ih:  [['mouthSmileLeft', 'mouthSmileRight'], ['mouthSmileRight']],
+};
+
+function buildVisemeMorphCache() {
+  visemeMorphCache = {};
+  for (const [visemeKey, aliases] of Object.entries(VISEME_MORPH_ALIASES)) {
+    const entries = [];
+    // Check if we have a compound ARKit mapping for this viseme
+    const compoundOptions = VISEME_COMPOUND_ARKIT[visemeKey];
+    if (compoundOptions) {
+      // Try each compound option; use the first one where all names exist on at least one mesh
+      let usedCompound = false;
+      for (const nameList of compoundOptions) {
+        // Collect entries for all names across all meshes
+        const compoundEntries = [];
+        for (const mesh of mouthMeshes) {
+          if (!mesh.morphTargetDictionary) continue;
+          const dict = mesh.morphTargetDictionary;
+          const dictKeysLower = Object.fromEntries(Object.keys(dict).map(k => [k.toLowerCase(), k]));
+          for (const name of nameList) {
+            const found = dictKeysLower[name.toLowerCase()];
+            if (found !== undefined) {
+              compoundEntries.push({ influences: mesh.morphTargetInfluences, idx: dict[found], morphName: found });
+            }
+          }
+        }
+        if (compoundEntries.length > 0) {
+          entries.push(...compoundEntries);
+          usedCompound = true;
+          break;
+        }
+      }
+      if (usedCompound) {
+        visemeMorphCache[visemeKey] = entries;
+        continue;
+      }
+    }
+    // Fallback: single-alias lookup
+    for (const mesh of mouthMeshes) {
+      if (!mesh.morphTargetDictionary) continue;
+      const dictKeys = Object.keys(mesh.morphTargetDictionary);
+      for (const alias of aliases) {
+        const found = dictKeys.find(k => k.toLowerCase() === alias.toLowerCase());
+        if (found !== undefined) {
+          entries.push({ influences: mesh.morphTargetInfluences, idx: mesh.morphTargetDictionary[found], morphName: found });
+          break;
+        }
+      }
+    }
+    if (entries.length > 0) visemeMorphCache[visemeKey] = entries;
+  }
+  const found = Object.keys(visemeMorphCache);
+  log('Viseme cache: ' + (found.length > 0 ? found.join(', ') : 'none — check morph target names'));
+
+  // Always log all available morphs so we can see what the model actually has
+  if (mouthMeshes.length > 0) {
+    const allMorphs = Object.keys(mouthMeshes[0].morphTargetDictionary || {});
+    log('Available morphs: ' + (allMorphs.length > 0 ? allMorphs.join(', ') : 'none'));
+  }
+}
+
+function applyViseme(visemeKey, weight) {
+  if (!visemeMorphCache) buildVisemeMorphCache();
+  if (visemeKey === 'sil' || weight <= 0) {
+    for (const key of Object.keys(visemeState)) visemeState[key] = 0;
+    return;
+  }
+  visemeState[visemeKey] = Math.min(1, weight);
+  visemeStateLastSet.set(visemeKey, Date.now());
+}
+
+const RHUBARB_DEFAULT_VISEME_WEIGHT = 0.72;
+const RHUBARB_LABIAL_VISEME_WEIGHT = 0.85;
+const RHUBARB_AA_VISEME_WEIGHT = 0.72;
+const RHUBARB_ROUNDED_VISEME_WEIGHT = 0.62;
+const RHUBARB_FALLBACK_AMPLITUDE_CAP = 0.72;
+const RHUBARB_FALLBACK_AMPLITUDE_GAIN = 0.75;
+const RHUBARB_VISEME_WEIGHTS = {
+  PP: RHUBARB_LABIAL_VISEME_WEIGHT,
+  FF: 0.78,
+  ee: 0.72,
+  ih: 0.68,
+  oh: RHUBARB_ROUNDED_VISEME_WEIGHT,
+  ou: 0.58,
+  aa: RHUBARB_AA_VISEME_WEIGHT,
+};
+
+const RHUBARB_TO_VISEME = {
+  A: 'aa',
+  B: 'PP',
+  C: 'ih',
+  D: 'FF',
+  E: 'ee',
+  F: 'oh',
+  G: 'ou',
+  H: 'nn',
+  X: 'sil',
+};
+
+let rhubarbMorphCache = null;
+let visemeTimers = [];
+let activeVisemeScheduleId = 0;
+let visemeModeUntil = 0;
+const visemeStateLastSet = new Map();
+
+function buildRhubarbMorphCache() {
+  if (!visemeMorphCache) buildVisemeMorphCache();
+  rhubarbMorphCache = {};
+  for (const [rhubarbShape, visemeKey] of Object.entries(RHUBARB_TO_VISEME)) {
+    if (visemeKey === 'sil') { rhubarbMorphCache[rhubarbShape] = null; continue; }
+    rhubarbMorphCache[rhubarbShape] = visemeMorphCache[visemeKey] || null;
+  }
+  log('Rhubarb morph cache built');
+}
+
+function applyRhubarbCue(shape) {
+  if (!rhubarbMorphCache) buildRhubarbMorphCache();
+  // Zero all active viseme channels first
+  for (const key of Object.keys(visemeState)) visemeState[key] = 0;
+  if (shape === 'X' || !rhubarbMorphCache[shape]) return;
+  const visemeKey = RHUBARB_TO_VISEME[shape];
+  if (visemeKey && visemeKey !== 'sil') {
+    visemeState[visemeKey] = RHUBARB_VISEME_WEIGHTS[visemeKey] || RHUBARB_DEFAULT_VISEME_WEIGHT;
+    visemeStateLastSet.set(visemeKey, Date.now());
+  }
+}
+
+function clearScheduledVisemes() {
+  activeVisemeScheduleId++;
+  for (const id of visemeTimers) clearTimeout(id);
+  visemeTimers = [];
+  visemeModeUntil = 0;
+  // Zero all mouth morphs
+  for (const key of Object.keys(visemeState)) visemeState[key] = 0;
+}
+
+function tickVisemeDecay(deltaSeconds) {
+  if (!visemeMorphCache) return;
+
+  const isScheduled = Date.now() < visemeModeUntil;
+  const hasSpecificLipShape =
+    visemeState.PP > 0.05 ||
+    visemeState.FF > 0.05 ||
+    visemeState.kk > 0.05 ||
+    visemeState.ee > 0.05 ||
+    visemeState.ih > 0.05;
+
+  for (const [key, weight] of Object.entries(visemeState)) {
+    // Only decay if we aren't in the middle of a viseme schedule.
+    // Scheduled visemes are cleared manually by timeouts.
+    if (!isScheduled) {
+      // Time-delta-aware decay: maintain consistent feel regardless of frame rate.
+      // Base rate is calibrated for 60 fps (0.82 per frame = ~12 frames to 10%).
+      // pow(0.82, delta*60) is frame-rate independent.
+      const dt = deltaSeconds ?? (1 / 60);
+      const decayFactor = Math.pow(0.82, dt * 60);
+      const decayed = weight * decayFactor;
+      visemeState[key] = decayed < 0.01 ? 0 : decayed;
+    }
+
+    const entries = visemeMorphCache[key];
+    if (!entries) continue;
+
+    let targetWeight = visemeState[key];
+    if (key === 'aa' && hasSpecificLipShape) targetWeight = Math.min(targetWeight, 0.45);
+
+    for (const e of entries) {
+      // When TalkingHead is active, write through its morph API so the internal
+      // render loop doesn't overwrite our values every frame.
+      // Use realtime (not newvalue) — newvalue is consumed and cleared after
+      // a single frame, so scheduled visemes would vanish immediately.
+      // realtime persists until explicitly set to null.
+      if (head && head.mtAvatar && e.morphName && head.mtAvatar[e.morphName]) {
+        const mt = head.mtAvatar[e.morphName];
+        mt.realtime = targetWeight > 0 ? targetWeight : null;
+        mt.needsUpdate = true;
+      } else {
+        e.influences[e.idx] = targetWeight;
+      }
+    }
+  }
+}
+
+function scheduleVisemes(schedule) {
+  clearScheduledVisemes();
+
+  // Prune visemeState keys that haven't been written in the last 2 seconds to
+  // prevent unbounded accumulation across many utterances.
+  const staleThreshold = Date.now() - 2000;
+  for (const key of Object.keys(visemeState)) {
+    if ((visemeStateLastSet.get(key) ?? 0) < staleThreshold) {
+      delete visemeState[key];
+      visemeStateLastSet.delete(key);
+    }
+  }
+
+  if (!schedule || !Array.isArray(schedule.cues) || schedule.cues.length === 0) return;
+
+  const myScheduleId = activeVisemeScheduleId;
+  // Anchor selection priority:
+  //   1. audioStartedAtMs — stamped when audio actually begins playing (most accurate)
+  //   2. startedAtMs + pipeline delay — stamped at TTS request fire time
+  //
+  // AUDIO_PIPELINE_DELAY_MS compensates for the gap between "TTS request fired"
+  // and "audio audible from speaker". Qwen3-TTS on local/tailnet is ~80–150 ms;
+  // LiveKit adds ~50–80 ms of jitter buffer on top. 150 ms is conservative but
+  // avoids the mouth running ahead of audio on fast connections.
+  const AUDIO_PIPELINE_DELAY_MS = 50;
+  let startedAt = schedule.audioStartedAtMs
+    ?? ((schedule.startedAtMs || Date.now()) + AUDIO_PIPELINE_DELAY_MS);
+  const durationMs = schedule.durationMs || 0;
+  const now = Date.now();
+  let elapsedMs = Math.max(0, now - startedAt);
+
+  // If the schedule still arrives late after the pipeline offset, shift further
+  if (elapsedMs > 300 && schedule.cues.length > 3) {
+    const shift = Math.min(elapsedMs - 50, 500);
+    startedAt += shift;
+    elapsedMs -= shift;
+    log('Viseme schedule arrived late, shifting anchor forward by ' + shift + 'ms');
+  }
+
+  const remainingMs = Math.max(0, durationMs - elapsedMs);
+  let scheduledCueCount = 0;
+  let skippedCueCount = 0;
+
+  // Gate amplitude fallback for the locally remaining duration plus a small buffer.
+  // If the schedule arrives a bit late, keep amplitude out of the way for the rest
+  // of the utterance instead of expiring immediately from the original timestamp.
+  visemeModeUntil = now + remainingMs + 200;
+
+  for (const cue of schedule.cues) {
+    const delay = cue.startMs - (Date.now() - startedAt);
+    if (delay < -50) {
+      skippedCueCount++;
+      continue; // already in the past, skip
+    }
+    scheduledCueCount++;
+
+    const applyId = setTimeout(() => {
+      if (activeVisemeScheduleId !== myScheduleId) return;
+      applyRhubarbCue(cue.viseme);
+    }, Math.max(0, delay));
+
+    const clearId = setTimeout(() => {
+      if (activeVisemeScheduleId !== myScheduleId) return;
+      // Only clear if the next cue hasn't already overwritten state
+      visemeState[RHUBARB_TO_VISEME[cue.viseme]] = 0;
+    }, Math.max(0, delay + (cue.endMs - cue.startMs)));
+
+    visemeTimers.push(applyId, clearId);
+  }
+
+  log(
+    'Viseme schedule received: requestId=' +
+      (schedule.requestId || 'unknown') +
+      ' cues=' + schedule.cues.length +
+      ' scheduled=' + scheduledCueCount +
+      ' skipped=' + skippedCueCount +
+      ' elapsedMs=' + elapsedMs +
+      ' remainingMs=' + remainingMs,
+  );
+
+  // Ensure silence at schedule end
+  const endDelay = durationMs - (Date.now() - startedAt);
+  if (endDelay > 0) {
+    visemeTimers.push(setTimeout(() => {
+      if (activeVisemeScheduleId !== myScheduleId) return;
+      for (const key of Object.keys(visemeState)) visemeState[key] = 0;
+      visemeModeUntil = 0;
+    }, endDelay + 100));
+  }
+}
+// ============ END RHUBARB VISEME SCHEDULER ============
+
+`;
+}

package/dist/html.d.ts

@@ -1,7 +1,7 @@
 export type AvatarConfig = {
     avatarUrl: string;
     authToken?: string | null;
-    mood: 'neutral' | 'happy' | 'sad' | 'angry' | 'excited' | 'thinking' | 'concerned' | 'surprised';
+    mood: 'neutral' | 'happy' | 'sad' | 'angry' | 'fear' | 'disgust' | 'love' | 'sleep' | 'excited' | 'thinking' | 'concerned' | 'surprised';
     cameraView: 'head' | 'upper' | 'full';
     cameraDistance: number;
     /** Initial colors only — live updates go via postMessage (setHairColor etc.) */