talking-head-studio 0.2.0

package/package.json

@@ -0,0 +1,120 @@
+{
+  "name": "talking-head-studio",
+  "version": "0.2.0",
+  "description": "Cross-platform 3D avatar component for React Native & web — lip-sync, gestures, accessories, and LLM integration. Powered by TalkingHead + Three.js.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "source": "src/index.ts",
+  "exports": {
+    ".": {
+      "react-native": "./dist/index.js",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.web.js"
+    },
+    "./editor": {
+      "types": "./dist/editor/index.d.ts",
+      "default": "./dist/editor/index.js"
+    },
+    "./voice": {
+      "types": "./dist/voice/index.d.ts",
+      "default": "./dist/voice/index.js"
+    },
+    "./appearance": {
+      "types": "./dist/appearance/index.d.ts",
+      "default": "./dist/appearance/index.js"
+    },
+    "./sketchfab": {
+      "types": "./dist/sketchfab/index.d.ts",
+      "default": "./dist/sketchfab/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "build": "tsc --project tsconfig.json",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint 'src/**/*.{ts,tsx}'",
+    "format": "prettier --write 'src/**/*.{ts,tsx}'",
+    "test": "jest",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "react-native",
+    "avatar",
+    "3d",
+    "talking-head",
+    "lipsync",
+    "tts",
+    "threejs",
+    "glb",
+    "lip-sync",
+    "voice",
+    "webview",
+    "expo",
+    "cross-platform",
+    "llm",
+    "motion",
+    "gesture",
+    "face-tracking",
+    "avatar-editor",
+    "gizmo",
+    "accessories",
+    "audio-recording",
+    "react-three-fiber",
+    "self-hosted"
+  ],
+  "author": "SiteBay <hello@sitebay.org>",
+  "license": "MIT",
+  "homepage": "https://github.com/nichochar/avatar-studio",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/nichochar/avatar-studio.git"
+  },
+  "bugs": {
+    "url": "https://github.com/nichochar/avatar-studio/issues"
+  },
+  "sideEffects": false,
+  "peerDependencies": {
+    "react": ">=18",
+    "react-native": ">=0.73",
+    "react-native-webview": ">=13",
+    "@react-three/fiber": ">=8",
+    "@react-three/drei": ">=9",
+    "three": ">=0.170"
+  },
+  "peerDependenciesMeta": {
+    "react-native": { "optional": true },
+    "react-native-webview": { "optional": true },
+    "@react-three/fiber": { "optional": true },
+    "@react-three/drei": { "optional": true },
+    "three": { "optional": true }
+  },
+  "devDependencies": {
+    "@babel/core": "^7.29.0",
+    "@babel/preset-env": "^7.29.0",
+    "@babel/preset-typescript": "^7.28.5",
+    "@react-native/babel-preset": "^0.84.1",
+    "@testing-library/react-native": "^13.3.3",
+    "@types/jest": "^30.0.0",
+    "@types/react": "^19.2.14",
+    "@types/react-native": "^0.73.0",
+    "@typescript-eslint/eslint-plugin": "^8.56.1",
+    "@typescript-eslint/parser": "^8.56.1",
+    "babel-jest": "^30.2.0",
+    "eslint": "^9.39.3",
+    "eslint-config-prettier": "^10.1.8",
+    "eslint-plugin-react": "^7.37.5",
+    "eslint-plugin-react-hooks": "^7.0.1",
+    "eslint-plugin-react-native": "^5.0.0",
+    "express": "^5.2.1",
+    "jest": "^30.2.0",
+    "jest-environment-jsdom": "^30.2.0",
+    "metro-react-native-babel-preset": "^0.77.0",
+    "multer": "^2.1.0",
+    "prettier": "^3.8.1",
+    "react-test-renderer": "^19.2.4",
+    "ts-jest": "^29.4.6"
+  }
+}

package/src/TalkingHead.tsx

@@ -0,0 +1,207 @@
+import React, {
+  forwardRef,
+  useCallback,
+  useEffect,
+  useImperativeHandle,
+  useMemo,
+  useRef,
+} from 'react';
+import { type StyleProp, StyleSheet, View, type ViewStyle } from 'react-native';
+import { WebView, type WebViewMessageEvent } from 'react-native-webview';
+import { buildAvatarHtml } from './html';
+
+export type TalkingHeadMood =
+  | 'neutral'
+  | 'happy'
+  | 'sad'
+  | 'angry'
+  | 'excited'
+  | 'thinking'
+  | 'concerned'
+  | 'surprised';
+
+export interface TalkingHeadAccessory {
+  id: string;
+  url: string;
+  bone: string;
+  position: [number, number, number];
+  rotation: [number, number, number];
+  scale: number;
+}
+
+export interface TalkingHeadProps {
+  avatarUrl: string;
+  authToken?: string | null;
+  mood?: TalkingHeadMood;
+  cameraView?: 'head' | 'upper' | 'full';
+  cameraDistance?: number;
+  hairColor?: string;
+  skinColor?: string;
+  eyeColor?: string;
+  accessories?: TalkingHeadAccessory[];
+  onReady?: () => void;
+  onError?: (message: string) => void;
+  style?: StyleProp<ViewStyle>;
+}
+
+export interface TalkingHeadRef {
+  sendAmplitude: (amplitude: number) => void;
+  setMood: (mood: TalkingHeadMood) => void;
+  setHairColor: (color: string) => void;
+  setSkinColor: (color: string) => void;
+  setEyeColor: (color: string) => void;
+  setAccessories: (accessories: TalkingHeadAccessory[]) => void;
+}
+
+export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
+  (
+    {
+      avatarUrl,
+      authToken,
+      mood = 'neutral',
+      cameraView = 'upper',
+      cameraDistance = -0.5,
+      hairColor,
+      skinColor,
+      eyeColor,
+      accessories,
+      onReady,
+      onError,
+      style,
+    },
+    ref,
+  ) => {
+    const webViewRef = useRef<WebView>(null);
+
+    const post = useCallback((msg: object) => {
+      webViewRef.current?.postMessage(JSON.stringify(msg));
+    }, []);
+
+    useImperativeHandle(
+      ref,
+      () => ({
+        sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
+        setMood: (nextMood) => post({ type: 'mood', value: nextMood }),
+        setHairColor: (color) => post({ type: 'hair_color', value: color }),
+        setSkinColor: (color) => post({ type: 'skin_color', value: color }),
+        setEyeColor: (color) => post({ type: 'eye_color', value: color }),
+        setAccessories: (newAccessories) =>
+          post({ type: 'set_accessories', accessories: newAccessories }),
+      }),
+      [post],
+    );
+
+    useEffect(() => {
+      if (readyRef.current) post({ type: 'mood', value: mood });
+    }, [mood, post]);
+
+    // Track whether the WebView JS is ready to receive messages
+    const readyRef = useRef(false);
+    // Always hold the latest accessories so the ready handler can send them
+    const accessoriesRef = useRef(accessories);
+    useEffect(() => {
+      accessoriesRef.current = accessories;
+    }, [accessories]);
+
+    useEffect(() => {
+      // Only post if the WebView is already ready; otherwise the ready handler sends them
+      if (accessories && readyRef.current) {
+        post({ type: 'set_accessories', accessories });
+      }
+    }, [accessories, post]);
+
+    useEffect(() => {
+      if (hairColor && readyRef.current) post({ type: 'hair_color', value: hairColor });
+    }, [hairColor, post]);
+
+    useEffect(() => {
+      if (skinColor && readyRef.current) post({ type: 'skin_color', value: skinColor });
+    }, [skinColor, post]);
+
+    useEffect(() => {
+      if (eyeColor && readyRef.current) post({ type: 'eye_color', value: eyeColor });
+    }, [eyeColor, post]);
+
+    // Color props are intentionally excluded from deps — live updates go via postMessage.
+    // Only avatarUrl, authToken, cameraView, cameraDistance cause a full WebView reload.
+    const [initialMood] = React.useState(mood);
+    const [initialHairColor] = React.useState(hairColor);
+    const [initialSkinColor] = React.useState(skinColor);
+    const [initialEyeColor] = React.useState(eyeColor);
+
+    const html = useMemo(
+      () =>
+        buildAvatarHtml({
+          avatarUrl,
+          authToken,
+          mood: initialMood,
+          cameraView,
+          cameraDistance,
+          initialHairColor: initialHairColor,
+          initialSkinColor: initialSkinColor,
+          initialEyeColor: initialEyeColor,
+        }),
+      [
+        avatarUrl,
+        authToken,
+        cameraView,
+        cameraDistance,
+        initialMood,
+        initialHairColor,
+        initialSkinColor,
+        initialEyeColor,
+      ],
+    );
+
+    const onMessage = useCallback(
+      (event: WebViewMessageEvent) => {
+        try {
+          const msg = JSON.parse(event.nativeEvent.data);
+          if (msg.type === 'ready') {
+            readyRef.current = true;
+            // Flush any accessories that arrived before the WebView was ready
+            if (accessoriesRef.current?.length) {
+              post({ type: 'set_accessories', accessories: accessoriesRef.current });
+            }
+            onReady?.();
+          } else if (msg.type === 'error') onError?.(msg.message);
+          else if (msg.type === 'log') console.log('[TalkingHead]', msg.message);
+        } catch (err) {
+          console.warn('[TalkingHead] Invalid message received from WebView:', err);
+        }
+      },
+      [onReady, onError, post],
+    );
+
+    return (
+      <View style={[styles.container, style]}>
+        <WebView
+          ref={webViewRef}
+          source={{ html }}
+          style={styles.webview}
+          javaScriptEnabled
+          domStorageEnabled
+          allowsInlineMediaPlayback
+          mediaPlaybackRequiresUserAction={false}
+          onMessage={onMessage}
+          originWhitelist={['*']}
+          mixedContentMode="always"
+        />
+      </View>
+    );
+  },
+);
+
+TalkingHead.displayName = 'TalkingHead';
+
+const styles = StyleSheet.create({
+  container: {
+    overflow: 'hidden',
+    borderRadius: 12,
+    backgroundColor: 'transparent',
+  },
+  webview: {
+    flex: 1,
+    backgroundColor: 'transparent',
+  },
+});

package/src/TalkingHead.web.tsx

@@ -0,0 +1,210 @@
+import React, {
+  forwardRef,
+  useCallback,
+  useEffect,
+  useImperativeHandle,
+  useMemo,
+  useRef,
+} from 'react';
+import { type StyleProp, StyleSheet, View, type ViewStyle } from 'react-native';
+import { buildAvatarHtml } from './html';
+
+export type TalkingHeadMood =
+  | 'neutral'
+  | 'happy'
+  | 'sad'
+  | 'angry'
+  | 'excited'
+  | 'thinking'
+  | 'concerned'
+  | 'surprised';
+
+export interface TalkingHeadAccessory {
+  id: string;
+  url: string;
+  bone: string;
+  position: [number, number, number];
+  rotation: [number, number, number];
+  scale: number;
+}
+
+export interface TalkingHeadProps {
+  avatarUrl: string;
+  authToken?: string | null;
+  mood?: TalkingHeadMood;
+  cameraView?: 'head' | 'upper' | 'full';
+  cameraDistance?: number;
+  hairColor?: string;
+  skinColor?: string;
+  eyeColor?: string;
+  accessories?: TalkingHeadAccessory[];
+  onReady?: () => void;
+  onError?: (message: string) => void;
+  style?: StyleProp<ViewStyle>;
+}
+
+export interface TalkingHeadRef {
+  sendAmplitude: (amplitude: number) => void;
+  setMood: (mood: TalkingHeadMood) => void;
+  setHairColor: (color: string) => void;
+  setSkinColor: (color: string) => void;
+  setEyeColor: (color: string) => void;
+  setAccessories: (accessories: TalkingHeadAccessory[]) => void;
+}
+
+export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
+  (
+    {
+      avatarUrl,
+      authToken,
+      mood = 'neutral',
+      cameraView = 'upper',
+      cameraDistance = -0.5,
+      hairColor,
+      skinColor,
+      eyeColor,
+      accessories,
+      onReady,
+      onError,
+      style,
+    },
+    ref,
+  ) => {
+    const iframeRef = useRef<HTMLIFrameElement>(null);
+    const readyRef = useRef(false);
+    const accessoriesRef = useRef(accessories);
+
+    useEffect(() => {
+      accessoriesRef.current = accessories;
+    }, [accessories]);
+
+    const post = useCallback((msg: object) => {
+      iframeRef.current?.contentWindow?.postMessage(JSON.stringify(msg), '*');
+    }, []);
+
+    useImperativeHandle(
+      ref,
+      () => ({
+        sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
+        setMood: (nextMood) => post({ type: 'mood', value: nextMood }),
+        setHairColor: (color) => post({ type: 'hair_color', value: color }),
+        setSkinColor: (color) => post({ type: 'skin_color', value: color }),
+        setEyeColor: (color) => post({ type: 'eye_color', value: color }),
+        setAccessories: (newAccessories) =>
+          post({ type: 'set_accessories', accessories: newAccessories }),
+      }),
+      [post],
+    );
+
+    useEffect(() => {
+      if (readyRef.current) post({ type: 'mood', value: mood });
+    }, [mood, post]);
+
+    useEffect(() => {
+      if (accessories && readyRef.current) {
+        post({ type: 'set_accessories', accessories });
+      }
+    }, [accessories, post]);
+
+    useEffect(() => {
+      if (hairColor && readyRef.current) post({ type: 'hair_color', value: hairColor });
+    }, [hairColor, post]);
+
+    useEffect(() => {
+      if (skinColor && readyRef.current) post({ type: 'skin_color', value: skinColor });
+    }, [skinColor, post]);
+
+    useEffect(() => {
+      if (eyeColor && readyRef.current) post({ type: 'eye_color', value: eyeColor });
+    }, [eyeColor, post]);
+
+    const [initialMood] = React.useState(mood);
+    const [initialHairColor] = React.useState(hairColor);
+    const [initialSkinColor] = React.useState(skinColor);
+    const [initialEyeColor] = React.useState(eyeColor);
+
+    const html = useMemo(
+      () =>
+        buildAvatarHtml({
+          avatarUrl,
+          authToken,
+          mood: initialMood,
+          cameraView,
+          cameraDistance,
+          initialHairColor,
+          initialSkinColor,
+          initialEyeColor,
+        }),
+      [
+        avatarUrl,
+        authToken,
+        cameraView,
+        cameraDistance,
+        initialMood,
+        initialHairColor,
+        initialSkinColor,
+        initialEyeColor,
+      ],
+    );
+
+    // The HTML references window.ReactNativeWebView.postMessage — we need to
+    // inject that shim so messages come back to us via window.addEventListener.
+    const srcdoc = useMemo(() => {
+      const shim = `<script>window.ReactNativeWebView = { postMessage: function(d) { window.parent.postMessage(d, '*'); } };</script>`;
+      // Insert the shim right after <head> so it's available before the module script runs
+      return html.replace('<head>', '<head>' + shim);
+    }, [html]);
+
+    useEffect(() => {
+      const onMessage = (event: MessageEvent) => {
+        // Only accept messages from our iframe
+        if (iframeRef.current && event.source !== iframeRef.current.contentWindow) return;
+        try {
+          const msg = typeof event.data === 'string' ? JSON.parse(event.data) : event.data;
+          if (msg.type === 'ready') {
+            readyRef.current = true;
+            if (accessoriesRef.current?.length) {
+              post({ type: 'set_accessories', accessories: accessoriesRef.current });
+            }
+            onReady?.();
+          } else if (msg.type === 'error') {
+            onError?.(msg.message);
+          } else if (msg.type === 'log') {
+            console.log('[TalkingHead]', msg.message);
+          }
+        } catch {
+          // ignore non-JSON messages
+        }
+      };
+      window.addEventListener('message', onMessage);
+      return () => window.removeEventListener('message', onMessage);
+    }, [onReady, onError, post]);
+
+    return (
+      <View style={[styles.container, style]}>
+        <iframe
+          ref={iframeRef as any}
+          srcDoc={srcdoc}
+          style={{
+            width: '100%',
+            height: '100%',
+            border: 'none',
+            backgroundColor: 'transparent',
+          }}
+          sandbox="allow-scripts allow-same-origin"
+          title="TalkingHead Avatar"
+        />
+      </View>
+    );
+  },
+);
+
+TalkingHead.displayName = 'TalkingHead';
+
+const styles = StyleSheet.create({
+  container: {
+    overflow: 'hidden',
+    borderRadius: 12,
+    backgroundColor: 'transparent',
+  },
+});

package/src/__tests__/TalkingHead.test.tsx

@@ -0,0 +1,32 @@
+import React from 'react';
+import { render } from '@testing-library/react-native';
+import { TalkingHead } from '../TalkingHead';
+
+// Mock react-native-webview
+jest.mock('react-native-webview', () => {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  const React = require('react');
+  // eslint-disable-next-line @typescript-eslint/no-require-imports
+  const { View } = require('react-native');
+  
+  const MockWebView = React.forwardRef((props: { testID?: string }, ref: React.Ref<unknown>) => {
+    React.useImperativeHandle(ref, () => ({
+      postMessage: jest.fn(),
+    }));
+    return <View {...props} />;
+  });
+  MockWebView.displayName = 'MockWebView';
+
+  return { WebView: MockWebView };
+});
+
+describe('TalkingHead', () => {
+  it('renders without crashing', () => {
+    render(
+      <TalkingHead
+        avatarUrl="https://example.com/avatar.glb"
+        style={{ flex: 1 }}
+      />
+    );
+  });
+});

package/src/__tests__/sketchfab.test.ts

@@ -0,0 +1,24 @@
+import { ACCESSORY_CATEGORIES } from '../sketchfab';
+
+describe('ACCESSORY_CATEGORIES', () => {
+  it('includes the full accessory category set used by the app browsers', () => {
+    expect(ACCESSORY_CATEGORIES.map((category) => category.id)).toEqual(
+      expect.arrayContaining([
+        'hair',
+        'hat',
+        'glasses',
+        'necklace',
+        'handheld',
+        'glove',
+        'wings',
+        'tail',
+        'cape',
+        'belt',
+        'shoulder_pad',
+        'top',
+        'bottom',
+        'footwear',
+      ]),
+    );
+  });
+});

package/src/appearance/apply.ts

@@ -0,0 +1,94 @@
+import { pickTargetForMaterialName } from './matchers';
+import { type AvatarAppearance, normalizeAppearance } from './schema';
+
+type MaterialLike = {
+  name?: string;
+  color?: {
+    set: (value: string) => void;
+  };
+};
+
+type MeshLike = {
+  isMesh?: boolean;
+  material?: MaterialLike | MaterialLike[];
+  geometry?: {
+    attributes?: {
+      position?: { count?: number };
+    };
+  };
+};
+
+type Object3DLike = {
+  traverse?: (visitor: (object: unknown) => void) => void;
+};
+
+function asMaterialArray(material: MeshLike['material']): MaterialLike[] {
+  if (!material) {
+    return [];
+  }
+
+  return Array.isArray(material) ? material : [material];
+}
+
+function getVertexCount(mesh: MeshLike): number {
+  return mesh.geometry?.attributes?.position?.count ?? 0;
+}
+
+export function applyAppearanceToObject3D(
+  object3d: Object3DLike,
+  appearance: AvatarAppearance,
+): void {
+  if (typeof object3d.traverse !== 'function') {
+    return;
+  }
+
+  const normalizedAppearance = normalizeAppearance(appearance);
+
+  // First pass: apply name-matched colors and track whether skin was matched,
+  // also track the largest mesh for fallback skin coloring.
+  let skinMatched = false;
+  let largestVertexCount = 0;
+  let largestMeshMaterial: MaterialLike | null = null;
+
+  object3d.traverse((object) => {
+    const mesh = object as MeshLike;
+    if (!mesh?.isMesh) {
+      return;
+    }
+
+    const vertexCount = getVertexCount(mesh);
+
+    for (const material of asMaterialArray(mesh.material)) {
+      if (!material || typeof material.name !== 'string') {
+        continue;
+      }
+
+      const target = pickTargetForMaterialName(material.name);
+      if (!target) {
+        // Track the largest unmatched mesh as a skin fallback candidate
+        if (vertexCount > largestVertexCount && material.color && typeof material.color.set === 'function') {
+          largestVertexCount = vertexCount;
+          largestMeshMaterial = material;
+        }
+        continue;
+      }
+
+      if (target === 'skinColor') {
+        skinMatched = true;
+      }
+
+      const color = normalizedAppearance[target];
+      if (!color || !material.color || typeof material.color.set !== 'function') {
+        continue;
+      }
+
+      material.color.set(color);
+    }
+  });
+
+  // Second pass: if no skin material was found by name, apply skin color to
+  // the largest mesh — skin is the largest organ after all.
+  if (!skinMatched && largestMeshMaterial && normalizedAppearance.skinColor) {
+    (largestMeshMaterial as MaterialLike).color!.set(normalizedAppearance.skinColor);
+  }
+}

package/src/appearance/index.ts

@@ -0,0 +1,4 @@
+export { applyAppearanceToObject3D } from './apply';
+export { pickTargetForMaterialName } from './matchers';
+export { type AvatarAppearance, normalizeAppearance } from './schema';
+export type { AppearanceTarget } from './matchers';