talking-head-studio 0.2.6 → 0.2.7

package/src/TalkingHead.web.tsx

@@ -1,220 +0,0 @@
-import React, {
-  forwardRef,
-  useCallback,
-  useEffect,
-  useImperativeHandle,
-  useMemo,
-  useRef,
-} from 'react';
-import { type StyleProp, StyleSheet, View, type ViewStyle } from 'react-native';
-import { buildAvatarHtml } from './html';
-import type {
-  TalkingHeadVisemeCue,
-  TalkingHeadVisemeSchedule,
-} from './TalkingHead';
-
-export type { TalkingHeadVisemeCue, TalkingHeadVisemeSchedule };
-
-export type TalkingHeadMood =
-  | 'neutral'
-  | 'happy'
-  | 'sad'
-  | 'angry'
-  | 'excited'
-  | 'thinking'
-  | 'concerned'
-  | 'surprised';
-
-export interface TalkingHeadAccessory {
-  id: string;
-  url: string;
-  bone: string;
-  position: [number, number, number];
-  rotation: [number, number, number];
-  scale: number;
-}
-
-export interface TalkingHeadProps {
-  avatarUrl: string;
-  authToken?: string | null;
-  mood?: TalkingHeadMood;
-  cameraView?: 'head' | 'upper' | 'full';
-  cameraDistance?: number;
-  hairColor?: string;
-  skinColor?: string;
-  eyeColor?: string;
-  accessories?: TalkingHeadAccessory[];
-  onReady?: () => void;
-  onError?: (message: string) => void;
-  style?: StyleProp<ViewStyle>;
-}
-
-export interface TalkingHeadRef {
-  sendAmplitude: (amplitude: number) => void;
-  scheduleVisemes: (schedule: TalkingHeadVisemeSchedule) => void;
-  clearVisemes: () => void;
-  setMood: (mood: TalkingHeadMood) => void;
-  setHairColor: (color: string) => void;
-  setSkinColor: (color: string) => void;
-  setEyeColor: (color: string) => void;
-  setAccessories: (accessories: TalkingHeadAccessory[]) => void;
-}
-
-export const TalkingHead = forwardRef<TalkingHeadRef, TalkingHeadProps>(
-  (
-    {
-      avatarUrl,
-      authToken,
-      mood = 'neutral',
-      cameraView = 'upper',
-      cameraDistance = -0.5,
-      hairColor,
-      skinColor,
-      eyeColor,
-      accessories,
-      onReady,
-      onError,
-      style,
-    },
-    ref,
-  ) => {
-    const iframeRef = useRef<HTMLIFrameElement>(null);
-    const readyRef = useRef(false);
-    const accessoriesRef = useRef(accessories);
-
-    useEffect(() => {
-      accessoriesRef.current = accessories;
-    }, [accessories]);
-
-    const post = useCallback((msg: object) => {
-      iframeRef.current?.contentWindow?.postMessage(JSON.stringify(msg), '*');
-    }, []);
-
-    useImperativeHandle(
-      ref,
-      () => ({
-        sendAmplitude: (amplitude) => post({ type: 'amplitude', value: amplitude }),
-        scheduleVisemes: (schedule) => post({ type: 'schedule_visemes', schedule }),
-        clearVisemes: () => post({ type: 'clear_visemes' }),
-        setMood: (nextMood) => post({ type: 'mood', value: nextMood }),
-        setHairColor: (color) => post({ type: 'hair_color', value: color }),
-        setSkinColor: (color) => post({ type: 'skin_color', value: color }),
-        setEyeColor: (color) => post({ type: 'eye_color', value: color }),
-        setAccessories: (newAccessories) =>
-          post({ type: 'set_accessories', accessories: newAccessories }),
-      }),
-      [post],
-    );
-
-    useEffect(() => {
-      if (readyRef.current) post({ type: 'mood', value: mood });
-    }, [mood, post]);
-
-    useEffect(() => {
-      if (accessories && readyRef.current) {
-        post({ type: 'set_accessories', accessories });
-      }
-    }, [accessories, post]);
-
-    useEffect(() => {
-      if (hairColor && readyRef.current) post({ type: 'hair_color', value: hairColor });
-    }, [hairColor, post]);
-
-    useEffect(() => {
-      if (skinColor && readyRef.current) post({ type: 'skin_color', value: skinColor });
-    }, [skinColor, post]);
-
-    useEffect(() => {
-      if (eyeColor && readyRef.current) post({ type: 'eye_color', value: eyeColor });
-    }, [eyeColor, post]);
-
-    const [initialMood] = React.useState(mood);
-    const [initialHairColor] = React.useState(hairColor);
-    const [initialSkinColor] = React.useState(skinColor);
-    const [initialEyeColor] = React.useState(eyeColor);
-
-    const html = useMemo(
-      () =>
-        buildAvatarHtml({
-          avatarUrl,
-          authToken,
-          mood: initialMood,
-          cameraView,
-          cameraDistance,
-          initialHairColor,
-          initialSkinColor,
-          initialEyeColor,
-        }),
-      [
-        avatarUrl,
-        authToken,
-        cameraView,
-        cameraDistance,
-        initialMood,
-        initialHairColor,
-        initialSkinColor,
-        initialEyeColor,
-      ],
-    );
-
-    // The HTML references window.ReactNativeWebView.postMessage — we need to
-    // inject that shim so messages come back to us via window.addEventListener.
-    const srcdoc = useMemo(() => {
-      const shim = `<script>window.ReactNativeWebView = { postMessage: function(d) { window.parent.postMessage(d, '*'); } };</script>`;
-      // Insert the shim right after <head> so it's available before the module script runs
-      return html.replace('<head>', '<head>' + shim);
-    }, [html]);
-
-    useEffect(() => {
-      const onMessage = (event: MessageEvent) => {
-        // Only accept messages from our iframe
-        if (iframeRef.current && event.source !== iframeRef.current.contentWindow) return;
-        try {
-          const msg = typeof event.data === 'string' ? JSON.parse(event.data) : event.data;
-          if (msg.type === 'ready') {
-            readyRef.current = true;
-            if (accessoriesRef.current?.length) {
-              post({ type: 'set_accessories', accessories: accessoriesRef.current });
-            }
-            onReady?.();
-          } else if (msg.type === 'error') {
-            onError?.(msg.message);
-          } else if (msg.type === 'log') {
-            console.log('[TalkingHead]', msg.message);
-          }
-        } catch {
-          // ignore non-JSON messages
-        }
-      };
-      window.addEventListener('message', onMessage);
-      return () => window.removeEventListener('message', onMessage);
-    }, [onReady, onError, post]);
-
-    return (
-      <View style={[styles.container, style]}>
-        <iframe
-          ref={iframeRef as any}
-          srcDoc={srcdoc}
-          style={{
-            width: '100%',
-            height: '100%',
-            border: 'none',
-            backgroundColor: 'transparent',
-          }}
-          sandbox="allow-scripts allow-same-origin"
-          title="TalkingHead Avatar"
-        />
-      </View>
-    );
-  },
-);
-
-TalkingHead.displayName = 'TalkingHead';
-
-const styles = StyleSheet.create({
-  container: {
-    overflow: 'hidden',
-    borderRadius: 12,
-    backgroundColor: 'transparent',
-  },
-});

package/src/__tests__/TalkingHead.test.tsx

@@ -1,32 +0,0 @@
-import React from 'react';
-import { render } from '@testing-library/react-native';
-import { TalkingHead } from '../TalkingHead';
-
-// Mock react-native-webview
-jest.mock('react-native-webview', () => {
-  // eslint-disable-next-line @typescript-eslint/no-require-imports
-  const React = require('react');
-  // eslint-disable-next-line @typescript-eslint/no-require-imports
-  const { View } = require('react-native');
-  
-  const MockWebView = React.forwardRef((props: { testID?: string }, ref: React.Ref<unknown>) => {
-    React.useImperativeHandle(ref, () => ({
-      postMessage: jest.fn(),
-    }));
-    return <View {...props} />;
-  });
-  MockWebView.displayName = 'MockWebView';
-
-  return { WebView: MockWebView };
-});
-
-describe('TalkingHead', () => {
-  it('renders without crashing', () => {
-    render(
-      <TalkingHead
-        avatarUrl="https://example.com/avatar.glb"
-        style={{ flex: 1 }}
-      />
-    );
-  });
-});

package/src/__tests__/sketchfab.test.ts

@@ -1,24 +0,0 @@
-import { ACCESSORY_CATEGORIES } from '../sketchfab';
-
-describe('ACCESSORY_CATEGORIES', () => {
-  it('includes the full accessory category set used by the app browsers', () => {
-    expect(ACCESSORY_CATEGORIES.map((category) => category.id)).toEqual(
-      expect.arrayContaining([
-        'hair',
-        'hat',
-        'glasses',
-        'necklace',
-        'handheld',
-        'glove',
-        'wings',
-        'tail',
-        'cape',
-        'belt',
-        'shoulder_pad',
-        'top',
-        'bottom',
-        'footwear',
-      ]),
-    );
-  });
-});

package/src/appearance/apply.ts

@@ -1,94 +0,0 @@
-import { pickTargetForMaterialName } from './matchers';
-import { type AvatarAppearance, normalizeAppearance } from './schema';
-
-type MaterialLike = {
-  name?: string;
-  color?: {
-    set: (value: string) => void;
-  };
-};
-
-type MeshLike = {
-  isMesh?: boolean;
-  material?: MaterialLike | MaterialLike[];
-  geometry?: {
-    attributes?: {
-      position?: { count?: number };
-    };
-  };
-};
-
-type Object3DLike = {
-  traverse?: (visitor: (object: unknown) => void) => void;
-};
-
-function asMaterialArray(material: MeshLike['material']): MaterialLike[] {
-  if (!material) {
-    return [];
-  }
-
-  return Array.isArray(material) ? material : [material];
-}
-
-function getVertexCount(mesh: MeshLike): number {
-  return mesh.geometry?.attributes?.position?.count ?? 0;
-}
-
-export function applyAppearanceToObject3D(
-  object3d: Object3DLike,
-  appearance: AvatarAppearance,
-): void {
-  if (typeof object3d.traverse !== 'function') {
-    return;
-  }
-
-  const normalizedAppearance = normalizeAppearance(appearance);
-
-  // First pass: apply name-matched colors and track whether skin was matched,
-  // also track the largest mesh for fallback skin coloring.
-  let skinMatched = false;
-  let largestVertexCount = 0;
-  let largestMeshMaterial: MaterialLike | null = null;
-
-  object3d.traverse((object) => {
-    const mesh = object as MeshLike;
-    if (!mesh?.isMesh) {
-      return;
-    }
-
-    const vertexCount = getVertexCount(mesh);
-
-    for (const material of asMaterialArray(mesh.material)) {
-      if (!material || typeof material.name !== 'string') {
-        continue;
-      }
-
-      const target = pickTargetForMaterialName(material.name);
-      if (!target) {
-        // Track the largest unmatched mesh as a skin fallback candidate
-        if (vertexCount > largestVertexCount && material.color && typeof material.color.set === 'function') {
-          largestVertexCount = vertexCount;
-          largestMeshMaterial = material;
-        }
-        continue;
-      }
-
-      if (target === 'skinColor') {
-        skinMatched = true;
-      }
-
-      const color = normalizedAppearance[target];
-      if (!color || !material.color || typeof material.color.set !== 'function') {
-        continue;
-      }
-
-      material.color.set(color);
-    }
-  });
-
-  // Second pass: if no skin material was found by name, apply skin color to
-  // the largest mesh — skin is the largest organ after all.
-  if (!skinMatched && largestMeshMaterial && normalizedAppearance.skinColor) {
-    (largestMeshMaterial as MaterialLike).color!.set(normalizedAppearance.skinColor);
-  }
-}

package/src/appearance/index.ts

@@ -1,4 +0,0 @@
-export { applyAppearanceToObject3D } from './apply';
-export { pickTargetForMaterialName } from './matchers';
-export { type AvatarAppearance, normalizeAppearance } from './schema';
-export type { AppearanceTarget } from './matchers';

package/src/appearance/matchers.ts

@@ -1,43 +0,0 @@
-export type AppearanceTarget = 'hairColor' | 'skinColor' | 'eyeColor';
-
-function tokenizeMaterialName(name: string): string[] {
-  return name
-    .replace(/([a-z\d])([A-Z])/g, '$1 $2')
-    .toLowerCase()
-    .split(/[^a-z\d]+/)
-    .filter(Boolean);
-}
-
-// Exact token match: any token equals the candidate exactly
-function hasToken(tokens: string[], candidates: string[]): boolean {
-  return candidates.some((candidate) => tokens.includes(candidate));
-}
-
-// Prefix match: candidate appears at the START of a token.
-// e.g. "eye" matches token "eyeball", "eyelid", "eyebrow" but NOT "chair" (c-hair).
-// We intentionally avoid suffix/contains to prevent false positives like chair→hair.
-function hasTokenPrefix(tokens: string[], candidates: string[]): boolean {
-  return candidates.some((candidate) =>
-    tokens.some((token) => token.startsWith(candidate) && token.length > candidate.length),
-  );
-}
-
-export function pickTargetForMaterialName(name: string): AppearanceTarget | null {
-  const tokens = tokenizeMaterialName(name);
-
-  if (hasToken(tokens, ['hair', 'fur']) || hasTokenPrefix(tokens, ['hair', 'fur'])) {
-    return 'hairColor';
-  }
-
-  if (hasToken(tokens, ['skin', 'body', 'face']) || hasTokenPrefix(tokens, ['skin', 'body', 'face'])) {
-    return 'skinColor';
-  }
-
-  // "eye" prefix catches: eyeball, eyelid, eyebrow, eyelash
-  // "eye" suffix catches: lefteye, righteye
-  if (hasToken(tokens, ['eye', 'iris']) || hasTokenPrefix(tokens, ['eye', 'iris'])) {
-    return 'eyeColor';
-  }
-
-  return null;
-}

package/src/appearance/schema.ts

@@ -1,35 +0,0 @@
-export type AppearanceVersion = 1;
-
-export interface AvatarAppearance {
-  version: AppearanceVersion;
-  hairColor?: string;
-  skinColor?: string;
-  eyeColor?: string;
-}
-
-const HEX_COLOR_PATTERN = /^#(?:[\dA-Fa-f]{3}|[\dA-Fa-f]{6})$/;
-
-function normalizeHexColor(value: string): string {
-  if (!HEX_COLOR_PATTERN.test(value)) {
-    throw new Error(`Invalid color format: ${value}`);
-  }
-
-  const raw = value.slice(1);
-  const expanded =
-    raw.length === 3 ? `${raw[0]}${raw[0]}${raw[1]}${raw[1]}${raw[2]}${raw[2]}` : raw;
-
-  return `#${expanded.toUpperCase()}`;
-}
-
-export function normalizeAppearance(input: AvatarAppearance): AvatarAppearance {
-  if (input.version !== 1) {
-    throw new Error(`Unsupported appearance version: ${input.version}`);
-  }
-
-  return {
-    version: input.version,
-    hairColor: input.hairColor != null ? normalizeHexColor(input.hairColor) : undefined,
-    skinColor: input.skinColor != null ? normalizeHexColor(input.skinColor) : undefined,
-    eyeColor: input.eyeColor != null ? normalizeHexColor(input.eyeColor) : undefined,
-  };
-}

package/src/editor/AvatarCanvas.tsx

@@ -1,167 +0,0 @@
-// @ts-nocheck
-import { Environment, OrbitControls } from '@react-three/drei';
-import { Canvas, useThree } from '@react-three/fiber';
-import { Suspense, useCallback, useEffect, useRef, useState } from 'react';
-import type { Vector3 } from 'three';
-import type { AvatarAppearance } from '../appearance';
-import { AvatarCanvasErrorBoundary } from './AvatarCanvasErrorBoundary';
-import { AvatarModel } from './AvatarModel';
-import { RigidAccessory } from './RigidAccessory';
-import { SkinnedClothing } from './SkinnedClothing';
-import type { AssetPlacement, EquippedAsset } from './types';
-
-/** Captures the R3F scene object (which contains everything) and passes it up. */
-function SceneRefCapture({ onSceneRef }: { onSceneRef: (scene: any) => void }) {
-  const { scene } = useThree();
-  useEffect(() => {
-    onSceneRef(scene);
-  }, [scene, onSceneRef]);
-  return null;
-}
-
-interface AvatarCanvasProps {
-  avatarUrl: string;
-  appearance?: AvatarAppearance;
-  equipped?: EquippedAsset[];
-  placements?: Record<string, AssetPlacement>;
-  editingAssetId?: string | null;
-  onPlacementChange?: (assetId: string, placement: AssetPlacement) => void;
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  onSceneRef?: (scene: any) => void;
-  style?: React.CSSProperties;
-  className?: string;
-}
-
-export function AvatarCanvas({
-  avatarUrl,
-  appearance,
-  equipped = [],
-  placements = {},
-  editingAssetId = null,
-  onPlacementChange,
-  onSceneRef,
-  style,
-  className,
-}: AvatarCanvasProps) {
-  const [skeleton, setSkeleton] = useState<any>(null);
-  const [avatarScene, setAvatarScene] = useState<any>(null);
-  const [cameraTarget, setCameraTarget] = useState<[number, number, number]>([0, 1, 0]);
-  const [cameraPosition, setCameraPosition] = useState<[number, number, number]>([0, 1.2, 2.5]);
-  const canvasRef = useRef<HTMLCanvasElement>(null);
-
-  const handleSkeletonReady = useCallback((skel: any) => {
-    setSkeleton(skel);
-    if (skel.bones.length > 0) {
-      const root = skel.bones[0].parent ?? skel.bones[0];
-      setAvatarScene(root);
-    }
-  }, []);
-
-  const handleBoundsReady = useCallback((center: Vector3, height: number) => {
-    const dist = Math.max(height * 1.4, 1.5);
-    setCameraTarget([center.x, center.y, center.z]);
-    setCameraPosition([center.x, center.y + height * 0.1, center.z + dist]);
-  }, []);
-
-  const equippedItems = equipped.filter(Boolean);
-
-  const wrapperClass = [
-    'w-full h-full overflow-hidden relative bg-[radial-gradient(ellipse_at_center,_rgba(30,20,60,0.6)_0%,_rgba(5,5,15,0.95)_100%)] [background-size:100%_100%]',
-    className,
-  ]
-    .filter(Boolean)
-    .join(' ');
-
-  return (
-    <div className={wrapperClass} style={style}>
-      {/* Subtle grid floor */}
-      <div className="absolute inset-0 opacity-[0.04] pointer-events-none"
-        style={{ backgroundImage: 'linear-gradient(rgba(255,255,255,0.3) 1px,transparent 1px),linear-gradient(90deg,rgba(255,255,255,0.3) 1px,transparent 1px)', backgroundSize: '60px 60px' }} />
-      {avatarUrl ? (
-        <AvatarCanvasErrorBoundary>
-          <Canvas
-            ref={canvasRef}
-            camera={{ position: cameraPosition, fov: 45 }}
-            style={{ width: '100%', height: '100%' }}
-          >
-            <Suspense fallback={null}>
-              {onSceneRef && <SceneRefCapture onSceneRef={onSceneRef} />}
-              <ambientLight intensity={0.6} />
-              <directionalLight position={[5, 5, 5]} intensity={0.8} />
-              <Environment preset="studio" />
-              <OrbitControls
-                target={cameraTarget}
-                minDistance={1}
-                maxDistance={10}
-                enablePan={false}
-                makeDefault
-              />
-
-              <AvatarModel
-                url={avatarUrl}
-                appearance={appearance}
-                scale={1}
-                onSkeletonReady={handleSkeletonReady}
-                onBoundsReady={handleBoundsReady}
-              />
-
-              {equippedItems.map((asset) => {
-                if (!asset) return null;
-                const llmPlacement = placements[asset.id];
-                const isEditing = editingAssetId === asset.id;
-
-                // If a placement is provided, always render as rigid attachment
-                if (llmPlacement) {
-                  const bone = llmPlacement.bone ?? asset.attach_bone;
-                  if (!bone) return null;
-                  return (
-                    <RigidAccessory
-                      key={asset.id}
-                      assetId={asset.id}
-                      url={asset.url}
-                      avatarScene={avatarScene}
-                      attachBone={bone}
-                      offsetPosition={llmPlacement.position}
-                      offsetRotation={llmPlacement.rotation}
-                      scale={llmPlacement.scale}
-                      isEditing={isEditing}
-                      onPlacementChange={onPlacementChange}
-                    />
-                  );
-                }
-
-                if (asset.type === 'skinned') {
-                  return (
-                    <SkinnedClothing key={asset.id} url={asset.url} avatarSkeleton={skeleton} />
-                  );
-                }
-                if (asset.type === 'rigid') {
-                  const bone = asset.attach_bone;
-                  if (!bone) return null;
-                  return (
-                    <RigidAccessory
-                      key={asset.id}
-                      assetId={asset.id}
-                      url={asset.url}
-                      avatarScene={avatarScene}
-                      attachBone={bone}
-                      offsetPosition={asset.offset_position}
-                      offsetRotation={asset.offset_rotation}
-                      isEditing={isEditing}
-                      onPlacementChange={onPlacementChange}
-                    />
-                  );
-                }
-                return null;
-              })}
-            </Suspense>
-          </Canvas>
-        </AvatarCanvasErrorBoundary>
-      ) : (
-        <div className="flex items-center justify-center h-full text-muted-foreground">
-          <p>Select a base avatar to get started</p>
-        </div>
-      )}
-    </div>
-  );
-}

package/src/editor/AvatarCanvasErrorBoundary.tsx

@@ -1,64 +0,0 @@
-import { Component, type ErrorInfo, type ReactNode } from 'react';
-
-interface AvatarCanvasErrorBoundaryProps {
-  children: ReactNode;
-}
-
-interface AvatarCanvasErrorBoundaryState {
-  hasError: boolean;
-  retryKey: number;
-}
-
-export class AvatarCanvasErrorBoundary extends Component<
-  AvatarCanvasErrorBoundaryProps,
-  AvatarCanvasErrorBoundaryState
-> {
-  state: AvatarCanvasErrorBoundaryState = {
-    hasError: false,
-    retryKey: 0,
-  };
-
-  componentDidCatch(error: Error, errorInfo: ErrorInfo): void {
-    console.error('[AvatarCanvas] WebGL error', error, errorInfo.componentStack);
-  }
-
-  static getDerivedStateFromError(): AvatarCanvasErrorBoundaryState {
-    return {
-      hasError: true,
-      retryKey: 0,
-    };
-  }
-
-  private handleRetry = () => {
-    this.setState((prev) => ({
-      hasError: false,
-      retryKey: prev.retryKey + 1,
-    }));
-  };
-
-  render() {
-    if (this.state.hasError) {
-      return (
-        <div className="flex h-full w-full flex-col items-center justify-center gap-3 p-4 text-center">
-          <p className="text-sm font-medium text-foreground">3D rendering unavailable</p>
-          <p className="text-xs text-muted-foreground">
-            Your device or browser cannot initialize WebGL.
-          </p>
-          <button
-            className="rounded-md border px-3 py-1.5 text-xs font-medium hover:bg-muted"
-            onClick={this.handleRetry}
-            type="button"
-          >
-            Retry
-          </button>
-        </div>
-      );
-    }
-
-    return (
-      <div className="h-full w-full" key={this.state.retryKey}>
-        {this.props.children}
-      </div>
-    );
-  }
-}