talking-head-studio 0.2.0

package/src/appearance/matchers.ts

@@ -0,0 +1,43 @@
+export type AppearanceTarget = 'hairColor' | 'skinColor' | 'eyeColor';
+
+function tokenizeMaterialName(name: string): string[] {
+  return name
+    .replace(/([a-z\d])([A-Z])/g, '$1 $2')
+    .toLowerCase()
+    .split(/[^a-z\d]+/)
+    .filter(Boolean);
+}
+
+// Exact token match: any token equals the candidate exactly
+function hasToken(tokens: string[], candidates: string[]): boolean {
+  return candidates.some((candidate) => tokens.includes(candidate));
+}
+
+// Prefix match: candidate appears at the START of a token.
+// e.g. "eye" matches token "eyeball", "eyelid", "eyebrow" but NOT "chair" (c-hair).
+// We intentionally avoid suffix/contains to prevent false positives like chair→hair.
+function hasTokenPrefix(tokens: string[], candidates: string[]): boolean {
+  return candidates.some((candidate) =>
+    tokens.some((token) => token.startsWith(candidate) && token.length > candidate.length),
+  );
+}
+
+export function pickTargetForMaterialName(name: string): AppearanceTarget | null {
+  const tokens = tokenizeMaterialName(name);
+
+  if (hasToken(tokens, ['hair', 'fur']) || hasTokenPrefix(tokens, ['hair', 'fur'])) {
+    return 'hairColor';
+  }
+
+  if (hasToken(tokens, ['skin', 'body', 'face']) || hasTokenPrefix(tokens, ['skin', 'body', 'face'])) {
+    return 'skinColor';
+  }
+
+  // "eye" prefix catches: eyeball, eyelid, eyebrow, eyelash
+  // "eye" suffix catches: lefteye, righteye
+  if (hasToken(tokens, ['eye', 'iris']) || hasTokenPrefix(tokens, ['eye', 'iris'])) {
+    return 'eyeColor';
+  }
+
+  return null;
+}

package/src/appearance/schema.ts

@@ -0,0 +1,35 @@
+export type AppearanceVersion = 1;
+
+export interface AvatarAppearance {
+  version: AppearanceVersion;
+  hairColor?: string;
+  skinColor?: string;
+  eyeColor?: string;
+}
+
+const HEX_COLOR_PATTERN = /^#(?:[\dA-Fa-f]{3}|[\dA-Fa-f]{6})$/;
+
+function normalizeHexColor(value: string): string {
+  if (!HEX_COLOR_PATTERN.test(value)) {
+    throw new Error(`Invalid color format: ${value}`);
+  }
+
+  const raw = value.slice(1);
+  const expanded =
+    raw.length === 3 ? `${raw[0]}${raw[0]}${raw[1]}${raw[1]}${raw[2]}${raw[2]}` : raw;
+
+  return `#${expanded.toUpperCase()}`;
+}
+
+export function normalizeAppearance(input: AvatarAppearance): AvatarAppearance {
+  if (input.version !== 1) {
+    throw new Error(`Unsupported appearance version: ${input.version}`);
+  }
+
+  return {
+    version: input.version,
+    hairColor: input.hairColor != null ? normalizeHexColor(input.hairColor) : undefined,
+    skinColor: input.skinColor != null ? normalizeHexColor(input.skinColor) : undefined,
+    eyeColor: input.eyeColor != null ? normalizeHexColor(input.eyeColor) : undefined,
+  };
+}

package/src/editor/AvatarCanvas.tsx

@@ -0,0 +1,167 @@
+// @ts-nocheck
+import { Environment, OrbitControls } from '@react-three/drei';
+import { Canvas, useThree } from '@react-three/fiber';
+import { Suspense, useCallback, useEffect, useRef, useState } from 'react';
+import type { Vector3 } from 'three';
+import type { AvatarAppearance } from '../appearance';
+import { AvatarCanvasErrorBoundary } from './AvatarCanvasErrorBoundary';
+import { AvatarModel } from './AvatarModel';
+import { RigidAccessory } from './RigidAccessory';
+import { SkinnedClothing } from './SkinnedClothing';
+import type { AssetPlacement, EquippedAsset } from './types';
+
+/** Captures the R3F scene object (which contains everything) and passes it up. */
+function SceneRefCapture({ onSceneRef }: { onSceneRef: (scene: any) => void }) {
+  const { scene } = useThree();
+  useEffect(() => {
+    onSceneRef(scene);
+  }, [scene, onSceneRef]);
+  return null;
+}
+
+interface AvatarCanvasProps {
+  avatarUrl: string;
+  appearance?: AvatarAppearance;
+  equipped?: EquippedAsset[];
+  placements?: Record<string, AssetPlacement>;
+  editingAssetId?: string | null;
+  onPlacementChange?: (assetId: string, placement: AssetPlacement) => void;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  onSceneRef?: (scene: any) => void;
+  style?: React.CSSProperties;
+  className?: string;
+}
+
+export function AvatarCanvas({
+  avatarUrl,
+  appearance,
+  equipped = [],
+  placements = {},
+  editingAssetId = null,
+  onPlacementChange,
+  onSceneRef,
+  style,
+  className,
+}: AvatarCanvasProps) {
+  const [skeleton, setSkeleton] = useState<any>(null);
+  const [avatarScene, setAvatarScene] = useState<any>(null);
+  const [cameraTarget, setCameraTarget] = useState<[number, number, number]>([0, 1, 0]);
+  const [cameraPosition, setCameraPosition] = useState<[number, number, number]>([0, 1.2, 2.5]);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+
+  const handleSkeletonReady = useCallback((skel: any) => {
+    setSkeleton(skel);
+    if (skel.bones.length > 0) {
+      const root = skel.bones[0].parent ?? skel.bones[0];
+      setAvatarScene(root);
+    }
+  }, []);
+
+  const handleBoundsReady = useCallback((center: Vector3, height: number) => {
+    const dist = Math.max(height * 1.4, 1.5);
+    setCameraTarget([center.x, center.y, center.z]);
+    setCameraPosition([center.x, center.y + height * 0.1, center.z + dist]);
+  }, []);
+
+  const equippedItems = equipped.filter(Boolean);
+
+  const wrapperClass = [
+    'w-full h-full overflow-hidden relative bg-[radial-gradient(ellipse_at_center,_rgba(30,20,60,0.6)_0%,_rgba(5,5,15,0.95)_100%)] [background-size:100%_100%]',
+    className,
+  ]
+    .filter(Boolean)
+    .join(' ');
+
+  return (
+    <div className={wrapperClass} style={style}>
+      {/* Subtle grid floor */}
+      <div className="absolute inset-0 opacity-[0.04] pointer-events-none"
+        style={{ backgroundImage: 'linear-gradient(rgba(255,255,255,0.3) 1px,transparent 1px),linear-gradient(90deg,rgba(255,255,255,0.3) 1px,transparent 1px)', backgroundSize: '60px 60px' }} />
+      {avatarUrl ? (
+        <AvatarCanvasErrorBoundary>
+          <Canvas
+            ref={canvasRef}
+            camera={{ position: cameraPosition, fov: 45 }}
+            style={{ width: '100%', height: '100%' }}
+          >
+            <Suspense fallback={null}>
+              {onSceneRef && <SceneRefCapture onSceneRef={onSceneRef} />}
+              <ambientLight intensity={0.6} />
+              <directionalLight position={[5, 5, 5]} intensity={0.8} />
+              <Environment preset="studio" />
+              <OrbitControls
+                target={cameraTarget}
+                minDistance={1}
+                maxDistance={10}
+                enablePan={false}
+                makeDefault
+              />
+
+              <AvatarModel
+                url={avatarUrl}
+                appearance={appearance}
+                scale={1}
+                onSkeletonReady={handleSkeletonReady}
+                onBoundsReady={handleBoundsReady}
+              />
+
+              {equippedItems.map((asset) => {
+                if (!asset) return null;
+                const llmPlacement = placements[asset.id];
+                const isEditing = editingAssetId === asset.id;
+
+                // If a placement is provided, always render as rigid attachment
+                if (llmPlacement) {
+                  const bone = llmPlacement.bone ?? asset.attach_bone;
+                  if (!bone) return null;
+                  return (
+                    <RigidAccessory
+                      key={asset.id}
+                      assetId={asset.id}
+                      url={asset.url}
+                      avatarScene={avatarScene}
+                      attachBone={bone}
+                      offsetPosition={llmPlacement.position}
+                      offsetRotation={llmPlacement.rotation}
+                      scale={llmPlacement.scale}
+                      isEditing={isEditing}
+                      onPlacementChange={onPlacementChange}
+                    />
+                  );
+                }
+
+                if (asset.type === 'skinned') {
+                  return (
+                    <SkinnedClothing key={asset.id} url={asset.url} avatarSkeleton={skeleton} />
+                  );
+                }
+                if (asset.type === 'rigid') {
+                  const bone = asset.attach_bone;
+                  if (!bone) return null;
+                  return (
+                    <RigidAccessory
+                      key={asset.id}
+                      assetId={asset.id}
+                      url={asset.url}
+                      avatarScene={avatarScene}
+                      attachBone={bone}
+                      offsetPosition={asset.offset_position}
+                      offsetRotation={asset.offset_rotation}
+                      isEditing={isEditing}
+                      onPlacementChange={onPlacementChange}
+                    />
+                  );
+                }
+                return null;
+              })}
+            </Suspense>
+          </Canvas>
+        </AvatarCanvasErrorBoundary>
+      ) : (
+        <div className="flex items-center justify-center h-full text-muted-foreground">
+          <p>Select a base avatar to get started</p>
+        </div>
+      )}
+    </div>
+  );
+}

package/src/editor/AvatarCanvasErrorBoundary.tsx

@@ -0,0 +1,64 @@
+import { Component, type ErrorInfo, type ReactNode } from 'react';
+
+interface AvatarCanvasErrorBoundaryProps {
+  children: ReactNode;
+}
+
+interface AvatarCanvasErrorBoundaryState {
+  hasError: boolean;
+  retryKey: number;
+}
+
+export class AvatarCanvasErrorBoundary extends Component<
+  AvatarCanvasErrorBoundaryProps,
+  AvatarCanvasErrorBoundaryState
+> {
+  state: AvatarCanvasErrorBoundaryState = {
+    hasError: false,
+    retryKey: 0,
+  };
+
+  componentDidCatch(error: Error, errorInfo: ErrorInfo): void {
+    console.error('[AvatarCanvas] WebGL error', error, errorInfo.componentStack);
+  }
+
+  static getDerivedStateFromError(): AvatarCanvasErrorBoundaryState {
+    return {
+      hasError: true,
+      retryKey: 0,
+    };
+  }
+
+  private handleRetry = () => {
+    this.setState((prev) => ({
+      hasError: false,
+      retryKey: prev.retryKey + 1,
+    }));
+  };
+
+  render() {
+    if (this.state.hasError) {
+      return (
+        <div className="flex h-full w-full flex-col items-center justify-center gap-3 p-4 text-center">
+          <p className="text-sm font-medium text-foreground">3D rendering unavailable</p>
+          <p className="text-xs text-muted-foreground">
+            Your device or browser cannot initialize WebGL.
+          </p>
+          <button
+            className="rounded-md border px-3 py-1.5 text-xs font-medium hover:bg-muted"
+            onClick={this.handleRetry}
+            type="button"
+          >
+            Retry
+          </button>
+        </div>
+      );
+    }
+
+    return (
+      <div className="h-full w-full" key={this.state.retryKey}>
+        {this.props.children}
+      </div>
+    );
+  }
+}

package/src/editor/AvatarModel.tsx

@@ -0,0 +1,49 @@
+// @ts-nocheck
+import { useGLTF } from '@react-three/drei';
+import { useEffect, useRef } from 'react';
+import { Box3, Vector3 } from 'three';
+import { applyAppearanceToObject3D } from '../appearance';
+import type { AvatarAppearance } from '../appearance';
+
+interface AvatarModelProps {
+  url: string;
+  appearance?: AvatarAppearance;
+  scale?: number;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  onSkeletonReady?: (skeleton: any) => void;
+  onBoundsReady?: (center: typeof Vector3.prototype, height: number) => void;
+}
+
+export function AvatarModel({
+  url,
+  appearance,
+  scale = 1,
+  onSkeletonReady,
+  onBoundsReady,
+}: AvatarModelProps) {
+  const { scene } = useGLTF(url);
+  const groupRef = useRef(null);
+
+  useEffect(() => {
+    if (!scene) return;
+    scene.traverse((child: any) => {
+      if (child.isSkinnedMesh && child.skeleton) {
+        onSkeletonReady?.(child.skeleton);
+      }
+    });
+    if (onBoundsReady) {
+      const box = new Box3().setFromObject(scene);
+      const center = new Vector3();
+      box.getCenter(center);
+      const height = box.max.y - box.min.y;
+      onBoundsReady(center, height);
+    }
+  }, [scene, onSkeletonReady, onBoundsReady]);
+
+  useEffect(() => {
+    if (!scene) return;
+    applyAppearanceToObject3D(scene, appearance ?? { version: 1 as const });
+  }, [scene, appearance]);
+
+  return <primitive ref={groupRef} object={scene} scale={scale} />;
+}

package/src/editor/RigidAccessory.tsx

@@ -0,0 +1,130 @@
+// @ts-nocheck
+import { useGLTF, PivotControls } from '@react-three/drei';
+import { useEffect, useMemo, useState } from 'react';
+import { Box3, Vector3 } from 'three';
+import type { AssetPlacement } from './types';
+
+interface RigidAccessoryProps {
+  assetId: string;
+  url: string;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  avatarScene: any;
+  attachBone: string;
+  offsetPosition?: [number, number, number];
+  offsetRotation?: [number, number, number];
+  scale?: number;
+  isEditing?: boolean;
+  onPlacementChange?: (assetId: string, placement: AssetPlacement) => void;
+}
+
+export function RigidAccessory({
+  assetId,
+  url,
+  avatarScene,
+  attachBone,
+  offsetPosition,
+  offsetRotation,
+  scale = 1,
+  isEditing = false,
+  onPlacementChange,
+}: RigidAccessoryProps) {
+  const { scene } = useGLTF(url);
+
+  // Clone uniquely for this instance
+  const clone = useMemo(() => scene.clone(true), [scene]);
+  const [basePos, setBasePos] = useState<typeof Vector3.prototype | null>(null);
+
+  useEffect(() => {
+    if (!avatarScene) return;
+
+    let targetBone: any = null;
+    avatarScene.traverse((child: any) => {
+      if (child.isBone && child.name === attachBone) {
+        targetBone = child;
+      }
+    });
+
+    if (!targetBone) return;
+
+    const box = new Box3().setFromObject(clone);
+    const meshMin = box.min;
+    const meshCenter = new Vector3();
+    box.getCenter(meshCenter);
+
+    const boneWorldPos = new Vector3();
+    targetBone.getWorldPosition(boneWorldPos);
+
+    setBasePos(new Vector3(
+      boneWorldPos.x - meshCenter.x,
+      boneWorldPos.y - meshMin.y,
+      boneWorldPos.z - meshCenter.z,
+    ));
+  }, [avatarScene, attachBone, clone]);
+
+  // Apply transforms live
+  useEffect(() => {
+    if (!basePos) return;
+
+    clone.position.copy(basePos);
+
+    if (offsetPosition) {
+      clone.position.x += offsetPosition[0];
+      clone.position.y += offsetPosition[1];
+      clone.position.z += offsetPosition[2];
+    }
+
+    if (offsetRotation) {
+      clone.rotation.set(...offsetRotation);
+    }
+
+    if (scale !== undefined) {
+      clone.scale.set(scale, scale, scale);
+    }
+  }, [basePos, offsetPosition, offsetRotation, scale, clone]);
+
+  const handleDragEnd = () => {
+    if (!basePos) return;
+
+    // The PivotControls autoTransform updates the object directly
+    const newOffsetPos: [number, number, number] = [
+      clone.position.x - basePos.x,
+      clone.position.y - basePos.y,
+      clone.position.z - basePos.z,
+    ];
+
+    const newOffsetRot: [number, number, number] = [
+      clone.rotation.x,
+      clone.rotation.y,
+      clone.rotation.z,
+    ];
+
+    const currentScale = clone.scale.x;
+
+    onPlacementChange?.(assetId, {
+      bone: attachBone,
+      position: newOffsetPos,
+      rotation: newOffsetRot,
+      scale: currentScale,
+    });
+  };
+
+  if (!basePos) return null;
+
+  if (isEditing) {
+    return (
+      <PivotControls
+        anchor={[0, 0, 0]}
+        scale={100}
+        fixed={true}
+        depthTest={false}
+        activeAxes={[true, true, true]}
+        onDragEnd={handleDragEnd}
+        autoTransform={true}
+      >
+        <primitive object={clone} />
+      </PivotControls>
+    );
+  }
+
+  return <primitive object={clone} />;
+}

package/src/editor/SkinnedClothing.tsx

@@ -0,0 +1,114 @@
+// @ts-nocheck
+import { useGLTF } from '@react-three/drei';
+import { useEffect, useRef } from 'react';
+import * as THREE from 'three';
+import * as SkeletonUtils from 'three/examples/jsm/utils/SkeletonUtils.js';
+
+interface SkinnedClothingProps {
+  url: string;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  avatarSkeleton: any;
+}
+
+/**
+ * Rebind a skinned mesh's bones to the avatar skeleton by matching bone names.
+ * This handles hair packs / clothing that share the same rig convention but
+ * were exported as separate files with their own skeleton.
+ */
+function rebindToAvatarSkeleton(mesh: THREE.SkinnedMesh, avatarSkeleton: THREE.Skeleton) {
+  const avatarBonesByName = new Map<string, THREE.Bone>();
+  avatarSkeleton.bones.forEach((bone: THREE.Bone) => avatarBonesByName.set(bone.name, bone));
+
+  // Build a new bone array for this mesh by matching names
+  const newBones: THREE.Bone[] = [];
+  const oldBones = mesh.skeleton.bones;
+  const boneIndexMap = new Map<number, number>(); // old index → new index
+
+  oldBones.forEach((oldBone: THREE.Bone, oldIdx: number) => {
+    const matched = avatarBonesByName.get(oldBone.name);
+    if (matched) {
+      boneIndexMap.set(oldIdx, newBones.length);
+      newBones.push(matched);
+    }
+  });
+
+  if (newBones.length === 0) {
+    // No bone names matched at all — just bind directly and hope for the best
+    mesh.bind(avatarSkeleton);
+    return;
+  }
+
+  // Remap skinIndex buffer to point at the new bone indices
+  const skinIndex = mesh.geometry.attributes.skinIndex;
+  const remappedIndex = skinIndex.array.slice();
+
+  for (let i = 0; i < remappedIndex.length; i++) {
+    const oldIdx = remappedIndex[i];
+    const newIdx = boneIndexMap.get(oldIdx);
+    remappedIndex[i] = newIdx !== undefined ? newIdx : 0;
+  }
+
+  mesh.geometry.setAttribute(
+    'skinIndex',
+    new THREE.Uint16BufferAttribute(remappedIndex, skinIndex.itemSize),
+  );
+
+  // Build inverse bind matrices for the matched bones
+  const inverseBindMatrices: number[] = [];
+  newBones.forEach((bone: THREE.Bone) => {
+    const m = new THREE.Matrix4();
+    // Use the avatar skeleton's inverse bind matrix if bone index aligns,
+    // otherwise compute from world matrix
+    const avatarIdx = avatarSkeleton.bones.indexOf(bone);
+    if (avatarIdx >= 0 && avatarSkeleton.boneInverses[avatarIdx]) {
+      m.copy(avatarSkeleton.boneInverses[avatarIdx]);
+    } else {
+      m.copy(bone.matrixWorld).invert();
+    }
+    inverseBindMatrices.push(...m.elements);
+  });
+
+  const newSkeleton = new THREE.Skeleton(
+    newBones,
+    newBones.map((_: THREE.Bone, i: number) => {
+      const m = new THREE.Matrix4();
+      m.fromArray(inverseBindMatrices, i * 16);
+      return m;
+    }),
+  );
+
+  mesh.bind(newSkeleton);
+}
+
+export function SkinnedClothing({ url, avatarSkeleton }: SkinnedClothingProps) {
+  const { scene } = useGLTF(url);
+  const groupRef = useRef<any>(null);
+
+  useEffect(() => {
+    const group = groupRef.current;
+    if (!group || !scene || !avatarSkeleton) return;
+
+    const clone = SkeletonUtils.clone(scene as any);
+
+    (clone as any).traverse((child: any) => {
+      if (child.isSkinnedMesh) {
+        rebindToAvatarSkeleton(child as THREE.SkinnedMesh, avatarSkeleton);
+      }
+    });
+
+    group.add(clone as any);
+
+    return () => {
+      group.remove(clone as any);
+      (clone as any).traverse((child: any) => {
+        if (child.geometry) child.geometry.dispose();
+        if (child.material) {
+          const materials = Array.isArray(child.material) ? child.material : [child.material];
+          materials.forEach((m: any) => m.dispose());
+        }
+      });
+    };
+  }, [scene, avatarSkeleton]);
+
+  return <group ref={groupRef} />;
+}

package/src/editor/index.ts

@@ -0,0 +1,5 @@
+export { AvatarCanvas } from './AvatarCanvas';
+export { AvatarModel } from './AvatarModel';
+export { RigidAccessory } from './RigidAccessory';
+export { SkinnedClothing } from './SkinnedClothing';
+export type { AvatarEditorProps, AssetPlacement, EquippedAsset } from './types';

package/src/editor/r3f-shim.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Minimal JSX intrinsic shims for React Three Fiber elements.
+ * These are provided at runtime by @react-three/fiber (a peer dependency).
+ * This file prevents TS errors when the peer dep types are not installed.
+ */
+import type React from 'react';
+
+declare module 'react' {
+  namespace JSX {
+    interface IntrinsicElements {
+      // R3F primitives
+      primitive: {
+        object: any;
+        ref?: React.Ref<any>;
+        scale?: any;
+        [key: string]: any;
+      };
+      group: {
+        ref?: React.Ref<any>;
+        [key: string]: any;
+      };
+      // Three.js light intrinsics
+      ambientLight: {
+        intensity?: number;
+        [key: string]: any;
+      };
+      directionalLight: {
+        position?: [number, number, number];
+        intensity?: number;
+        [key: string]: any;
+      };
+    }
+  }
+}

package/src/editor/types.ts

@@ -0,0 +1,30 @@
+import type { AvatarAppearance } from '../appearance';
+import type React from 'react';
+
+export interface AssetPlacement {
+  bone: string;
+  position: [number, number, number];
+  rotation: [number, number, number];
+  scale?: number;
+}
+
+export interface EquippedAsset {
+  id: string;
+  url: string;
+  type: 'rigid' | 'skinned';
+  slot?: string;
+  attach_bone?: string;
+  offset_position?: [number, number, number];
+  offset_rotation?: [number, number, number];
+}
+
+export interface AvatarEditorProps {
+  avatarUrl: string;
+  appearance?: AvatarAppearance;
+  equipped?: EquippedAsset[];
+  placements?: Record<string, AssetPlacement>;
+  editingAssetId?: string | null;
+  onPlacementChange?: (assetId: string, placement: AssetPlacement) => void;
+  className?: string;
+  style?: React.CSSProperties;
+}