talking-head-studio 0.2.0

package/src/sketchfab/categories.ts

@@ -0,0 +1,127 @@
+import type { AccessoryCategory } from './types';
+
+export const ACCESSORY_CATEGORIES: AccessoryCategory[] = [
+  {
+    id: 'hair',
+    label: 'Hair',
+    type: 'skinned',
+    defaultQuery: 'anime hair 3d model',
+    quickTags: ['anime hair', 'long hair', 'short hair', 'ponytail', 'braids', 'curly hair'],
+  },
+  {
+    id: 'hat',
+    label: 'Hats',
+    type: 'rigid',
+    attach_bone: 'Head',
+    defaultQuery: 'hat cap 3d model',
+    quickTags: ['baseball cap', 'witch hat', 'top hat', 'beanie', 'crown', 'helmet'],
+  },
+  {
+    id: 'glasses',
+    label: 'Glasses',
+    type: 'rigid',
+    attach_bone: 'Head',
+    defaultQuery: 'glasses eyewear 3d model',
+    quickTags: ['sunglasses', 'round glasses', 'goggles', 'visor', 'eyeglasses', 'monocle'],
+  },
+  {
+    id: 'necklace',
+    label: 'Necklaces',
+    type: 'rigid',
+    attach_bone: 'Neck',
+    defaultQuery: 'necklace pendant 3d model',
+    quickTags: ['pendant necklace', 'chain necklace', 'choker', 'amulet', 'collar'],
+  },
+  {
+    id: 'handheld',
+    label: 'Handheld',
+    type: 'rigid',
+    attach_bone: 'RightHand',
+    defaultQuery: 'sword weapon prop 3d model',
+    quickTags: ['sword', 'wand', 'staff', 'book', 'flower', 'microphone'],
+  },
+  {
+    id: 'glove',
+    label: 'Gloves',
+    type: 'rigid',
+    attach_bone: 'LeftHand',
+    defaultQuery: 'glove hand accessory 3d model',
+    quickTags: ['gloves', 'armor gloves', 'fingerless gloves', 'boxing gloves', 'mittens', 'gauntlet'],
+  },
+  {
+    id: 'wings',
+    label: 'Wings',
+    type: 'rigid',
+    attach_bone: 'Spine2',
+    defaultQuery: 'wings 3d model fantasy',
+    quickTags: ['angel wings', 'fairy wings', 'dragon wings', 'demon wings', 'butterfly wings', 'feather wings'],
+  },
+  {
+    id: 'tail',
+    label: 'Tail',
+    type: 'rigid',
+    attach_bone: 'Hips',
+    defaultQuery: 'tail 3d model character',
+    quickTags: ['fox tail', 'cat tail', 'dragon tail', 'demon tail', 'fluffy tail', 'anime tail'],
+  },
+  {
+    id: 'cape',
+    label: 'Cape',
+    type: 'skinned',
+    attach_bone: 'Spine2',
+    defaultQuery: 'cape cloak 3d model',
+    quickTags: ['cloak', 'superhero cape', 'wizard cloak', 'fantasy cape', 'hood cloak', 'mantle'],
+  },
+  {
+    id: 'belt',
+    label: 'Belt',
+    type: 'rigid',
+    attach_bone: 'Hips',
+    defaultQuery: 'belt sash armor waist',
+    quickTags: ['leather belt', 'utility belt', 'sash', 'waist armor', 'holster', 'fantasy belt'],
+  },
+  {
+    id: 'shoulder_pad',
+    label: 'Shoulder Pads',
+    type: 'rigid',
+    attach_bone: 'LeftShoulder',
+    defaultQuery: 'pauldron armor shoulder knight',
+    quickTags: ['pauldron', 'armor shoulder', 'fantasy armor', 'sci-fi shoulder', 'knight armor', 'epaulette'],
+  },
+  {
+    id: 'top',
+    label: 'Tops',
+    type: 'skinned',
+    defaultQuery: 'shirt jacket clothing 3d model',
+    quickTags: ['shirt', 'jacket', 'hoodie', 'coat', 'vest', 'uniform'],
+  },
+  {
+    id: 'bottom',
+    label: 'Bottoms',
+    type: 'skinned',
+    defaultQuery: 'pants skirt clothing 3d model',
+    quickTags: ['pants', 'skirt', 'shorts', 'jeans', 'dress', 'leggings'],
+  },
+  {
+    id: 'footwear',
+    label: 'Footwear',
+    type: 'skinned',
+    defaultQuery: 'shoes boots footwear 3d model',
+    quickTags: ['sneakers', 'boots', 'heels', 'sandals', 'loafers', 'slippers'],
+  },
+];
+
+/** Tags indicating a humanoid / character model (for badge display in avatar browsers) */
+export const HUMANOID_TAGS = [
+  'character',
+  'humanoid',
+  'human',
+  'anime',
+  'avatar',
+  'person',
+  'figure',
+  'girl',
+  'boy',
+  'woman',
+  'man',
+] as const;

package/src/sketchfab/index.ts

@@ -0,0 +1,6 @@
+export { searchSketchfab, getDownloadUrl, downloadModel, getBestThumbnail } from './api';
+export type { SketchfabSearchOptions } from './api';
+export { useSketchfabSearch } from './useSketchfabSearch';
+export type { UseSketchfabSearchOptions, UseSketchfabSearchResult } from './useSketchfabSearch';
+export { ACCESSORY_CATEGORIES, HUMANOID_TAGS } from './categories';
+export type { SketchfabModel, SketchfabResponse, AccessoryCategory } from './types';

package/src/sketchfab/types.ts

@@ -0,0 +1,40 @@
+export interface SketchfabThumbnailImage {
+  url: string;
+  width: number;
+  height: number;
+}
+
+export interface SketchfabModel {
+  uid: string;
+  name: string;
+  description: string;
+  thumbnails: { images: SketchfabThumbnailImage[] };
+  viewerUrl: string;
+  publishedAt: string;
+  likeCount: number;
+  viewCount: number;
+  vertexCount: number;
+  faceCount: number;
+  animationCount: number;
+  tags: Array<{ name: string; slug: string }>;
+  user: { username: string; displayName: string };
+  license: { label: string } | null;
+  isDownloadable: boolean;
+}
+
+export interface SketchfabResponse {
+  results: SketchfabModel[];
+  next: string | null;
+  previous: string | null;
+  cursors?: { next: string | null; previous: string | null };
+}
+
+export interface AccessoryCategory {
+  id: string;
+  label: string;
+  /** 'rigid' = bone-attached prop; 'skinned' = bone-weight clothing */
+  type: 'rigid' | 'skinned';
+  attach_bone?: string;
+  defaultQuery: string;
+  quickTags: string[];
+}

package/src/sketchfab/useSketchfabSearch.ts

@@ -0,0 +1,110 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { searchSketchfab } from './api';
+import type { SketchfabModel } from './types';
+
+export interface UseSketchfabSearchOptions {
+  apiKey: string;
+  initialQuery?: string;
+  debounceMs?: number;
+}
+
+export interface UseSketchfabSearchResult {
+  models: SketchfabModel[];
+  isLoading: boolean;
+  error: string | null;
+  query: string;
+  setQuery: (query: string) => void;
+  hasNext: boolean;
+  hasPrev: boolean;
+  nextPage: () => void;
+  prevPage: () => void;
+}
+
+export function useSketchfabSearch({
+  apiKey,
+  initialQuery = 'character humanoid avatar',
+  debounceMs = 400,
+}: UseSketchfabSearchOptions): UseSketchfabSearchResult {
+  const [query, setQueryRaw] = useState(initialQuery);
+  const [debouncedQuery, setDebouncedQuery] = useState(initialQuery);
+  const [cursorStack, setCursorStack] = useState<string[]>([]);
+  const [currentCursor, setCurrentCursor] = useState<string | undefined>();
+  const [models, setModels] = useState<SketchfabModel[]>([]);
+  const [isLoading, setIsLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+  const [cursors, setCursors] = useState<{ next: string | null; previous: string | null }>();
+
+  const debounceTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+  const setQuery = useCallback((value: string) => {
+    setQueryRaw(value);
+    if (debounceTimer.current) {
+      clearTimeout(debounceTimer.current);
+    }
+    debounceTimer.current = setTimeout(() => {
+      setDebouncedQuery(value);
+      setCurrentCursor(undefined);
+      setCursorStack([]);
+    }, debounceMs);
+  }, [debounceMs]);
+
+  useEffect(() => {
+    let cancelled = false;
+    setIsLoading(true);
+    setError(null);
+
+    searchSketchfab({ apiKey, query: debouncedQuery, cursor: currentCursor })
+      .then((response) => {
+        if (cancelled) {
+          return;
+        }
+        setModels(response.results);
+        setCursors(response.cursors);
+      })
+      .catch((reason) => {
+        if (cancelled) {
+          return;
+        }
+        setError(reason instanceof Error ? reason.message : 'Search failed');
+      })
+      .finally(() => {
+        if (!cancelled) {
+          setIsLoading(false);
+        }
+      });
+
+    return () => {
+      cancelled = true;
+    };
+  }, [apiKey, currentCursor, debouncedQuery]);
+
+  const nextPage = useCallback(() => {
+    const nextCursor = cursors?.next ?? undefined;
+    if (!nextCursor) {
+      return;
+    }
+    setCursorStack((current) => [...current, currentCursor ?? '']);
+    setCurrentCursor(nextCursor);
+  }, [currentCursor, cursors]);
+
+  const prevPage = useCallback(() => {
+    setCursorStack((current) => {
+      const next = [...current];
+      const previousCursor = next.pop();
+      setCurrentCursor(previousCursor || undefined);
+      return next;
+    });
+  }, []);
+
+  return {
+    models,
+    isLoading,
+    error,
+    query,
+    setQuery,
+    hasNext: Boolean(cursors?.next),
+    hasPrev: cursorStack.length > 0,
+    nextPage,
+    prevPage,
+  };
+}

package/src/voice/convertToWav.ts

@@ -0,0 +1,87 @@
+/**
+ * Convert any audio blob to WAV format using the Web Audio API.
+ * Handles WebM/opus output from MediaRecorder without requiring ffmpeg.
+ */
+export async function convertToWav(audioBlob: Blob): Promise<Blob> {
+  // Already WAV — nothing to do.
+  if (audioBlob.type === 'audio/wav') return audioBlob;
+
+  const audioContext = new AudioContext();
+
+  try {
+    const arrayBuffer = await audioBlob.arrayBuffer();
+    const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+    return audioBufferToWav(audioBuffer);
+  } finally {
+    await audioContext.close();
+  }
+}
+
+/**
+ * Encode an AudioBuffer as a 16-bit PCM WAV blob.
+ */
+function audioBufferToWav(buffer: AudioBuffer): Blob {
+  const numberOfChannels = buffer.numberOfChannels;
+  const sampleRate = buffer.sampleRate;
+  const format = 1; // PCM
+  const bitDepth = 16;
+
+  const bytesPerSample = bitDepth / 8;
+  const blockAlign = numberOfChannels * bytesPerSample;
+
+  const interleaved = interleaveChannels(buffer);
+
+  const dataLength = interleaved.length * bytesPerSample;
+  const wavBuffer = new ArrayBuffer(44 + dataLength);
+  const view = new DataView(wavBuffer);
+
+  // RIFF header
+  writeString(view, 0, 'RIFF');
+  view.setUint32(4, 36 + dataLength, true);
+  writeString(view, 8, 'WAVE');
+
+  // fmt chunk
+  writeString(view, 12, 'fmt ');
+  view.setUint32(16, 16, true);              // chunk size
+  view.setUint16(20, format, true);          // PCM
+  view.setUint16(22, numberOfChannels, true);
+  view.setUint32(24, sampleRate, true);
+  view.setUint32(28, sampleRate * blockAlign, true); // byte rate
+  view.setUint16(32, blockAlign, true);
+  view.setUint16(34, bitDepth, true);
+
+  // data chunk
+  writeString(view, 36, 'data');
+  view.setUint32(40, dataLength, true);
+  floatTo16BitPCM(view, 44, interleaved);
+
+  return new Blob([wavBuffer], { type: 'audio/wav' });
+}
+
+function interleaveChannels(buffer: AudioBuffer): Float32Array {
+  const numberOfChannels = buffer.numberOfChannels;
+  const length = buffer.length;
+  const interleaved = new Float32Array(length * numberOfChannels);
+
+  for (let channel = 0; channel < numberOfChannels; channel++) {
+    const channelData = buffer.getChannelData(channel);
+    for (let i = 0; i < length; i++) {
+      interleaved[i * numberOfChannels + channel] = channelData[i];
+    }
+  }
+
+  return interleaved;
+}
+
+function writeString(view: DataView, offset: number, value: string): void {
+  for (let i = 0; i < value.length; i++) {
+    view.setUint8(offset + i, value.charCodeAt(i));
+  }
+}
+
+function floatTo16BitPCM(view: DataView, offset: number, input: Float32Array): void {
+  for (let i = 0; i < input.length; i++, offset += 2) {
+    const s = Math.max(-1, Math.min(1, input[i]));
+    view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+  }
+}

package/src/voice/index.ts

@@ -0,0 +1,7 @@
+export { useAudioRecording } from './useAudioRecording';
+export type { UseAudioRecordingOptions, UseAudioRecordingReturn } from './useAudioRecording';
+
+export { useAudioPlayer } from './useAudioPlayer';
+export type { UseAudioPlayerOptions, UseAudioPlayerReturn } from './useAudioPlayer';
+
+export { convertToWav } from './convertToWav';

package/src/voice/useAudioPlayer.ts

@@ -0,0 +1,78 @@
+import { useRef, useState } from 'react';
+
+export interface UseAudioPlayerOptions {
+  /** Called when a playback error occurs. Defaults to `console.error`. */
+  onError?: (error: Error) => void;
+}
+
+export interface UseAudioPlayerReturn {
+  isPlaying: boolean;
+  playPause: (file: File | Blob | null | undefined) => void;
+  cleanup: () => void;
+}
+
+export function useAudioPlayer({
+  onError,
+}: UseAudioPlayerOptions = {}): UseAudioPlayerReturn {
+  const [isPlaying, setIsPlaying] = useState(false);
+  const audioRef = useRef<HTMLAudioElement | null>(null);
+
+  const playPause = (file: File | Blob | null | undefined) => {
+    if (!file) return;
+
+    if (audioRef.current) {
+      if (isPlaying) {
+        audioRef.current.pause();
+        setIsPlaying(false);
+      } else {
+        audioRef.current.play();
+        setIsPlaying(true);
+      }
+    } else {
+      const audio = new Audio(URL.createObjectURL(file));
+      audioRef.current = audio;
+
+      audio.addEventListener('ended', () => {
+        setIsPlaying(false);
+        if (audioRef.current) {
+          URL.revokeObjectURL(audioRef.current.src);
+        }
+        audioRef.current = null;
+      });
+
+      audio.addEventListener('error', () => {
+        setIsPlaying(false);
+        const err = new Error('Failed to play audio file');
+        if (onError) {
+          onError(err);
+        } else {
+          console.error('[useAudioPlayer] Playback error:', err.message);
+        }
+        if (audioRef.current) {
+          URL.revokeObjectURL(audioRef.current.src);
+        }
+        audioRef.current = null;
+      });
+
+      audio.play();
+      setIsPlaying(true);
+    }
+  };
+
+  const cleanup = () => {
+    if (audioRef.current) {
+      audioRef.current.pause();
+      if (audioRef.current.src.startsWith('blob:')) {
+        URL.revokeObjectURL(audioRef.current.src);
+      }
+      audioRef.current = null;
+    }
+    setIsPlaying(false);
+  };
+
+  return {
+    isPlaying,
+    playPause,
+    cleanup,
+  };
+}

package/src/voice/useAudioRecording.ts

@@ -0,0 +1,207 @@
+import { useCallback, useEffect, useRef, useState } from 'react';
+import { convertToWav } from './convertToWav';
+
+export interface UseAudioRecordingOptions {
+  maxDurationSeconds?: number;
+  onRecordingComplete?: (blob: Blob, duration?: number) => void;
+}
+
+export interface UseAudioRecordingReturn {
+  isRecording: boolean;
+  duration: number;
+  error: string | null;
+  startRecording: () => Promise<void>;
+  stopRecording: () => void;
+  cancelRecording: () => void;
+}
+
+export function useAudioRecording({
+  maxDurationSeconds = 29,
+  onRecordingComplete,
+}: UseAudioRecordingOptions = {}): UseAudioRecordingReturn {
+  const [isRecording, setIsRecording] = useState(false);
+  const [duration, setDuration] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const chunksRef = useRef<Blob[]>([]);
+  const streamRef = useRef<MediaStream | null>(null);
+  const timerRef = useRef<number | null>(null);
+  const startTimeRef = useRef<number | null>(null);
+  const cancelledRef = useRef<boolean>(false);
+
+  const startRecording = useCallback(async () => {
+    try {
+      setError(null);
+      chunksRef.current = [];
+      cancelledRef.current = false;
+      setDuration(0);
+
+      if (typeof navigator === 'undefined') {
+        const errorMsg = 'Navigator API is not available.';
+        setError(errorMsg);
+        throw new Error(errorMsg);
+      }
+
+      if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+        // Give the webview a brief moment to initialise before failing.
+        await new Promise((resolve) => setTimeout(resolve, 100));
+
+        if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+          const errorMsg =
+            'Microphone access is not available. Please ensure you are using a ' +
+            'secure context (HTTPS or localhost) and that your browser has ' +
+            'microphone permissions enabled.';
+          setError(errorMsg);
+          throw new Error(errorMsg);
+        }
+      }
+
+      const stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          autoGainControl: true,
+        },
+      });
+
+      streamRef.current = stream;
+
+      const options: MediaRecorderOptions = {
+        mimeType: 'audio/webm;codecs=opus',
+      };
+
+      if (!MediaRecorder.isTypeSupported(options.mimeType!)) {
+        delete options.mimeType;
+      }
+
+      const mediaRecorder = new MediaRecorder(stream, options);
+      mediaRecorderRef.current = mediaRecorder;
+
+      mediaRecorder.ondataavailable = (event) => {
+        if (event.data.size > 0) {
+          chunksRef.current.push(event.data);
+        }
+      };
+
+      mediaRecorder.onstop = async () => {
+        // Snapshot the cancellation flag and recorded duration immediately —
+        // cancelRecording() clears chunks and sets cancelledRef synchronously
+        // before this async handler runs, so we must check it first.
+        const wasCancelled = cancelledRef.current;
+        const recordedDuration = startTimeRef.current
+          ? (Date.now() - startTimeRef.current) / 1000
+          : undefined;
+
+        const webmBlob = new Blob(chunksRef.current, { type: 'audio/webm' });
+
+        // Stop all tracks now that we have the data.
+        streamRef.current?.getTracks().forEach((track) => {
+          track.stop();
+        });
+        streamRef.current = null;
+
+        if (wasCancelled) return;
+
+        // Convert to WAV format to avoid needing ffmpeg on the backend.
+        try {
+          const wavBlob = await convertToWav(webmBlob);
+          onRecordingComplete?.(wavBlob, recordedDuration);
+        } catch (err) {
+          console.error('Error converting audio to WAV:', err);
+          // Fallback to original blob if conversion fails.
+          onRecordingComplete?.(webmBlob, recordedDuration);
+        }
+      };
+
+      mediaRecorder.onerror = (event) => {
+        setError('Recording error occurred');
+        console.error('MediaRecorder error:', event);
+      };
+
+      mediaRecorder.start(100); // Collect data every 100 ms.
+      setIsRecording(true);
+      startTimeRef.current = Date.now();
+
+      timerRef.current = window.setInterval(() => {
+        if (startTimeRef.current) {
+          const elapsed = (Date.now() - startTimeRef.current) / 1000;
+          setDuration(elapsed);
+
+          if (elapsed >= maxDurationSeconds) {
+            if (
+              mediaRecorderRef.current &&
+              mediaRecorderRef.current.state !== 'inactive'
+            ) {
+              mediaRecorderRef.current.stop();
+              setIsRecording(false);
+              if (timerRef.current !== null) {
+                clearInterval(timerRef.current);
+                timerRef.current = null;
+              }
+            }
+          }
+        }
+      }, 100);
+    } catch (err) {
+      const errorMessage =
+        err instanceof Error
+          ? err.message
+          : 'Failed to access microphone. Please check permissions.';
+      setError(errorMessage);
+      setIsRecording(false);
+    }
+  }, [maxDurationSeconds, onRecordingComplete]);
+
+  const stopRecording = useCallback(() => {
+    if (mediaRecorderRef.current && isRecording) {
+      mediaRecorderRef.current.stop();
+      setIsRecording(false);
+
+      if (timerRef.current !== null) {
+        clearInterval(timerRef.current);
+        timerRef.current = null;
+      }
+    }
+  }, [isRecording]);
+
+  const cancelRecording = useCallback(() => {
+    if (mediaRecorderRef.current) {
+      cancelledRef.current = true; // Must be set before stop() triggers onstop.
+      chunksRef.current = [];
+      mediaRecorderRef.current.stop();
+      setIsRecording(false);
+      setDuration(0);
+    }
+
+    streamRef.current?.getTracks().forEach((track) => {
+      track.stop();
+    });
+    streamRef.current = null;
+
+    if (timerRef.current !== null) {
+      clearInterval(timerRef.current);
+      timerRef.current = null;
+    }
+  }, []);
+
+  // Cleanup on unmount.
+  useEffect(() => {
+    return () => {
+      if (timerRef.current !== null) {
+        clearInterval(timerRef.current);
+      }
+      streamRef.current?.getTracks().forEach((track) => {
+        track.stop();
+      });
+    };
+  }, []);
+
+  return {
+    isRecording,
+    duration,
+    error,
+    startRecording,
+    stopRecording,
+    cancelRecording,
+  };
+}