varg.ai-sdk 0.1.0 → 0.4.0-alpha.1

package/src/studio/stages.ts

@@ -0,0 +1,251 @@
+import type { VargElement, VargNode } from "../react/types";
+
+export type StageType = "image" | "video" | "animate" | "speech" | "music";
+
+export interface RenderStage {
+  id: string;
+  type: StageType;
+  label: string;
+  element: VargElement;
+  path: string[]; // path in the tree for identification
+  dependsOn: string[]; // stage ids this depends on
+  status: "pending" | "running" | "complete" | "error";
+  result?: StageResult;
+}
+
+export interface StageResult {
+  type: "image" | "video" | "audio";
+  path: string;
+  previewUrl?: string;
+  mimeType: string;
+}
+
+export interface ExtractedStages {
+  stages: RenderStage[];
+  order: string[]; // topologically sorted stage ids
+}
+
+/**
+ * Extracts all renderable stages from a JSX tree in dependency order.
+ * Images come before videos that use them, etc.
+ */
+export function extractStages(element: VargElement): ExtractedStages {
+  const stages: RenderStage[] = [];
+  let stageCounter = 0;
+
+  function generateId(): string {
+    return `stage-${stageCounter++}`;
+  }
+
+  function getLabel(type: StageType, element: VargElement): string {
+    const props = element.props as Record<string, unknown>;
+
+    if (type === "image") {
+      const prompt = props.prompt;
+      if (typeof prompt === "string") {
+        return `image: ${prompt.slice(0, 30)}${prompt.length > 30 ? "..." : ""}`;
+      }
+      if (prompt && typeof prompt === "object" && "text" in prompt) {
+        const text = (prompt as { text?: string }).text ?? "";
+        return `image: ${text.slice(0, 30)}${text.length > 30 ? "..." : ""}`;
+      }
+      if (props.src) {
+        return `image: ${String(props.src).split("/").pop()}`;
+      }
+      return "image";
+    }
+
+    if (type === "video") {
+      const prompt = props.prompt;
+      if (typeof prompt === "string") {
+        return `video: ${prompt.slice(0, 30)}${prompt.length > 30 ? "..." : ""}`;
+      }
+      if (prompt && typeof prompt === "object" && "text" in prompt) {
+        const text = (prompt as { text?: string }).text ?? "";
+        return `video: ${text.slice(0, 30)}${text.length > 30 ? "..." : ""}`;
+      }
+      if (props.src) {
+        return `video: ${String(props.src).split("/").pop()}`;
+      }
+      return "video";
+    }
+
+    if (type === "animate") {
+      const motion = props.motion;
+      return motion ? `animate: ${motion}` : "animate";
+    }
+
+    if (type === "speech") {
+      const text = getTextContent(element.children);
+      return `speech: ${text.slice(0, 30)}${text.length > 30 ? "..." : ""}`;
+    }
+
+    if (type === "music") {
+      const prompt = props.prompt;
+      if (typeof prompt === "string") {
+        return `music: ${prompt.slice(0, 30)}${prompt.length > 30 ? "..." : ""}`;
+      }
+      if (props.src) {
+        return `music: ${String(props.src).split("/").pop()}`;
+      }
+      return "music";
+    }
+
+    return type;
+  }
+
+  function getTextContent(children: VargNode[]): string {
+    const texts: string[] = [];
+    for (const child of children) {
+      if (typeof child === "string") {
+        texts.push(child);
+      } else if (typeof child === "number") {
+        texts.push(String(child));
+      }
+    }
+    return texts.join(" ");
+  }
+
+  function walkTree(
+    node: VargNode,
+    path: string[],
+    parentDeps: string[],
+  ): string[] {
+    if (!node || typeof node !== "object" || !("type" in node)) {
+      return [];
+    }
+
+    const element = node as VargElement;
+    const currentPath = [...path, element.type];
+    const collectedDeps: string[] = [...parentDeps];
+
+    // Check if this is a renderable stage
+    const stageTypes: StageType[] = [
+      "image",
+      "video",
+      "animate",
+      "speech",
+      "music",
+    ];
+
+    if (stageTypes.includes(element.type as StageType)) {
+      const stageType = element.type as StageType;
+      const props = element.props as Record<string, unknown>;
+
+      // Skip if it's just a src reference (no generation needed)
+      if (props.src && !props.prompt) {
+        return [];
+      }
+
+      // For video/animate with image inputs, we need to find dependent images first
+      const imageDeps: string[] = [];
+
+      if (stageType === "video" || stageType === "animate") {
+        // Check prompt.images for nested Image elements
+        const prompt = props.prompt as { images?: VargNode[] } | undefined;
+        if (prompt?.images) {
+          for (const imgInput of prompt.images) {
+            if (
+              imgInput &&
+              typeof imgInput === "object" &&
+              "type" in imgInput
+            ) {
+              const imgElement = imgInput as VargElement;
+              if (imgElement.type === "image") {
+                const deps = walkTree(imgElement, currentPath, collectedDeps);
+                imageDeps.push(...deps);
+              }
+            }
+          }
+        }
+
+        // Check for image prop in animate
+        if (stageType === "animate" && props.image) {
+          const imgElement = props.image as VargElement;
+          if (imgElement.type === "image") {
+            const deps = walkTree(imgElement, currentPath, collectedDeps);
+            imageDeps.push(...deps);
+          }
+        }
+      }
+
+      const id = generateId();
+      const stage: RenderStage = {
+        id,
+        type: stageType,
+        label: getLabel(stageType, element),
+        element,
+        path: currentPath,
+        dependsOn: [...new Set([...collectedDeps, ...imageDeps])],
+        status: "pending",
+      };
+
+      stages.push(stage);
+      return [id];
+    }
+
+    // For container elements (render, clip, overlay), recurse into children
+    const childDeps: string[] = [];
+    for (const child of element.children) {
+      const deps = walkTree(child, currentPath, collectedDeps);
+      childDeps.push(...deps);
+    }
+
+    return childDeps;
+  }
+
+  // Start walking from root
+  if (element.type === "render") {
+    walkTree(element, [], []);
+  }
+
+  // Topological sort based on dependencies
+  const order = topologicalSort(stages);
+
+  return { stages, order };
+}
+
+function topologicalSort(stages: RenderStage[]): string[] {
+  const result: string[] = [];
+  const visited = new Set<string>();
+  const visiting = new Set<string>();
+
+  const stageMap = new Map(stages.map((s) => [s.id, s]));
+
+  function visit(id: string) {
+    if (visited.has(id)) return;
+    if (visiting.has(id)) {
+      throw new Error(`Circular dependency detected at stage ${id}`);
+    }
+
+    visiting.add(id);
+    const stage = stageMap.get(id);
+    if (stage) {
+      for (const depId of stage.dependsOn) {
+        visit(depId);
+      }
+    }
+    visiting.delete(id);
+    visited.add(id);
+    result.push(id);
+  }
+
+  for (const stage of stages) {
+    visit(stage.id);
+  }
+
+  return result;
+}
+
+/**
+ * Serializes stages for API response (strips non-serializable element)
+ */
+export function serializeStages(extracted: ExtractedStages): {
+  stages: Omit<RenderStage, "element">[];
+  order: string[];
+} {
+  return {
+    stages: extracted.stages.map(({ element, ...rest }) => rest),
+    order: extracted.order,
+  };
+}

package/src/studio/step-renderer.ts

@@ -0,0 +1,279 @@
+import { generateImage } from "ai";
+import { withCache } from "../ai-sdk/cache";
+import { fileCache } from "../ai-sdk/file-cache";
+import { generateVideo } from "../ai-sdk/generate-video";
+import { renderAnimate } from "../react/renderers/animate";
+import type { RenderContext } from "../react/renderers/context";
+import { renderImage } from "../react/renderers/image";
+import { renderMusic } from "../react/renderers/music";
+import { createProgressTracker } from "../react/renderers/progress";
+import { renderSpeech } from "../react/renderers/speech";
+import { renderVideo } from "../react/renderers/video";
+import type { RenderProps, VargElement } from "../react/types";
+import type { ExtractedStages, RenderStage, StageResult } from "./stages";
+import { extractStages } from "./stages";
+
+export interface StepSession {
+  id: string;
+  code: string;
+  rootElement: VargElement;
+  extracted: ExtractedStages;
+  ctx: RenderContext;
+  results: Map<string, StageResult>;
+  currentStageIndex: number;
+}
+
+const sessions = new Map<string, StepSession>();
+
+export function createStepSession(
+  code: string,
+  rootElement: VargElement,
+  cacheDir?: string,
+): StepSession {
+  const props = rootElement.props as RenderProps;
+  const cache = cacheDir ? fileCache({ dir: cacheDir }) : undefined;
+
+  const ctx: RenderContext = {
+    width: props.width ?? 1920,
+    height: props.height ?? 1080,
+    fps: props.fps ?? 30,
+    cache,
+    generateImage: cache
+      ? withCache(generateImage, { storage: cache })
+      : generateImage,
+    generateVideo: cache
+      ? withCache(generateVideo, { storage: cache })
+      : generateVideo,
+    tempFiles: [],
+    progress: createProgressTracker(false),
+    pending: new Map(),
+  };
+
+  const extracted = extractStages(rootElement);
+  const sessionId = `step-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+
+  const session: StepSession = {
+    id: sessionId,
+    code,
+    rootElement,
+    extracted,
+    ctx,
+    results: new Map(),
+    currentStageIndex: 0,
+  };
+
+  sessions.set(sessionId, session);
+  return session;
+}
+
+export function getSession(sessionId: string): StepSession | undefined {
+  return sessions.get(sessionId);
+}
+
+export function deleteSession(sessionId: string): void {
+  sessions.delete(sessionId);
+}
+
+export async function executeStage(
+  session: StepSession,
+  stageId: string,
+): Promise<StageResult> {
+  const stage = session.extracted.stages.find((s) => s.id === stageId);
+  if (!stage) {
+    throw new Error(`Stage ${stageId} not found`);
+  }
+
+  for (const depId of stage.dependsOn) {
+    if (!session.results.has(depId)) {
+      throw new Error(`Dependency ${depId} not yet executed`);
+    }
+  }
+
+  stage.status = "running";
+  console.log(`[step] executing stage ${stageId}: ${stage.label}`);
+
+  try {
+    let result: StageResult;
+
+    switch (stage.type) {
+      case "image": {
+        const path = await renderImage(
+          stage.element as VargElement<"image">,
+          session.ctx,
+        );
+        result = {
+          type: "image",
+          path,
+          previewUrl: `/api/step/preview/${session.id}/${stageId}`,
+          mimeType: "image/png",
+        };
+        break;
+      }
+
+      case "video": {
+        const path = await renderVideo(
+          stage.element as VargElement<"video">,
+          session.ctx,
+        );
+        result = {
+          type: "video",
+          path,
+          previewUrl: `/api/step/preview/${session.id}/${stageId}`,
+          mimeType: "video/mp4",
+        };
+        break;
+      }
+
+      case "animate": {
+        const path = await renderAnimate(
+          stage.element as VargElement<"animate">,
+          session.ctx,
+        );
+        result = {
+          type: "video",
+          path,
+          previewUrl: `/api/step/preview/${session.id}/${stageId}`,
+          mimeType: "video/mp4",
+        };
+        break;
+      }
+
+      case "speech": {
+        const speechResult = await renderSpeech(
+          stage.element as VargElement<"speech">,
+          session.ctx,
+        );
+        result = {
+          type: "audio",
+          path: speechResult.path,
+          previewUrl: `/api/step/preview/${session.id}/${stageId}`,
+          mimeType: "audio/mp3",
+        };
+        break;
+      }
+
+      case "music": {
+        const musicResult = await renderMusic(
+          stage.element as VargElement<"music">,
+          session.ctx,
+        );
+        result = {
+          type: "audio",
+          path: musicResult.path,
+          previewUrl: `/api/step/preview/${session.id}/${stageId}`,
+          mimeType: "audio/mp3",
+        };
+        break;
+      }
+
+      default:
+        throw new Error(`Unknown stage type: ${stage.type}`);
+    }
+
+    stage.status = "complete";
+    stage.result = result;
+    session.results.set(stageId, result);
+    console.log(`[step] stage ${stageId} complete: ${result.path}`);
+
+    return result;
+  } catch (error) {
+    stage.status = "error";
+    console.error(`[step] stage ${stageId} failed:`, error);
+    throw error;
+  }
+}
+
+export async function executeNextStage(session: StepSession): Promise<{
+  stage: RenderStage;
+  result: StageResult;
+  isLast: boolean;
+} | null> {
+  const { order } = session.extracted;
+
+  if (session.currentStageIndex >= order.length) {
+    return null;
+  }
+
+  const stageId = order[session.currentStageIndex];
+  if (!stageId) {
+    return null;
+  }
+
+  const stage = session.extracted.stages.find((s) => s.id === stageId);
+  if (!stage) {
+    return null;
+  }
+
+  const result = await executeStage(session, stageId);
+  session.currentStageIndex++;
+
+  return {
+    stage,
+    result,
+    isLast: session.currentStageIndex >= order.length,
+  };
+}
+
+export function getStagePreviewPath(
+  session: StepSession,
+  stageId: string,
+): string | null {
+  const result = session.results.get(stageId);
+  return result?.path ?? null;
+}
+
+export async function finalizeRender(
+  session: StepSession,
+  outputDir: string,
+): Promise<string> {
+  const allComplete = session.extracted.stages.every(
+    (s) => s.status === "complete",
+  );
+  if (!allComplete) {
+    throw new Error("Not all stages are complete");
+  }
+
+  const { render } = await import("../react/render");
+  const outputPath = `${outputDir}/render-${session.id}.mp4`;
+
+  await render(session.rootElement, {
+    output: outputPath,
+    cache: session.ctx.cache ? ".cache/ai" : undefined,
+    quiet: true,
+  });
+
+  return outputPath;
+}
+
+export function getSessionStatus(session: StepSession): {
+  sessionId: string;
+  totalStages: number;
+  completedStages: number;
+  currentStageIndex: number;
+  stages: Array<{
+    id: string;
+    type: string;
+    label: string;
+    status: string;
+    hasResult: boolean;
+    dependsOn: string[];
+  }>;
+} {
+  return {
+    sessionId: session.id,
+    totalStages: session.extracted.stages.length,
+    completedStages: session.results.size,
+    currentStageIndex: session.currentStageIndex,
+    stages: session.extracted.order.map((id) => {
+      const stage = session.extracted.stages.find((s) => s.id === id)!;
+      return {
+        id: stage.id,
+        type: stage.type,
+        label: stage.label,
+        status: stage.status,
+        hasResult: session.results.has(id),
+        dependsOn: stage.dependsOn,
+      };
+    }),
+  };
+}

package/src/studio/types.ts

@@ -0,0 +1,60 @@
+export interface CacheItem {
+  id: string;
+  filename: string;
+  type: "image" | "video" | "unknown";
+  size: number;
+  createdAt: Date;
+  metadata: Record<string, unknown>;
+}
+
+export interface CacheEntry {
+  value: unknown;
+  expires: number;
+}
+
+export interface ImageData {
+  uint8ArrayData?: {
+    __type: "Uint8Array";
+    data: string;
+  };
+  base64?: string;
+  url?: string;
+}
+
+export interface VideoData {
+  _data?: {
+    __type: "Uint8Array";
+    data: string;
+  };
+  uint8ArrayData?: {
+    __type: "Uint8Array";
+    data: string;
+  };
+  base64?: string;
+  url?: string;
+}
+
+export interface RenderRequest {
+  code: string;
+  cacheDir?: string;
+}
+
+export interface RenderProgress {
+  step: "parsing" | "rendering" | "complete" | "error";
+  progress: number;
+  message: string;
+}
+
+export interface Template {
+  id: string;
+  name: string;
+  description: string;
+}
+
+export interface HistoryItem {
+  id: string;
+  code: string;
+  videoUrl: string;
+  thumbnail?: string;
+  createdAt: Date;
+}