videoclaw 3.0.0-alpha.1

package/src/video/motion-overlay/slice.ts

@@ -0,0 +1,98 @@
+/**
+ * Pure take-slicing.
+ *
+ * Omni Flash has a hard ~10s-per-clip limit, so a longer talking-head video is
+ * sliced into takes ≤ maxTakeSeconds, cutting only at sentence (segment)
+ * boundaries — never mid-segment — so the joined reel reads cleanly.
+ *
+ * Algorithm (ported from the source skill, Step 5):
+ *  1. Iterate transcript segments, accumulating into the current take while
+ *     `segment.end - takeStart <= max`.
+ *  2. When the next segment would overrun `max`, close the current take at the
+ *     last accumulated segment's end and start a fresh take at that segment.
+ *  3. Edge case: a single segment whose own duration exceeds `max` becomes its
+ *     own take with a HARD CUT at exactly `takeStart + max` (the only place a
+ *     cut lands mid-segment — unavoidable when one breath runs > max).
+ *
+ * Pure: no I/O. Deterministic for a given transcript + max.
+ */
+
+import type { TakeSplit, Transcript } from './types.js';
+
+const EPSILON = 1e-9;
+
+/**
+ * Group transcript segments into takes ≤ maxTakeSeconds at sentence boundaries.
+ *
+ * @param transcript segments with absolute start/end seconds (must be ordered).
+ * @param maxTakeSeconds Omni hard limit per clip (must be > 0).
+ * @returns ordered `TakeSplit[]` with absolute start/end + the segment indices
+ *          each take covers. Empty transcript → `[]`.
+ */
+export function planSplits(transcript: Transcript, maxTakeSeconds: number): TakeSplit[] {
+  if (!(maxTakeSeconds > 0)) {
+    throw new Error(`planSplits: maxTakeSeconds must be > 0, got ${maxTakeSeconds}`);
+  }
+  const segments = transcript.segments;
+  if (segments.length === 0) {
+    return [];
+  }
+
+  const splits: TakeSplit[] = [];
+  let takeStart: number | null = null;
+  let takeEnd = 0;
+  let segmentIndices: number[] = [];
+
+  const closeTake = (end: number): void => {
+    if (takeStart === null) {
+      return;
+    }
+    splits.push({
+      index: splits.length,
+      start: takeStart,
+      end,
+      segmentIndices,
+    });
+    takeStart = null;
+    segmentIndices = [];
+  };
+
+  for (let i = 0; i < segments.length; i++) {
+    const seg = segments[i];
+
+    if (takeStart === null) {
+      // Start a new take at this segment.
+      const segDuration = seg.end - seg.start;
+      if (segDuration > maxTakeSeconds + EPSILON) {
+        // A single segment longer than the cap: it must stand alone with a
+        // HARD CUT at exactly start + max (the one place we cut mid-segment).
+        splits.push({
+          index: splits.length,
+          start: seg.start,
+          end: seg.start + maxTakeSeconds,
+          segmentIndices: [i],
+        });
+        continue;
+      }
+      takeStart = seg.start;
+      takeEnd = seg.end;
+      segmentIndices = [i];
+      continue;
+    }
+
+    // We have an open take; would adding this segment overrun the cap?
+    if (seg.end - takeStart <= maxTakeSeconds + EPSILON) {
+      takeEnd = seg.end;
+      segmentIndices.push(i);
+      continue;
+    }
+
+    // Overrun: close the current take, then re-process this segment as the
+    // start of a fresh take (handles the oversized-segment edge case too).
+    closeTake(takeEnd);
+    i -= 1;
+  }
+
+  closeTake(takeEnd);
+  return splits;
+}

package/src/video/motion-overlay/transcribe.ts

@@ -0,0 +1,211 @@
+/**
+ * Gemini STT for motion-overlay.
+ *
+ * `transcribeAudio` turns an audio file into a `Transcript` (segment-level
+ * start/end/text) via the Gemini API, reusing the `gemini-key-pool` round-robin
+ * + retry plumbing. The transport is injectable (`deps.fetcher`) so tests never
+ * touch the network; a `deps.transcriber` override replaces the whole live path.
+ *
+ * The `--transcript <path>` operator override is handled by `loadTranscript`,
+ * which loads + validates a bring-your-own transcript JSON of the same shape.
+ *
+ * Side-effect: reads the audio file + calls the Gemini API. Kept thin — the
+ * shaping/validation is pure and unit-testable.
+ */
+
+import { readFile } from 'node:fs/promises';
+import { extname } from 'node:path';
+import { fetchGeminiWithPool } from '../gemini-key-pool.js';
+import type { Transcript, TranscriptSegment } from './types.js';
+
+const DEFAULT_STT_ENDPOINT =
+  'https://generativelanguage.googleapis.com/v1beta/models/gemini-3.5-flash:generateContent';
+
+const STT_PROMPT = `You transcribe a single short spoken-word audio clip into timed segments.
+
+Return ONLY valid JSON with this exact shape (no markdown fences):
+{ "language": "<bcp-47 code>", "segments": [ { "start": <seconds>, "end": <seconds>, "text": "<sentence>" } ] }
+
+Rules:
+- One segment per spoken sentence or natural breath group.
+- start/end are absolute seconds from the clip's 0:00, increasing, non-overlapping.
+- text is the spoken words verbatim, no speaker labels, no timestamps inside the text.
+- language is the detected spoken language as a BCP-47 code (e.g. "en", "pt", "es").
+- Do not invent content; transcribe only what is spoken.`;
+
+/** A pluggable transcriber. Tests inject this to avoid the live Gemini path. */
+export type TranscriberFn = (audioPath: string, language: string) => Promise<Transcript>;
+
+export interface TranscribeDeps {
+  /** Replace the whole live transcription path (highest precedence). */
+  transcriber?: TranscriberFn;
+  /** Inject a fetch for the live Gemini path (used when no transcriber given). */
+  fetcher?: typeof fetch;
+  /** Override the Gemini endpoint (falls back to env then the default). */
+  endpoint?: string;
+}
+
+/** Map an audio file extension to a Gemini inlineData mime type. */
+function audioMimeType(audioPath: string): string {
+  switch (extname(audioPath).toLowerCase()) {
+    case '.mp3':
+      return 'audio/mp3';
+    case '.aac':
+      return 'audio/aac';
+    case '.m4a':
+      return 'audio/mp4';
+    case '.flac':
+      return 'audio/flac';
+    case '.ogg':
+      return 'audio/ogg';
+    case '.wav':
+    default:
+      return 'audio/wav';
+  }
+}
+
+/** Pull the joined text parts out of a Gemini generateContent response. */
+function parseGeminiText(payload: unknown): string {
+  const candidates = (payload as { candidates?: Array<{ content?: { parts?: Array<{ text?: string }> } }> }).candidates;
+  const text = candidates?.[0]?.content?.parts
+    ?.map((part) => part.text ?? '')
+    .join('\n')
+    .trim();
+  if (!text) {
+    throw new Error('Gemini STT response did not contain text output.');
+  }
+  return text;
+}
+
+/**
+ * Validate + normalize an unknown value into a `Transcript`. Throws on any shape
+ * violation. Pure — shared by the live path and the `--transcript` loader so the
+ * contract is enforced in exactly one place.
+ */
+export function validateTranscript(value: unknown, source = 'transcript'): Transcript {
+  if (!value || typeof value !== 'object') {
+    throw new Error(`${source}: expected an object with { language, segments }.`);
+  }
+  const obj = value as { language?: unknown; segments?: unknown };
+  if (typeof obj.language !== 'string' || obj.language.trim() === '') {
+    throw new Error(`${source}: "language" must be a non-empty string.`);
+  }
+  if (!Array.isArray(obj.segments)) {
+    throw new Error(`${source}: "segments" must be an array.`);
+  }
+  const segments: TranscriptSegment[] = obj.segments.map((raw, i) => {
+    if (!raw || typeof raw !== 'object') {
+      throw new Error(`${source}: segment ${i} is not an object.`);
+    }
+    const seg = raw as { start?: unknown; end?: unknown; text?: unknown };
+    if (typeof seg.start !== 'number' || !Number.isFinite(seg.start) || seg.start < 0) {
+      throw new Error(`${source}: segment ${i} "start" must be a finite, non-negative number.`);
+    }
+    if (typeof seg.end !== 'number' || !Number.isFinite(seg.end) || seg.end < seg.start) {
+      throw new Error(`${source}: segment ${i} "end" must be a finite number >= start.`);
+    }
+    if (typeof seg.text !== 'string') {
+      throw new Error(`${source}: segment ${i} "text" must be a string.`);
+    }
+    return { start: seg.start, end: seg.end, text: seg.text };
+  });
+  return { language: obj.language.trim(), segments };
+}
+
+/**
+ * Load + validate a bring-your-own transcript JSON (`--transcript <path>`).
+ * Throws on a missing/unreadable file or a shape violation.
+ */
+export async function loadTranscript(path: string): Promise<Transcript> {
+  let raw: string;
+  try {
+    raw = await readFile(path, 'utf-8');
+  } catch (err) {
+    throw new Error(`--transcript file not readable: ${path} (${(err as Error).message})`);
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`--transcript file is not valid JSON: ${path} (${(err as Error).message})`);
+  }
+  return validateTranscript(parsed, `--transcript ${path}`);
+}
+
+/**
+ * Transcribe an audio file into a `Transcript`.
+ *
+ * Precedence:
+ *  1. `deps.transcriber` — full override (tests inject this; no network).
+ *  2. live Gemini STT via `fetchGeminiWithPool` (uses `deps.fetcher` if given).
+ *
+ * @param audioPath path to the extracted audio clip.
+ * @param language requested language hint ("auto" lets Gemini detect); the
+ *   returned transcript's `language` reflects the model's detection.
+ */
+export async function transcribeAudio(
+  audioPath: string,
+  language: string,
+  deps: TranscribeDeps = {},
+): Promise<Transcript> {
+  if (deps.transcriber) {
+    return deps.transcriber(audioPath, language);
+  }
+
+  const endpoint = deps.endpoint ?? process.env.VCLAW_GEMINI_API_ENDPOINT ?? DEFAULT_STT_ENDPOINT;
+  const audioBytes = await readFile(audioPath);
+  const audioData = audioBytes.toString('base64');
+  const mimeType = audioMimeType(audioPath);
+  const languageHint =
+    language && language.toLowerCase() !== 'auto'
+      ? `\nThe spoken language is "${language}"; transcribe in that language.`
+      : '';
+
+  const response = await fetchGeminiWithPool(
+    (key) => `${endpoint}${endpoint.includes('?') ? '&' : '?'}key=${encodeURIComponent(key)}`,
+    {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', Connection: 'close' },
+      body: JSON.stringify({
+        contents: [
+          {
+            parts: [
+              { inlineData: { mimeType, data: audioData } },
+              { text: `${STT_PROMPT}${languageHint}` },
+            ],
+          },
+        ],
+        generationConfig: {
+          temperature: 0,
+          maxOutputTokens: 2048,
+          responseMimeType: 'application/json',
+        },
+      }),
+    },
+    {
+      ...(deps.fetcher ? { fetcher: deps.fetcher } : {}),
+      onRetry: (label, status) => {
+        process.stderr.write(`[motion-overlay/stt] ${label} returned HTTP ${status}; rotating key\n`);
+      },
+    },
+  );
+
+  if (!response.ok) {
+    throw new Error(`Gemini STT request failed with HTTP ${response.status}`);
+  }
+
+  const payload = await response.json();
+  const text = parseGeminiText(payload);
+  const cleaned = text
+    .replace(/^```json\s*/i, '')
+    .replace(/^```\s*/i, '')
+    .replace(/\s*```$/i, '')
+    .trim();
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(cleaned);
+  } catch (err) {
+    throw new Error(`Gemini STT returned unparseable JSON: ${(err as Error).message}`);
+  }
+  return validateTranscript(parsed, 'Gemini STT');
+}

package/src/video/motion-overlay/types.ts

@@ -0,0 +1,89 @@
+/**
+ * Motion-overlay shared type contract.
+ *
+ * This is the binding contract for the whole `motion-overlay/` module family.
+ * Every pure module (slice, analyze-reel, motion-style, compose-prompt, plan)
+ * and every side-effect module (transcribe, ingest, write, preview, execute,
+ * avatar-host) speaks these types. Keep it in lockstep with
+ * `schemas/video/artifacts/motion-overlay-plan.schema.json` and the design/plan
+ * specs under `docs/superpowers/specs/`.
+ */
+
+export type MotionLayout = 'split' | 'overlay' | 'motion-only' | 'avatar-host';
+export type MotionStyleId = 'apple-clean' | 'editorial-dark' | 'knowledge-tool';
+export type ReelAnatomy = 'hook' | 'contextualize' | 'burst' | 'breath' | 'cta';
+
+export interface TranscriptSegment {
+  start: number;
+  end: number;
+  text: string;
+}
+export interface Transcript {
+  language: string;
+  segments: TranscriptSegment[];
+}
+
+export interface TakeSplit {
+  index: number;
+  start: number;
+  end: number;
+  segmentIndices: number[];
+}
+
+export type AnchorKind = 'number' | 'verb' | 'entity' | 'keyword';
+export interface AnchorWord {
+  word: string;
+  kind: AnchorKind;
+  relativeTime: number;
+}
+export interface TakePause {
+  start: number;
+  end: number;
+} // relative to take 0:00
+export interface TakeAnalysis {
+  index: number;
+  anatomy: ReelAnatomy;
+  anchorWords: AnchorWord[];
+  pauses: TakePause[];
+}
+
+export interface ResolvedStyle {
+  id: MotionStyleId;
+  accent: string; // hex, configurable (default per style)
+  visualBlock: string; // prose, no specs (CRITICAL RULE 2 safe)
+  motionBlock: string; // prose, no specs
+}
+
+export interface FrameObservations {
+  speakerSide?: 'left' | 'right' | 'center';
+  notes: string[]; // wall colour, untouchable regions, etc.
+}
+
+export interface MotionTake {
+  index: number;
+  start: number;
+  end: number;
+  durationSeconds: number;
+  file: string; // relative path to take mp4
+  promptFile: string; // relative path to prompt md
+  anatomy: ReelAnatomy;
+  prompt: string; // the full composed prompt
+}
+
+export interface MotionOverlayPlan {
+  schemaVersion: 1;
+  input: {
+    path: string;
+    durationSeconds: number;
+    width: number;
+    height: number;
+    aspect: '16:9' | '9:16' | '1:1' | string;
+  };
+  layout: MotionLayout;
+  style: MotionStyleId;
+  accent: string;
+  language: string;
+  workdir: string;
+  takes: MotionTake[];
+  generatedAt?: string; // stamped by caller, not pure code
+}

package/src/video/motion-overlay/v2v-transport.ts

@@ -0,0 +1,209 @@
+/**
+ * Real Google Flow omni-flash V2V transport for the standalone motion-overlay
+ * default runner (the per-take seam injected into {@link createDefaultMotionOverlayRunner}).
+ *
+ * Per take it: uploads the base footage as a Flow asset (referenceVideo_1),
+ * submits an omni-flash video-to-video edit with the composed prompt + a
+ * frame-window, polls the job to completion, and downloads the annotated take to
+ * `outputPath`. The audio-restore mux and the clip-stitch are handled by the
+ * surrounding default runner — this module is ONLY the V2V edit.
+ *
+ * Proven recipe (validated live against useapi.net 2026-06-05):
+ *  - upload:  POST {base}/assets/{email}  (Content-Type: video/mp4, raw bytes)
+ *             → { mediaGenerationId: { mediaGenerationId } }
+ *  - submit:  POST {base}/videos { model:'omni-flash', prompt, referenceVideo_1,
+ *             startFrameIndex_1:0, endFrameIndex_1:<frames>, aspectRatio, email,
+ *             async:true } → { jobid }   (NOTE: the field is `jobid`, lowercase i)
+ *  - poll:    GET  {base}/jobs/{jobId}   (jobId passed RAW — do NOT URL-encode it)
+ *             → status created → completed | failed
+ *  - result:  response.media[].videoUrl
+ *
+ * The Flow safety filter (PUBLIC_ERROR_UNSAFE_GENERATION / FINISH_REASON_INPUT_VIDEO_EDIT)
+ * is probabilistic AND input-video-specific (it rejects some source clips for
+ * editing regardless of how benign the prompt is), so a `failed` verdict is
+ * retried a few times before giving up with an actionable message.
+ *
+ * The network path is intentionally an injectable seam (`fetcher`); the pure
+ * request-shaping helpers below are unit-tested.
+ */
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import type { MotionOverlayV2VStep } from './execute.js';
+
+const execFileP = promisify(execFile);
+const FLOW_BASE = 'https://api.useapi.net/v1/google-flow';
+
+export interface FlowV2VConfig {
+  token: string;
+  email: string;
+  base?: string;
+  maxAttempts?: number;
+  pollIntervalMs?: number;
+  pollTimeoutMs?: number;
+  /** Injectable fetch (tests). Defaults to the global `fetch`. */
+  fetcher?: typeof fetch;
+}
+
+/**
+ * Read the useapi.net credentials from the environment. Throws a clear,
+ * actionable error when they are absent (so `--execute` fails fast rather than
+ * silently producing nothing).
+ */
+export function flowV2VConfigFromEnv(env: NodeJS.ProcessEnv = process.env): FlowV2VConfig {
+  const token = env.USEAPI_API_TOKEN;
+  const email = env.USEAPI_ACCOUNT_EMAIL;
+  if (!token || !email) {
+    throw new Error(
+      'motion-overlay --execute requires USEAPI_API_TOKEN and USEAPI_ACCOUNT_EMAIL in the environment — ' +
+        'the omni-flash V2V transport uploads each take to Google Flow via useapi.net. Export them (e.g. from .env) and retry.',
+    );
+  }
+  return { token, email };
+}
+
+/** Pure: build the POST /videos request body for an omni-flash V2V edit. */
+export function buildV2VRequestBody(args: {
+  prompt: string;
+  referenceVideoId: string;
+  frames: number;
+  aspect: string;
+  email: string;
+}): Record<string, unknown> {
+  return {
+    model: 'omni-flash',
+    prompt: args.prompt,
+    referenceVideo_1: args.referenceVideoId,
+    startFrameIndex_1: 0,
+    endFrameIndex_1: args.frames,
+    aspectRatio: args.aspect,
+    email: args.email,
+    async: true,
+  };
+}
+
+/** Pure: pull the job id from a submit response — the API returns `jobid`, some paths `jobId`. */
+export function extractJobId(payload: unknown): string | undefined {
+  const p = payload as { jobId?: unknown; jobid?: unknown };
+  const id = p?.jobId ?? p?.jobid;
+  return typeof id === 'string' && id.length > 0 ? id : undefined;
+}
+
+/** Pure: pull the first completed videoUrl out of a poll response. */
+export function extractVideoUrl(payload: unknown): string | undefined {
+  const media = (payload as { response?: { media?: Array<{ videoUrl?: unknown }> } })?.response?.media;
+  const url = media?.find((m) => typeof m.videoUrl === 'string' && m.videoUrl)?.videoUrl;
+  return typeof url === 'string' ? url : undefined;
+}
+
+/** Pure: extract a compact moderation/failure reason from a failed poll payload. */
+export function extractFailureReason(payload: unknown): string {
+  return (
+    JSON.stringify((payload as { response?: unknown })?.response ?? payload)
+      .match(/PUBLIC_ERROR_\w+|FINISH_REASON_\w+/g)
+      ?.slice(0, 3)
+      ?.join(', ') ?? 'unknown'
+  );
+}
+
+/** A `failed` verdict from the Flow safety filter (retryable — it is probabilistic). */
+class V2VModerationError extends Error {}
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** ffprobe the clip for an integer frame count + aspect (24fps + landscape fallback). */
+async function probeVideo(path: string): Promise<{ frames: number; aspect: 'landscape' | 'portrait' }> {
+  try {
+    const [{ stdout: wh }, { stdout: dur }] = await Promise.all([
+      execFileP('ffprobe', ['-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=width,height', '-of', 'csv=p=0', path]),
+      execFileP('ffprobe', ['-v', 'error', '-show_entries', 'format=duration', '-of', 'default=nk=1:nw=1', path]),
+    ]);
+    const [w, h] = wh.trim().split(',').map(Number);
+    const d = Number(dur.trim());
+    return {
+      frames: Math.max(1, Math.round((Number.isFinite(d) ? d : 5) * 24)),
+      aspect: w >= h ? 'landscape' : 'portrait',
+    };
+  } catch {
+    return { frames: 120, aspect: 'landscape' };
+  }
+}
+
+/**
+ * Build the live omni-flash V2V transport. The returned function uploads the
+ * take, submits the edit, polls, and writes the annotated take to `outputPath`.
+ */
+export function createFlowV2VTransport(cfg: FlowV2VConfig): (step: MotionOverlayV2VStep) => Promise<void> {
+  const base = cfg.base ?? FLOW_BASE;
+  const doFetch = cfg.fetcher ?? fetch;
+  const auth = { Authorization: `Bearer ${cfg.token}` };
+  const maxAttempts = cfg.maxAttempts ?? 3;
+  const pollIntervalMs = cfg.pollIntervalMs ?? 20000;
+  const pollTimeoutMs = cfg.pollTimeoutMs ?? 8 * 60 * 1000;
+
+  async function poll(jobId: string): Promise<string> {
+    const deadline = Date.now() + pollTimeoutMs;
+    while (Date.now() < deadline) {
+      await delay(pollIntervalMs);
+      const res = await doFetch(`${base}/jobs/${jobId}`, { headers: auth }); // RAW jobId — do NOT encode
+      const pj = (await res.json().catch(() => ({}))) as { status?: string };
+      if (pj.status === 'completed') {
+        const url = extractVideoUrl(pj);
+        if (url) return url;
+        throw new Error('motion-overlay V2V completed but the response carried no videoUrl.');
+      }
+      if (pj.status === 'failed') {
+        throw new V2VModerationError(extractFailureReason(pj));
+      }
+    }
+    throw new Error('motion-overlay V2V poll timed out.');
+  }
+
+  return async (step: MotionOverlayV2VStep): Promise<void> => {
+    const bytes = await readFile(step.baseFootage);
+    const up = await doFetch(`${base}/assets/${encodeURIComponent(cfg.email)}`, {
+      method: 'POST',
+      headers: { ...auth, 'Content-Type': 'video/mp4' },
+      body: bytes,
+    });
+    const upJson = (await up.json().catch(() => ({}))) as { mediaGenerationId?: { mediaGenerationId?: string } };
+    const referenceVideoId = upJson?.mediaGenerationId?.mediaGenerationId;
+    if (!referenceVideoId) {
+      throw new Error(`motion-overlay V2V upload failed (HTTP ${up.status}) for take ${step.index}.`);
+    }
+    const { frames, aspect } = await probeVideo(step.baseFootage);
+
+    let lastReason = '';
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+      const gen = await doFetch(`${base}/videos`, {
+        method: 'POST',
+        headers: { ...auth, 'Content-Type': 'application/json' },
+        body: JSON.stringify(buildV2VRequestBody({ prompt: step.prompt, referenceVideoId, frames, aspect, email: cfg.email })),
+      });
+      const jobId = extractJobId(await gen.json().catch(() => ({})));
+      if (!jobId) {
+        lastReason = `submit returned HTTP ${gen.status} with no job id`;
+        continue;
+      }
+      try {
+        const url = await poll(jobId);
+        const dl = await doFetch(url);
+        await mkdir(dirname(step.outputPath), { recursive: true });
+        await writeFile(step.outputPath, Buffer.from(await dl.arrayBuffer()));
+        return;
+      } catch (err) {
+        lastReason = (err as Error).message;
+        if (err instanceof V2VModerationError) continue; // probabilistic — retry
+        throw err; // timeout / transport / no-url — fail fast
+      }
+    }
+    throw new Error(
+      `motion-overlay V2V failed for take ${step.index} after ${maxAttempts} attempt(s): ${lastReason}. ` +
+        'The Google Flow safety filter rejects some input videos for editing (FINISH_REASON_INPUT_VIDEO_EDIT) ' +
+        'regardless of prompt — try a different, more stylized source clip.',
+    );
+  };
+}