videoclaw 3.0.0-alpha.1

package/src/video/verify-env.ts

@@ -0,0 +1,206 @@
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+import { join, resolve } from 'node:path';
+import { buildProviderStatusReport } from './provider-status.js';
+
+interface EnvVarStatus {
+  name: string;
+  required: boolean;
+  present: boolean;
+  source: 'process' | '.env.local' | '.env' | 'missing';
+}
+
+interface DependencyStatus {
+  name: string;
+  available: boolean;
+  path?: string;
+}
+
+export interface VideoEnvironmentReport {
+  generatedAt: string;
+  workspaceRoot: string;
+  envSources: string[];
+  envVars: EnvVarStatus[];
+  geminiKeyPool: {
+    count: number;
+    recommended: number;
+    ok: boolean;
+  };
+  localDependencies: DependencyStatus[];
+  build: {
+    path: string;
+    exists: boolean;
+    ageHours?: number;
+    fresh?: boolean;
+  };
+  providers: ReturnType<typeof buildProviderStatusReport>;
+  blockingIssues: string[];
+  warnings: string[];
+  ok: boolean;
+}
+
+interface BuildVideoEnvironmentReportOptions {
+  workspaceRoot?: string;
+  env?: NodeJS.ProcessEnv;
+  now?: Date;
+  probeExecutable?: (name: string) => string | undefined;
+}
+
+function readDotEnvLikeFile(path: string): Record<string, string> {
+  if (!existsSync(path)) return {};
+  const out: Record<string, string> = {};
+  const raw = readFileSync(path, 'utf-8');
+  for (const line of raw.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#') || !trimmed.includes('=')) continue;
+    const [key, ...rest] = trimmed.split('=');
+    out[key.trim()] = rest.join('=').trim().replace(/^['"]|['"]$/g, '');
+  }
+  return out;
+}
+
+function findExecutable(name: string): string | undefined {
+  try {
+    const resolved = execFileSync('which', [name], { encoding: 'utf-8' }).trim();
+    return resolved || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+function countGeminiKeys(env: Record<string, string | undefined>): number {
+  const sources = [
+    env.GEMINI_API_KEYS,
+    env.GOOGLE_API_KEYS,
+    env.GOOGLE_API_KEY,
+  ].filter((value): value is string => typeof value === 'string' && value.trim().length > 0);
+  const seen = new Set<string>();
+  for (const source of sources) {
+    for (const raw of source.split(/[,;\n\s]+/)) {
+      const key = raw.trim();
+      if (!key) continue;
+      seen.add(key);
+    }
+  }
+  return seen.size;
+}
+
+export function buildVideoEnvironmentReport(
+  options: BuildVideoEnvironmentReportOptions = {},
+): VideoEnvironmentReport {
+  const workspaceRoot = resolve(options.workspaceRoot ?? process.cwd());
+  const now = options.now ?? new Date();
+  const probe = options.probeExecutable ?? findExecutable;
+  const envPath = join(workspaceRoot, '.env');
+  const envLocalPath = join(workspaceRoot, '.env.local');
+  const envFile = readDotEnvLikeFile(envPath);
+  const envLocalFile = readDotEnvLikeFile(envLocalPath);
+  const mergedEnv = {
+    ...envFile,
+    ...envLocalFile,
+    ...(options.env ?? process.env),
+  };
+
+  const envSources = [envPath, envLocalPath].filter((path) => existsSync(path));
+  const requiredEnvVars = ['GOOGLE_API_KEY', 'GO_BANANAS_API_KEY', 'SUTUI_API_KEY'];
+  const optionalEnvVars = ['GEMINI_API_KEYS', 'GOOGLE_API_KEYS', 'ELEVENLABS_API_KEY'];
+  const envVars: EnvVarStatus[] = [...requiredEnvVars, ...optionalEnvVars].map((name) => {
+    const present = Boolean(mergedEnv[name]?.trim());
+    let source: EnvVarStatus['source'] = 'missing';
+    if (present) {
+      if ((options.env ?? process.env)[name]) {
+        source = 'process';
+      } else if (Object.prototype.hasOwnProperty.call(envLocalFile, name)) {
+        source = '.env.local';
+      } else if (Object.prototype.hasOwnProperty.call(envFile, name)) {
+        source = '.env';
+      }
+    }
+    return {
+      name,
+      required: requiredEnvVars.includes(name),
+      present,
+      source,
+    };
+  });
+
+  const geminiKeyPoolCount = countGeminiKeys(mergedEnv);
+  const localDependencyNames = ['node', 'npm', 'python3', 'ffmpeg', 'ffprobe', 'curl', 'bun'];
+  const localDependencies: DependencyStatus[] = localDependencyNames.map((name) => ({
+    name,
+    available: Boolean(probe(name)),
+    path: probe(name),
+  }));
+
+  const buildPath = join(workspaceRoot, 'dist', 'cli', 'vclaw.js');
+  const buildExists = existsSync(buildPath);
+  const buildAgeHours = buildExists
+    ? Math.max(0, Math.floor((now.getTime() - statSync(buildPath).mtime.getTime()) / 3_600_000))
+    : undefined;
+  const buildFresh = buildAgeHours !== undefined ? buildAgeHours < 24 : undefined;
+
+  const providers = buildProviderStatusReport({
+    workspaceRoot,
+    env: mergedEnv,
+    now,
+    probeExecutable: (name) => probe(name),
+  });
+
+  const blockingIssues: string[] = [];
+  const warnings: string[] = [];
+
+  for (const envVar of envVars) {
+    if (envVar.required && !envVar.present) {
+      blockingIssues.push(`Missing required environment variable: ${envVar.name}`);
+    }
+  }
+
+  for (const dependency of localDependencies) {
+    if (['node', 'npm', 'python3', 'ffmpeg', 'ffprobe', 'curl'].includes(dependency.name) && !dependency.available) {
+      blockingIssues.push(`Missing required local dependency: ${dependency.name}`);
+    }
+  }
+
+  if (!buildExists) {
+    blockingIssues.push('Build output missing: dist/cli/vclaw.js');
+  } else if (buildFresh === false) {
+    warnings.push(`Build output is ${buildAgeHours}h old; run npm run build if the code changed.`);
+  }
+
+  if (geminiKeyPoolCount === 0) {
+    warnings.push('No Gemini key pool detected; director decomposition will rely on GOOGLE_API_KEY only.');
+  } else if (geminiKeyPoolCount < 3) {
+    warnings.push(`Gemini key pool size is ${geminiKeyPoolCount}; recommend 3+ keys for longer director runs.`);
+  }
+
+  for (const route of providers.routes) {
+    if (route.availability === 'unavailable' && route.routeId === 'seedance-direct') {
+      blockingIssues.push(`Route ${route.routeId} unavailable: ${route.issues.join('; ')}`);
+    } else if (route.availability !== 'available') {
+      warnings.push(`Route ${route.routeId} is ${route.availability}: ${route.issues.join('; ') || route.notes.join('; ')}`);
+    }
+  }
+
+  return {
+    generatedAt: now.toISOString(),
+    workspaceRoot,
+    envSources,
+    envVars,
+    geminiKeyPool: {
+      count: geminiKeyPoolCount,
+      recommended: 3,
+      ok: geminiKeyPoolCount >= 3,
+    },
+    localDependencies,
+    build: {
+      path: buildPath,
+      exists: buildExists,
+      ...(buildAgeHours !== undefined ? { ageHours: buildAgeHours } : {}),
+      ...(buildFresh !== undefined ? { fresh: buildFresh } : {}),
+    },
+    providers,
+    blockingIssues,
+    warnings,
+    ok: blockingIssues.length === 0,
+  };
+}

package/src/video/verify-final.ts

@@ -0,0 +1,65 @@
+import { mkdir } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { probeMedia, resolveProjectFinalPath } from './final-media.js';
+
+const execFileAsync = promisify(execFile);
+
+export interface VideoFinalVerificationReport {
+  projectSlug?: string;
+  sourcePath: string;
+  durationSeconds?: number;
+  sizeBytes?: number;
+  videoCodec?: string;
+  width?: number;
+  height?: number;
+  frameRate?: string;
+  audioPresent: boolean;
+  audioCodec?: string;
+  framePath: string;
+}
+
+async function extractMidpointFrame(sourcePath: string, outputDir: string): Promise<string> {
+  await mkdir(outputDir, { recursive: true });
+  const metadata = await probeMedia(sourcePath);
+  const midpoint = Math.max(0, (metadata.durationSeconds ?? 0) / 2);
+  const framePath = join(outputDir, 'post-check-frame.jpg');
+
+  await execFileAsync('ffmpeg', [
+    '-y',
+    '-ss', String(midpoint),
+    '-i', sourcePath,
+    '-frames:v', '1',
+    '-update', '1',
+    framePath,
+  ], { encoding: 'utf-8' });
+
+  return framePath;
+}
+
+export async function verifyFinalOutput(options: {
+  projectSlug?: string;
+  filePath?: string;
+  root?: string;
+  outputDir?: string;
+}): Promise<VideoFinalVerificationReport> {
+  const root = options.root ?? process.cwd();
+  const sourcePath = options.filePath
+    ?? (options.projectSlug ? await resolveProjectFinalPath(options.projectSlug, root) : undefined);
+
+  if (!sourcePath) {
+    throw new Error('verifyFinalOutput requires either projectSlug or filePath');
+  }
+
+  const outputDir = options.outputDir ?? join(dirname(sourcePath), 'verification');
+  const metadata = await probeMedia(sourcePath);
+  const framePath = await extractMidpointFrame(sourcePath, outputDir);
+
+  return {
+    ...(options.projectSlug ? { projectSlug: options.projectSlug } : {}),
+    sourcePath,
+    ...metadata,
+    framePath,
+  };
+}

package/src/video/video-context.ts

@@ -0,0 +1,92 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+const CHANGELOG_HEADER = '## Dated changelog';
+
+function bootstrapContent(): string {
+  return [
+    '# Video Context',
+    '',
+    'Operational memory for this workspace. Treat as run learnings, not source code.',
+    '',
+    '## Provider defaults',
+    '- None recorded yet.',
+    '',
+    '## Winning prompts',
+    '- None recorded yet.',
+    '',
+    '## Failed prompt patterns',
+    '- None recorded yet.',
+    '',
+    '## Style presets by brand',
+    '- None recorded yet.',
+    '',
+    '## Cost/time benchmarks',
+    '- None recorded yet.',
+    '',
+    CHANGELOG_HEADER,
+    '- None recorded yet.',
+    '',
+  ].join('\n');
+}
+
+function withChangelogEntry(content: string, entry: string): string {
+  const changelogBlock = `${CHANGELOG_HEADER}\n- ${entry}`;
+  if (content.includes(`${CHANGELOG_HEADER}\n- None recorded yet.`)) {
+    return content.replace(`${CHANGELOG_HEADER}\n- None recorded yet.`, changelogBlock);
+  }
+  if (content.includes(CHANGELOG_HEADER)) {
+    return content.replace(CHANGELOG_HEADER, changelogBlock);
+  }
+  return `${content.trimEnd()}\n\n${changelogBlock}\n`;
+}
+
+export interface VideoContextPaths {
+  omxDir: string;
+  omxPath: string;
+  vclawDir: string;
+  vclawPath: string;
+}
+
+export function resolveVideoContextPaths(root = process.cwd()): VideoContextPaths {
+  const normalizedRoot = resolve(root);
+  return {
+    omxDir: join(normalizedRoot, '.omx'),
+    omxPath: join(normalizedRoot, '.omx', 'video-context.md'),
+    vclawDir: join(normalizedRoot, '.vclaw'),
+    vclawPath: join(normalizedRoot, '.vclaw', 'video-context.md'),
+  };
+}
+
+export async function ensureVideoContext(root = process.cwd()): Promise<VideoContextPaths> {
+  const paths = resolveVideoContextPaths(root);
+  await mkdir(paths.omxDir, { recursive: true });
+
+  if (!existsSync(paths.omxPath)) {
+    if (existsSync(paths.vclawPath)) {
+      const legacyContent = await readFile(paths.vclawPath, 'utf-8');
+      await writeFile(paths.omxPath, legacyContent);
+    } else {
+      await writeFile(paths.omxPath, `${bootstrapContent()}\n`);
+    }
+  }
+
+  return paths;
+}
+
+export async function appendVideoContextChangelog(
+  root: string | undefined,
+  entry: string,
+): Promise<VideoContextPaths> {
+  const paths = await ensureVideoContext(root);
+  const current = await readFile(paths.omxPath, 'utf-8');
+  const updated = withChangelogEntry(current, entry);
+  await writeFile(paths.omxPath, updated.endsWith('\n') ? updated : `${updated}\n`);
+
+  if (existsSync(paths.vclawPath)) {
+    await writeFile(paths.vclawPath, updated.endsWith('\n') ? updated : `${updated}\n`);
+  }
+
+  return paths;
+}

package/src/video/vocal-map.ts

@@ -0,0 +1,155 @@
+/**
+ * vocal-map.ts — turn a transcript into the VOCAL MAP the music-video planner
+ * needs: the song segmented by timestamp into rap / hook / instrumental / outro.
+ *
+ * {@link buildVocalMap} is a PURE classifier over whisper-style segments
+ * (`{start, end, text}`). The hook is the SUSTAINED, sung passage; the verses are
+ * the dense, fast-delivered rap. The discriminating signal is WORD DENSITY
+ * (words per second): a sung chorus stretches few words over many seconds (low
+ * wps) while rap packs many words into little time (high wps). The classifier
+ * finds the natural split by the largest gap in the sorted densities. ("Repeated
+ * line" is deliberately NOT the signal — verses repeat too, which mislabels them;
+ * density does not have that failure.) `hookKeywords` overrides when the caller
+ * knows the chorus words. Everything else with words is RAP; gaps with no words
+ * are INSTRUMENTAL; the trailing gap is the OUTRO. The result is a contiguous,
+ * gap-free section list covering [0, songEnd] — exactly the shape the planner
+ * consumes (the per-section performer clip is attached later by the assembler,
+ * rap → rapper clip / hook → singer clip).
+ *
+ * The map can also be authored by hand (an explicit artifact) and fed straight to
+ * the planner; {@link validateVocalMap} guards either source.
+ */
+import type { VocalSection, VocalSectionType } from './vocal-sync-plan.js';
+
+export interface WhisperSegment {
+  start: number;
+  end: number;
+  text: string;
+}
+
+export interface BuildVocalMapOptions {
+  /** Song length in seconds (the map covers [0, songEnd]). */
+  songEnd: number;
+  /**
+   * Phrases that mark the HOOK. If given, a segment is hook when its text
+   * contains any keyword. If omitted, hooks are auto-detected by word density.
+   */
+  hookKeywords?: string[];
+  /** A silent stretch ≥ this is its own INSTRUMENTAL section (default 2.0s). */
+  minInstrumentalSec?: number;
+  /** Vocal spans of the same type within this gap are merged (default 1.2s). */
+  mergeGapSec?: number;
+  /**
+   * Minimum words-per-second gap to declare a hook/rap split (default 0.4). If
+   * the song's densities don't separate this much (e.g. all-rap or all-sung),
+   * everything is treated as RAP — pass hookKeywords to force a hook.
+   */
+  hookDensityGapMin?: number;
+}
+
+const normalize = (text: string): string =>
+  text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, '').replace(/\s+/g, ' ').trim();
+
+/** Classify whisper segments into a contiguous vocal map. PURE. */
+export function buildVocalMap(segments: WhisperSegment[], options: BuildVocalMapOptions): VocalSection[] {
+  const { songEnd } = options;
+  const minGap = options.minInstrumentalSec ?? 2.0;
+  const mergeGap = options.mergeGapSec ?? 1.2;
+
+  // 1. classify each spoken segment as rap (dense) or hook (sustained), by the
+  //    natural gap in word density. wps = words / duration.
+  const spoken = segments.filter((s) => normalize(s.text).length > 0 && s.end > s.start);
+  const keywords = options.hookKeywords?.map(normalize).filter(Boolean);
+  const wpsOf = (s: WhisperSegment): number =>
+    normalize(s.text).split(' ').filter(Boolean).length / Math.max(0.5, s.end - s.start);
+  const densities = spoken.map(wpsOf).sort((a, b) => a - b);
+  let splitGap = 0;
+  let threshold = -1;
+  for (let i = 1; i < densities.length; i++) {
+    const g = densities[i] - densities[i - 1];
+    if (g > splitGap) {
+      splitGap = g;
+      threshold = (densities[i] + densities[i - 1]) / 2;
+    }
+  }
+  const minGapWps = options.hookDensityGapMin ?? 0.4;
+  const isHook = (s: WhisperSegment): boolean => {
+    if (keywords && keywords.length > 0) return keywords.some((k) => normalize(s.text).includes(k));
+    // below the density split = the sustained, sung cluster = hook. Without a
+    // meaningful separation, default to RAP (caller can pass hookKeywords).
+    return splitGap >= minGapWps && wpsOf(s) < threshold;
+  };
+
+  type Span = { start: number; end: number; type: VocalSectionType };
+  const spans: Span[] = spoken
+    .map((s) => ({ start: s.start, end: s.end, type: (isHook(s) ? 'hook' : 'rap') as VocalSectionType }))
+    .sort((a, b) => a.start - b.start);
+
+  // 2. merge consecutive same-type spans within mergeGap
+  const merged: Span[] = [];
+  for (const span of spans) {
+    const last = merged[merged.length - 1];
+    if (last && last.type === span.type && span.start - last.end <= mergeGap) {
+      last.end = Math.max(last.end, span.end);
+    } else {
+      merged.push({ ...span });
+    }
+  }
+
+  // 3. fill the timeline: large gaps -> instrumental, small gaps absorbed into the
+  //    following vocal section; the trailing gap -> outro.
+  const sections: VocalSection[] = [];
+  let cursor = 0;
+  for (const span of merged) {
+    // Transcripts can run past the declared song end (whisper timestamps beyond
+    // the trimmed song). Spans are sorted, so everything from here on is out of
+    // bounds — emitting them produced overlapping, beyond-songEnd sections.
+    if (span.start >= songEnd) break;
+    const gap = span.start - cursor;
+    let start = cursor;
+    if (gap >= minGap) {
+      sections.push({ start: cursor, end: span.start, type: 'instrumental' });
+      start = span.start;
+      cursor = span.start;
+    }
+    const end = Math.min(songEnd, span.end);
+    if (end > start) {
+      sections.push({ start, end, type: span.type });
+      cursor = end;
+    }
+  }
+  const tail = songEnd - cursor;
+  if (tail > 0.1) {
+    sections.push({ start: cursor, end: songEnd, type: cursor > 0 ? 'outro' : 'instrumental' });
+  } else if (tail > 1e-6) {
+    // A micro trailing gap (≤0.1s) is absorbed into the last section — leaving
+    // it uncovered fails validateVocalMap's 1e-6 coverage check and hard-threw
+    // the whole auto path on common transcripts.
+    if (sections.length > 0) sections[sections.length - 1].end = songEnd;
+    else sections.push({ start: 0, end: songEnd, type: 'instrumental' });
+  }
+  return sections;
+}
+
+/**
+ * Validate a vocal map (auto-built or hand-authored): contiguous, in-bounds,
+ * gap-free coverage of [0, songEnd]. Returns a list of human-readable issues
+ * (empty = valid).
+ */
+export function validateVocalMap(sections: VocalSection[], songEnd: number): string[] {
+  const issues: string[] = [];
+  if (sections.length === 0) {
+    return ['vocal map is empty'];
+  }
+  if (Math.abs(sections[0].start) > 1e-6) issues.push(`first section starts at ${sections[0].start}, expected 0`);
+  for (let i = 0; i < sections.length; i++) {
+    const s = sections[i];
+    if (s.end <= s.start) issues.push(`section ${i} (${s.type}) has end <= start`);
+    if (i > 0 && Math.abs(s.start - sections[i - 1].end) > 1e-6) {
+      issues.push(`gap/overlap between section ${i - 1} and ${i} (${sections[i - 1].end} -> ${s.start})`);
+    }
+  }
+  const last = sections[sections.length - 1];
+  if (Math.abs(last.end - songEnd) > 1e-6) issues.push(`last section ends at ${last.end}, expected songEnd ${songEnd}`);
+  return issues;
+}