videoclaw 3.0.0-alpha.1

package/src/video/gen-image-flow.ts

@@ -0,0 +1,491 @@
+import { existsSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import { validateFlowImageMarkers } from './flow-markers.js';
+import { FlowLibraryClient } from './flow-character-library.js';
+import {
+  composeGenImagePrompt,
+  defaultGenImageAspect,
+  type GenImageKind,
+  type GenImageResult,
+} from './gen-image.js';
+
+/**
+ * Google Flow image backend for `gen-image` (useapi.net Google Flow API v1,
+ * POST /google-flow/images). Direct useapi REST from Node — the same precedent
+ * as `native-runway.ts` / `native-dreamina.ts`; it does NOT shell out to the
+ * Bun `vclaw-cli` sidecar.
+ *
+ * Models: imagen-4 (text-to-image), nano-banana (character consistency),
+ * nano-banana-pro (max references, upscale-able). Reference images are
+ * `reference_1..10` mediaGenerationId slots; saved characters are
+ * `character_1..7` slots (June-2026 useapi update). Inline @-markers
+ * (`@reference_N` / `@character_N`, case-insensitive) anchor a slot to a
+ * position in the prompt — each marker MUST have a matching body slot or the
+ * API 400s, so `validateFlowImageMarkers` runs BEFORE any upload or spend.
+ */
+
+export const FLOW_IMAGE_MODELS = ['imagen-4', 'nano-banana', 'nano-banana-pro'] as const;
+export type FlowImageModel = typeof FLOW_IMAGE_MODELS[number];
+
+/**
+ * Per-model TOTAL reference-image budget — references AND character images
+ * share it (verified against the live useapi docs, 2026-06-11): imagen-4
+ * accepts at most 3, nano-banana and nano-banana-pro at most 10. This is the
+ * hard request budget, distinct from the auto-selection heuristic below.
+ */
+export const FLOW_IMAGE_MODEL_REF_BUDGET: Record<FlowImageModel, number> = {
+  'imagen-4': 3,
+  'nano-banana': 10,
+  'nano-banana-pro': 10,
+};
+
+/** POST /google-flow/images slot caps. */
+export const FLOW_IMAGE_MAX_REFERENCES = 10;
+export const FLOW_IMAGE_MAX_CHARACTERS = 7;
+
+const FLOW_IMAGE_DEFAULT_BASE_URL = 'https://api.useapi.net/v1';
+
+/** Aspect ratios POST /google-flow/images accepts (landscape/portrait are legacy aliases). */
+export const FLOW_IMAGE_ASPECTS = ['16:9', '4:3', '1:1', '3:4', '9:16', 'auto', 'landscape', 'portrait'];
+
+/**
+ * Resolve the Flow image model: an explicit id wins (validated against
+ * FLOW_IMAGE_MODELS); otherwise auto-select from the total reference-image
+ * count — 0 refs → imagen-4 (best pure text-to-image), 1-3 → nano-banana
+ * (character consistency), 4+ → nano-banana-pro (max references).
+ */
+export function resolveFlowImageModel(explicit: string | undefined, refCount: number): FlowImageModel {
+  if (explicit !== undefined) {
+    if (!(FLOW_IMAGE_MODELS as readonly string[]).includes(explicit)) {
+      throw new Error(
+        `gen-image (flow): model must be one of ${FLOW_IMAGE_MODELS.join(', ')}, got: ${explicit}`,
+      );
+    }
+    return explicit as FlowImageModel;
+  }
+  if (refCount <= 0) return 'imagen-4';
+  if (refCount <= 3) return 'nano-banana';
+  return 'nano-banana-pro';
+}
+
+/**
+ * How many reference images a saved-character ref contributes to the per-model
+ * budget. Character refs look like `user:1-character:<id>-imgs:N-voice:<v>`;
+ * the `-imgs:N-` segment carries the character's image count. Refs without the
+ * hint count as 1.
+ */
+export function flowCharacterRefImageCount(ref: string): number {
+  const match = /-imgs:(\d+)(?=-|$)/.exec(ref);
+  if (!match) return 1;
+  const n = Number.parseInt(match[1], 10);
+  return n >= 1 ? n : 1;
+}
+
+/**
+ * True when a string looks like a saved Flow character ref (the
+ * `flow-characters.json` `characterRef` format, e.g.
+ * `user:1-character:claw-imgs:1-voice:puck`) rather than a character name.
+ * The `-character:` segment is REQUIRED: mediaGenerationIds share the `user:`
+ * prefix, so a bare-prefix check would let a `--ref` value pasted into
+ * `--character` ship to the API instead of failing fast as character_not_found.
+ */
+export function looksLikeFlowCharacterRef(value: string): boolean {
+  return value.includes('-character:');
+}
+
+/**
+ * True when a `--ref` value is SHAPED like an already-uploaded Flow media
+ * reference (a mediaGenerationId) rather than a local image path. Aligned with
+ * `flow-character-library.ts`'s `isMediaRef` so both modules classify the same
+ * way. Anything NOT media-shaped is treated as a local path and must exist —
+ * classifying by shape (instead of `existsSync` polarity) means a typo'd local
+ * path fails fast up front rather than shipping to the provider as a bogus
+ * mediaGenerationId after other refs were already uploaded (spend).
+ */
+export function looksLikeFlowMediaRef(value: string): boolean {
+  return value.startsWith('user:') || value.includes('-image:') || value.includes('mediaGenerationId');
+}
+
+/**
+ * POST /google-flow/images request body (mirrors the useapi wire shape).
+ * `reference_1..10` are mediaGenerationId values; `character_1..7` are saved
+ * Flow character refs.
+ */
+export interface FlowImageParams {
+  email: string;
+  prompt: string;
+  model: FlowImageModel;
+  aspectRatio: string;
+  count: number;
+  seed?: number;
+  reference_1?: string;
+  reference_2?: string;
+  reference_3?: string;
+  reference_4?: string;
+  reference_5?: string;
+  reference_6?: string;
+  reference_7?: string;
+  reference_8?: string;
+  reference_9?: string;
+  reference_10?: string;
+  character_1?: string;
+  character_2?: string;
+  character_3?: string;
+  character_4?: string;
+  character_5?: string;
+  character_6?: string;
+  character_7?: string;
+}
+
+export interface BuildFlowImageParamsOptions {
+  prompt: string;
+  kind: GenImageKind;
+  /** useapi account email carried in the request body. */
+  email: string;
+  /** Explicit model id; omitted → auto-selected from the total reference count. */
+  model?: string;
+  /** Defaults per kind (16:9 for screen, 1:1 otherwise). */
+  aspectRatio?: string;
+  /** Images per generation, 1-4. Defaults to 1 (the API default of 4 would 4x the spend). */
+  count?: number;
+  /** Seed for reproducible results (non-negative integer). */
+  seed?: number;
+  /**
+   * Values for reference_1..N in order. Validation only depends on the COUNT,
+   * so callers may pass not-yet-uploaded local paths for fail-fast/dry-run
+   * composition and rebuild with resolved mediaGenerationIds before the POST.
+   */
+  references?: string[];
+  /** Saved Flow character refs for character_1..N in order. */
+  characterRefs?: string[];
+}
+
+/**
+ * Compose the POST /google-flow/images body (PURE — no I/O, no env). Weaves
+ * the gen-image per-kind render directive into the prompt (consistent with
+ * `buildGenImageRequest`), assigns `reference_1..N` / `character_1..N` slots in
+ * input order, and enforces the hard request contract as errors:
+ *
+ * - ≤10 references, ≤7 characters (slot caps);
+ * - model × reference budget (imagen-4 ≤3, nano-banana(-pro) ≤10) where
+ *   character refs COUNT TOWARD the same budget (each contributes its
+ *   `-imgs:N-` image count, default 1);
+ * - count 1-4, seed ≥0, aspect ratio membership (`auto` needs a nano-banana
+ *   model and at least one reference);
+ * - every inline @-marker has a matching slot (`validateFlowImageMarkers`).
+ */
+export function buildFlowImageParams(opts: BuildFlowImageParamsOptions): FlowImageParams {
+  const references = opts.references ?? [];
+  const characterRefs = opts.characterRefs ?? [];
+  if (references.length > FLOW_IMAGE_MAX_REFERENCES) {
+    throw new Error(
+      `gen-image (flow): at most ${FLOW_IMAGE_MAX_REFERENCES} references are supported (reference_1..${FLOW_IMAGE_MAX_REFERENCES}), got ${references.length}.`,
+    );
+  }
+  if (characterRefs.length > FLOW_IMAGE_MAX_CHARACTERS) {
+    throw new Error(
+      `gen-image (flow): at most ${FLOW_IMAGE_MAX_CHARACTERS} characters are supported (character_1..${FLOW_IMAGE_MAX_CHARACTERS}), got ${characterRefs.length}.`,
+    );
+  }
+
+  const count = opts.count ?? 1;
+  if (!Number.isInteger(count) || count < 1 || count > 4) {
+    throw new Error(`gen-image (flow): count must be an integer between 1 and 4, got: ${opts.count}.`);
+  }
+  if (opts.seed !== undefined && (!Number.isInteger(opts.seed) || opts.seed < 0)) {
+    throw new Error(`gen-image (flow): seed must be a non-negative integer, got: ${opts.seed}.`);
+  }
+
+  // Character refs roll their per-character image counts into the SAME
+  // per-model budget as plain references.
+  const characterImageCount = characterRefs.reduce((sum, ref) => sum + flowCharacterRefImageCount(ref), 0);
+  const totalRefImages = references.length + characterImageCount;
+  const model = resolveFlowImageModel(opts.model, totalRefImages);
+  const budget = FLOW_IMAGE_MODEL_REF_BUDGET[model];
+  if (totalRefImages > budget) {
+    throw new Error(
+      `gen-image (flow): ${model} accepts at most ${budget} reference images, got ${totalRefImages} ` +
+      `(${references.length} reference${references.length === 1 ? '' : 's'} + ${characterImageCount} character image${characterImageCount === 1 ? '' : 's'} — ` +
+      'character refs count toward the same per-model budget). ' +
+      'Use nano-banana-pro for up to 10, or drop references.',
+    );
+  }
+
+  const aspectRatio = opts.aspectRatio ?? defaultGenImageAspect(opts.kind);
+  if (!FLOW_IMAGE_ASPECTS.includes(aspectRatio)) {
+    throw new Error(
+      `gen-image (flow): aspect ratio must be one of ${FLOW_IMAGE_ASPECTS.join(', ')}, got: ${aspectRatio}.`,
+    );
+  }
+  if (aspectRatio === 'auto' && (model === 'imagen-4' || totalRefImages === 0)) {
+    throw new Error(
+      'gen-image (flow): aspect ratio "auto" requires a nano-banana model and at least one reference image.',
+    );
+  }
+
+  const prompt = composeGenImagePrompt(opts.prompt, opts.kind);
+  const markerErrors = validateFlowImageMarkers(prompt, {
+    characterCount: characterRefs.length,
+    referenceCount: references.length,
+  });
+  if (markerErrors.length > 0) {
+    throw new Error(`gen-image (flow): invalid @-markers in prompt:\n- ${markerErrors.join('\n- ')}`);
+  }
+
+  const params: FlowImageParams = {
+    email: opts.email,
+    prompt,
+    model,
+    aspectRatio,
+    count,
+    ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+  };
+  const slots = params as unknown as Record<string, unknown>;
+  references.forEach((ref, i) => {
+    slots[`reference_${i + 1}`] = ref;
+  });
+  characterRefs.forEach((ref, i) => {
+    slots[`character_${i + 1}`] = ref;
+  });
+  return params;
+}
+
+/**
+ * Output path for the Nth generated image (1-based): the first goes to
+ * `outputPath` verbatim, extras get a `-2`/`-3`/`-4` suffix before the
+ * extension. PURE.
+ */
+export function flowOutputPathForIndex(outputPath: string, ordinal: number): string {
+  if (ordinal <= 1) return outputPath;
+  return /\.[^./\\]+$/.test(outputPath)
+    ? outputPath.replace(/(\.[^./\\]+)$/, `-${ordinal}$1`)
+    : `${outputPath}-${ordinal}`;
+}
+
+export interface FlowGenImageOptions {
+  prompt: string;
+  kind: GenImageKind;
+  /** Where the first resulting image is written (extras get -2/-3/-4 suffixes). */
+  outputPath: string;
+  /** Explicit model id; omitted → auto-selected from the reference count. */
+  model?: string;
+  aspectRatio?: string;
+  /** Images per generation (1-4, default 1). */
+  count?: number;
+  seed?: number;
+  /**
+   * Already-uploaded mediaGenerationIds (media-ref-shaped, passed through) or
+   * local image paths (anything else; must exist, uploaded first).
+   */
+  refs?: string[];
+  /** Character names already resolved to saved Flow character refs. */
+  characterRefs?: string[];
+  /** useapi token; falls back to USEAPI_API_TOKEN. */
+  apiToken?: string;
+  /** useapi account email; falls back to USEAPI_ACCOUNT_EMAIL. */
+  accountEmail?: string;
+  /** useapi base URL override (default https://api.useapi.net/v1). */
+  baseUrl?: string;
+  /** Injectable fetch for tests; defaults to the global fetch. */
+  fetcher?: typeof fetch;
+}
+
+export interface FlowGenImageResult extends GenImageResult {
+  /** Every written file in order (paths[0] === path). */
+  paths: string[];
+  /** mediaGenerationIds of the generated image(s), reusable as reference_N inputs downstream. */
+  mediaGenerationIds: string[];
+  /** Images requested per generation. */
+  count: number;
+  /** Seed echoed when one was requested. */
+  seed?: number;
+  /** mediaGenerationIds minted for uploaded local reference paths (in --ref order). */
+  uploadedReferenceIds: string[];
+}
+
+/** Raw POST /google-flow/images response (mirrors the useapi wire shape). */
+interface FlowImageResponseRaw {
+  jobId?: string;
+  media?: Array<{
+    name?: string;
+    image?: {
+      generatedImage?: {
+        seed?: number;
+        mediaGenerationId?: string;
+        fifeUrl?: string;
+      };
+    };
+  }>;
+  model?: string;
+  error?: string;
+}
+
+function normalizeFlowHttpError(label: string, status: number, bodyText: string): Error {
+  let detail = bodyText.slice(0, 300);
+  try {
+    // useapi Flow `POST /images` 429/403 bodies changed shape on 2026-06-15
+    // (https://useapi.net/docs/changelog): the top-level `error` is now a
+    // classification STRING (e.g. "captcha_quality: ..." /
+    // "PUBLIC_ERROR_USER_QUOTA_REACHED"), and Google's structured error body +
+    // captcha metadata moved into a nested `response` wrapper. Older bodies put
+    // the structured object directly at `error`. Read both so the message keeps
+    // the classification AND the underlying Google reason regardless of shape.
+    const parsed = JSON.parse(bodyText) as {
+      error?: unknown;
+      response?: { error?: { code?: unknown; message?: unknown; details?: Array<{ reason?: unknown }> } };
+    };
+    const classification =
+      typeof parsed.error === 'string'
+        ? parsed.error
+        : parsed.error !== undefined
+          ? JSON.stringify(parsed.error)
+          : undefined;
+    const nested = parsed.response?.error;
+    const reason =
+      (Array.isArray(nested?.details) && typeof nested?.details?.[0]?.reason === 'string'
+        ? nested.details?.[0]?.reason
+        : undefined)
+      ?? (typeof nested?.message === 'string' ? nested.message : undefined)
+      ?? (typeof nested?.code === 'string' ? nested.code : undefined);
+    const parts = [classification, reason].filter(
+      (p): p is string => typeof p === 'string' && p.length > 0,
+    );
+    if (parts.length > 0) detail = parts.join(' — ');
+  } catch {
+    // non-JSON body — keep the truncated text
+  }
+  return new Error(`gen-image (flow) ${label} failed (HTTP ${status}): ${detail}`);
+}
+
+/**
+ * Generate image(s) via Google Flow and write them to disk. Sequence:
+ * validate markers + the model×reference matrix FIRST (zero network on
+ * validation failure) → classify refs by shape and fail fast on any
+ * non-media-shaped ref whose local path is missing (still zero network) →
+ * upload the local reference paths via the existing
+ * `FlowLibraryClient.uploadImage` → POST /google-flow/images → download every
+ * generated image (first to `outputPath`, extras suffixed -2/-3/-4). Returns
+ * the gen-image result shape with `backend: 'flow'` plus the generated
+ * `mediaGenerationIds` for downstream reuse.
+ */
+export async function generateViaFlow(opts: FlowGenImageOptions): Promise<FlowGenImageResult> {
+  const apiToken = opts.apiToken ?? process.env.USEAPI_API_TOKEN;
+  if (!apiToken) {
+    throw new Error('gen-image --backend flow requires a useapi token (set USEAPI_API_TOKEN or pass apiToken).');
+  }
+  const accountEmail = opts.accountEmail ?? process.env.USEAPI_ACCOUNT_EMAIL;
+  if (!accountEmail) {
+    throw new Error('gen-image --backend flow requires the useapi account email (set USEAPI_ACCOUNT_EMAIL or pass accountEmail).');
+  }
+  const baseUrl = (opts.baseUrl ?? FLOW_IMAGE_DEFAULT_BASE_URL).replace(/\/$/, '');
+  const fetcher = opts.fetcher ?? fetch;
+  const refs = opts.refs ?? [];
+  const characterRefs = opts.characterRefs ?? [];
+
+  const buildOpts = {
+    prompt: opts.prompt,
+    kind: opts.kind,
+    email: accountEmail,
+    ...(opts.model !== undefined ? { model: opts.model } : {}),
+    ...(opts.aspectRatio !== undefined ? { aspectRatio: opts.aspectRatio } : {}),
+    ...(opts.count !== undefined ? { count: opts.count } : {}),
+    ...(opts.seed !== undefined ? { seed: opts.seed } : {}),
+    characterRefs,
+  };
+  // Fail-fast pass BEFORE any upload/spend: validation only depends on slot
+  // counts and the prompt, so raw (not-yet-uploaded) reference values suffice.
+  buildFlowImageParams({ ...buildOpts, references: refs });
+
+  // Classify every ref by SHAPE first (looksLikeFlowMediaRef): media-ref-shaped
+  // values pass through verbatim; everything else is a local image path and
+  // must exist. The existence scan runs BEFORE any upload so a typo'd path
+  // fails fast with zero network instead of shipping as a bogus
+  // mediaGenerationId after sibling refs were already uploaded (spend).
+  const missingLocalRefs = refs.filter((ref) => !looksLikeFlowMediaRef(ref) && !existsSync(ref));
+  if (missingLocalRefs.length > 0) {
+    throw new Error(
+      `gen-image (flow): --ref local path${missingLocalRefs.length === 1 ? '' : 's'} not found: ${missingLocalRefs.join(', ')}. ` +
+      'Values not shaped like an uploaded Flow media ref (user:...) are treated as local image paths and must exist.',
+    );
+  }
+
+  const client = new FlowLibraryClient({ apiToken, accountEmail, baseUrl, fetchImpl: fetcher });
+  const uploadedReferenceIds: string[] = [];
+  const resolvedRefs: string[] = [];
+  for (const ref of refs) {
+    if (looksLikeFlowMediaRef(ref)) {
+      resolvedRefs.push(ref);
+    } else {
+      const mediaId = await client.uploadImage(ref);
+      uploadedReferenceIds.push(mediaId);
+      resolvedRefs.push(mediaId);
+    }
+  }
+
+  const params = buildFlowImageParams({ ...buildOpts, references: resolvedRefs });
+  const response = await fetcher(`${baseUrl}/google-flow/images`, {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${apiToken}`,
+      'Content-Type': 'application/json',
+      Accept: 'application/json',
+    },
+    body: JSON.stringify(params),
+  });
+  if (!response.ok) {
+    throw normalizeFlowHttpError('POST /google-flow/images', response.status, await response.text());
+  }
+  const raw = (await response.json()) as FlowImageResponseRaw;
+  const generated = (raw.media ?? [])
+    .map((item) => item.image?.generatedImage)
+    .filter((img): img is NonNullable<typeof img> => !!img?.mediaGenerationId);
+  if (generated.length === 0) {
+    throw new Error(
+      `gen-image (flow): POST /google-flow/images succeeded but returned no generated images${raw.error ? ` (${raw.error})` : ''}.`,
+    );
+  }
+
+  const paths: string[] = [];
+  const mediaGenerationIds: string[] = [];
+  let firstBytes = 0;
+  let firstUrl: string | undefined;
+  for (let i = 0; i < generated.length; i += 1) {
+    const image = generated[i];
+    mediaGenerationIds.push(image.mediaGenerationId!);
+    const url = image.fifeUrl;
+    if (!url) {
+      throw new Error(
+        `gen-image (flow): generated image ${i + 1} (${image.mediaGenerationId}) has no downloadable URL (fifeUrl missing).`,
+      );
+    }
+    const download = await fetcher(url, {});
+    if (!download.ok) {
+      throw new Error(`gen-image (flow): image download failed with HTTP ${download.status}: ${url}`);
+    }
+    const bytes = Buffer.from(await download.arrayBuffer());
+    const target = flowOutputPathForIndex(opts.outputPath, i + 1);
+    await mkdir(dirname(target), { recursive: true });
+    await writeFile(target, bytes);
+    paths.push(target);
+    if (i === 0) {
+      firstBytes = bytes.length;
+      firstUrl = url;
+    }
+  }
+
+  return {
+    path: paths[0],
+    kind: opts.kind,
+    backend: 'flow',
+    model: params.model,
+    aspectRatio: params.aspectRatio,
+    imageUrl: firstUrl,
+    sizeBytes: firstBytes,
+    paths,
+    mediaGenerationIds,
+    count: params.count,
+    ...(params.seed !== undefined ? { seed: params.seed } : {}),
+    uploadedReferenceIds,
+  };
+}