visual-ai-assertions 0.8.0 → 0.10.0

package/dist/index.js

@@ -115,6 +115,12 @@ var VisualAIImageError = class extends VisualAIError {
     this.name = "VisualAIImageError";
   }
 };
+var VisualAIVideoError = class extends VisualAIError {
+  constructor(message) {
+    super(message, "VIDEO_INVALID");
+    this.name = "VisualAIVideoError";
+  }
+};
 var VisualAIResponseParseError = class extends VisualAIError {
   rawResponse;
   constructor(message, rawResponse) {
@@ -148,7 +154,7 @@ var VisualAIAssertionError = class extends VisualAIError {
   }
 };
 function isVisualAIKnownError(error) {
-  return error instanceof VisualAIAuthError || error instanceof VisualAIRateLimitError || error instanceof VisualAIProviderError || error instanceof VisualAIImageError || error instanceof VisualAIResponseParseError || error instanceof VisualAITruncationError || error instanceof VisualAIConfigError || error instanceof VisualAIAssertionError;
+  return error instanceof VisualAIAuthError || error instanceof VisualAIRateLimitError || error instanceof VisualAIProviderError || error instanceof VisualAIImageError || error instanceof VisualAIVideoError || error instanceof VisualAIResponseParseError || error instanceof VisualAITruncationError || error instanceof VisualAIConfigError || error instanceof VisualAIAssertionError;
 }
 
 // src/core/prompt.ts
@@ -162,7 +168,7 @@ Each issue must have:
 - "description": what the issue is
 - "suggestion": how to fix or improve it
 `;
-var CHECK_OUTPUT_SCHEMA = `IMPORTANT: Follow this evaluation order:
+var CHECK_OUTPUT_SCHEMA_IMAGE = `IMPORTANT: Follow this evaluation order:
 1. First, evaluate EACH statement independently and populate the "statements" array
 2. Then, set "pass" to true ONLY if every statement passed (logical AND of all statement results)
 3. Write "reasoning" as a brief overall summary of the evaluation
@@ -202,7 +208,46 @@ Example for a failing check:
   ]
 }
 ${JSON_INSTRUCTIONS}`;
-var ASK_OUTPUT_SCHEMA = `Respond with a JSON object matching this exact structure:
+var CHECK_OUTPUT_SCHEMA_VIDEO = `IMPORTANT: Follow this evaluation order:
+1. First, evaluate EACH statement independently across the entire timeline and populate the "statements" array
+2. A statement passes if it is true at ANY frame of the timeline, unless the wording explicitly says otherwise (e.g. "throughout", "at all times")
+3. For each statement that passes, set "timestampSeconds" to the timestamp of the frame that most clearly demonstrates it (or where it first becomes true). Use null when the statement fails or applies across the whole clip.
+4. Then, set "pass" to true ONLY if every statement passed (logical AND of all statement results)
+5. Write "reasoning" as a brief overall summary of the evaluation
+6. Include "issues" only for statements that failed
+
+Respond with a JSON object matching this exact structure:
+{
+  "pass": boolean,          // true ONLY if ALL statements passed \u2014 derive from statements array
+  "reasoning": string,      // brief overall summary of the evaluation
+  "issues": [...],          // one issue per failing statement (empty if all pass)
+  "statements": [           // one entry per statement, in order \u2014 evaluate these FIRST
+    {
+      "statement": string,  // the original statement text
+      "pass": boolean,      // whether this statement is true at any point in the timeline
+      "reasoning": string,  // explanation for this statement, citing frame timestamps where relevant
+      "confidence": "high" | "medium" | "low",
+      "timestampSeconds": number | null
+        // seconds from the start of the clip where the statement is most clearly true,
+        // or null if it failed / applies across the whole clip
+    }
+  ]
+}
+${ISSUE_SCHEMA_INSTRUCTIONS}
+
+Only include issues for statements that fail. If all statements pass, issues should be an empty array.
+
+Example for a passing video check:
+{
+  "pass": true,
+  "reasoning": "The success toast appeared briefly around 3.5s.",
+  "issues": [],
+  "statements": [
+    { "statement": "A success toast with text 'Saved' appears", "pass": true, "reasoning": "A green toast labeled 'Saved' is visible in the bottom-right at the 3.5s frame", "confidence": "high", "timestampSeconds": 3.5 }
+  ]
+}
+${JSON_INSTRUCTIONS}`;
+var ASK_OUTPUT_SCHEMA_IMAGE = `Respond with a JSON object matching this exact structure:
 {
   "summary": string,        // high-level analysis summary
   "issues": [...]           // list of issues/findings, can be empty
@@ -223,6 +268,17 @@ Example:
   ]
 }
 ${JSON_INSTRUCTIONS}`;
+var ASK_OUTPUT_SCHEMA_VIDEO = `Respond with a JSON object matching this exact structure:
+{
+  "summary": string,            // high-level summary of what happens across the timeline
+  "issues": [...],              // list of issues/findings, can be empty
+  "frameReferences": number[]   // 0-based indices of frames the answer relies on (in order)
+}
+${ISSUE_SCHEMA_INSTRUCTIONS}
+
+Prioritize issues by severity (critical / major / minor) as for image input.
+Cite frame indices in "frameReferences" so the user can locate the moments you describe.
+${JSON_INSTRUCTIONS}`;
 var COMPARE_OUTPUT_SCHEMA = `Respond with a JSON object matching this exact structure:
 {
   "pass": boolean,          // true if no critical or major changes found
@@ -241,7 +297,19 @@ var COMPARE_OUTPUT_SCHEMA = `Respond with a JSON object matching this exact stru
 If the images appear identical, set pass to true, explain in reasoning, and return an empty changes array.
 ${JSON_INSTRUCTIONS}`;
 var DEFAULT_CHECK_ROLE = "You are a visual QA assistant. Evaluate the provided image precisely and objectively.";
+var DEFAULT_CHECK_ROLE_VIDEO = "You are a visual QA assistant. Evaluate the provided sequence of video frames precisely and objectively, treating them as a chronological timeline.";
 var DEFAULT_ASK_ROLE = "You are a visual QA assistant. Analyze the provided image based on the user's request.";
+var DEFAULT_ASK_ROLE_VIDEO = "You are a visual QA assistant. Analyze the provided sequence of video frames as a chronological timeline based on the user's request.";
+function buildVideoTimelineSection(frameTimestamps, durationSeconds) {
+  const formatted = frameTimestamps.map((t, i) => `  ${i}: ${t.toFixed(2)}s`).join("\n");
+  return `Video timeline:
+- Total duration: ${durationSeconds.toFixed(2)}s
+- ${frameTimestamps.length} frames sampled (in chronological order)
+- Frame index \u2192 timestamp:
+${formatted}
+
+Treat the attached images as a chronological timeline. The first image is the earliest frame, the last is the latest. Refer to frames by timestamp where helpful.`;
+}
 var COMPARE_ROLE = "You are performing a visual regression test. Compare the BEFORE image (baseline) to the AFTER image (current) and identify all visual differences. Flag changes that appear unintentional or problematic.";
 var COMPARE_EDGE_RULES = [
   "The BEFORE image is the baseline/expected state.",
@@ -254,22 +322,31 @@ function buildInstructionsSection(instructions) {
 function buildCheckPrompt(statements, options) {
   const stmts = Array.isArray(statements) ? statements : [statements];
   const statementsBlock = stmts.map((s, i) => `${i + 1}. "${s}"`).join("\n");
-  const sections = [options?.role ?? DEFAULT_CHECK_ROLE];
+  const media = options?.media;
+  const defaultRole = media?.kind === "video" ? DEFAULT_CHECK_ROLE_VIDEO : DEFAULT_CHECK_ROLE;
+  const sections = [options?.role ?? defaultRole];
+  if (media?.kind === "video") {
+    sections.push(buildVideoTimelineSection(media.frameTimestamps, media.durationSeconds));
+  }
   if (options?.instructions && options.instructions.length > 0) {
     sections.push(buildInstructionsSection(options.instructions));
   }
   sections.push(`Statements to evaluate:
 ${statementsBlock}`);
-  sections.push(CHECK_OUTPUT_SCHEMA);
+  sections.push(media?.kind === "video" ? CHECK_OUTPUT_SCHEMA_VIDEO : CHECK_OUTPUT_SCHEMA_IMAGE);
   return sections.join("\n\n");
 }
 function buildAskPrompt(userPrompt, options) {
-  const sections = [DEFAULT_ASK_ROLE];
+  const media = options?.media;
+  const sections = [media?.kind === "video" ? DEFAULT_ASK_ROLE_VIDEO : DEFAULT_ASK_ROLE];
+  if (media?.kind === "video") {
+    sections.push(buildVideoTimelineSection(media.frameTimestamps, media.durationSeconds));
+  }
   if (options?.instructions && options.instructions.length > 0) {
     sections.push(buildInstructionsSection(options.instructions));
   }
   sections.push(`User request: ${userPrompt}`);
-  sections.push(ASK_OUTPUT_SCHEMA);
+  sections.push(media?.kind === "video" ? ASK_OUTPUT_SCHEMA_VIDEO : ASK_OUTPUT_SCHEMA_IMAGE);
   return sections.join("\n\n");
 }
 function buildAiDiffPrompt() {
@@ -964,6 +1041,51 @@ async function generateAiDiff(imgA, imgB, model, driver) {
 import { readFile } from "fs/promises";
 import { extname } from "path";
 import sharp2 from "sharp";
+
+// src/core/input-detect.ts
+function isFilePath(input) {
+  return input.startsWith("/") || input.startsWith("./") || input.startsWith("../") || input.includes("\\");
+}
+function isUrl(input) {
+  return input.startsWith("http://") || input.startsWith("https://");
+}
+function isDataUrl(input) {
+  return input.startsWith("data:");
+}
+function parseDataUrl(input) {
+  const match = /^data:([^;]+);base64,(.+)$/.exec(input);
+  if (!match?.[1] || !match[2]) return null;
+  return { mimeType: match[1], base64Payload: match[2] };
+}
+function decodeBase64(payload) {
+  if (!/^[A-Za-z0-9+/\n\r]+=*$/.test(payload)) {
+    throw new Error("Invalid base64 string");
+  }
+  return Buffer.from(payload, "base64");
+}
+function looksLikeImageBase64(input) {
+  return input.startsWith("iVBOR") || // PNG  (0x89 0x50 0x4E 0x47)
+  input.startsWith("/9j/") || // JPEG (0xFF 0xD8 0xFF)
+  input.startsWith("R0lGOD") || // GIF  (0x47 0x49 0x46)
+  input.startsWith("UklGR");
+}
+function looksLikeVideoBase64(input) {
+  return input.startsWith("GkXf") || input.startsWith("AAAA");
+}
+async function fetchToBuffer(url, timeoutMs) {
+  const response = await fetch(url, {
+    signal: AbortSignal.timeout(timeoutMs)
+  });
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}`);
+  }
+  const arrayBuffer = await response.arrayBuffer();
+  const data = Buffer.from(arrayBuffer);
+  const contentType = response.headers.get("content-type")?.split(";")[0]?.trim() ?? null;
+  return { data, contentType };
+}
+
+// src/core/image.ts
 var SUPPORTED_FORMATS = /* @__PURE__ */ new Set([
   "image/jpeg",
   "image/png",
@@ -986,18 +1108,6 @@ function getMimeFromExtension(filePath) {
   const ext = extname(filePath).toLowerCase();
   return EXTENSION_TO_MIME[ext];
 }
-function isFilePath(input) {
-  return input.startsWith("/") || input.startsWith("./") || input.startsWith("../") || input.includes("\\");
-}
-function isUrl(input) {
-  return input.startsWith("http://") || input.startsWith("https://");
-}
-function isBase64Image(input) {
-  return input.startsWith("iVBOR") || // PNG  (0x89 0x50 0x4E 0x47)
-  input.startsWith("/9j/") || // JPEG (0xFF 0xD8 0xFF)
-  input.startsWith("R0lGOD") || // GIF  (0x47 0x49 0x46)
-  input.startsWith("UklGR");
-}
 function detectMimeType(data) {
   if (data[0] === 255 && data[1] === 216 && data[2] === 255) {
     return "image/jpeg";
@@ -1051,45 +1161,38 @@ async function loadFromFilePath(filePath) {
   return { data: fileData, mimeType };
 }
 async function loadFromUrl(url) {
-  let response;
+  let result;
   try {
-    response = await fetch(url, {
-      signal: AbortSignal.timeout(URL_FETCH_TIMEOUT_MS)
-    });
+    result = await fetchToBuffer(url, URL_FETCH_TIMEOUT_MS);
   } catch (err) {
     throw new VisualAIImageError(
       `Failed to fetch image from URL: ${url} \u2014 ${err instanceof Error ? err.message : String(err)}`
     );
   }
-  if (!response.ok) {
-    throw new VisualAIImageError(
-      `Failed to fetch image from URL: ${url} \u2014 HTTP ${response.status}`
-    );
-  }
-  const arrayBuffer = await response.arrayBuffer();
-  const data = Buffer.from(arrayBuffer);
-  const contentType = response.headers.get("content-type")?.split(";")[0]?.trim() ?? null;
+  const { data, contentType } = result;
   const mimeType = contentType && isSupportedMimeType(contentType) ? contentType : detectMimeType(data);
   return { data, mimeType };
 }
 function loadFromBase64(input) {
   let base64Data = input;
   let mimeType;
-  if (input.startsWith("data:")) {
-    const match = /^data:(image\/[^;]+);base64,(.+)$/.exec(input);
-    if (!match?.[1] || !match[2]) {
+  if (isDataUrl(input)) {
+    const parsed = parseDataUrl(input);
+    if (!parsed) {
       throw new VisualAIImageError("Invalid data URL format");
     }
-    if (!isSupportedMimeType(match[1])) {
-      throw new VisualAIImageError(`Unsupported image format: ${match[1]}`);
+    if (!isSupportedMimeType(parsed.mimeType)) {
+      throw new VisualAIImageError(`Unsupported image format: ${parsed.mimeType}`);
     }
-    mimeType = match[1];
-    base64Data = match[2];
+    mimeType = parsed.mimeType;
+    base64Data = parsed.base64Payload;
   }
-  if (!/^[A-Za-z0-9+/\n\r]+=*$/.test(base64Data)) {
+  let data;
+  try {
+    data = decodeBase64(base64Data);
+  } catch {
     throw new VisualAIImageError("Invalid base64 string");
   }
-  const data = Buffer.from(base64Data, "base64");
   if (data.length === 0) {
     throw new VisualAIImageError("Empty image data after base64 decode");
   }
@@ -1108,9 +1211,9 @@ async function normalizeImage(input) {
   } else if (typeof input === "string") {
     if (isUrl(input)) {
       ({ data, mimeType } = await loadFromUrl(input));
-    } else if (input.startsWith("data:")) {
+    } else if (isDataUrl(input)) {
       ({ data, mimeType } = loadFromBase64(input));
-    } else if (isBase64Image(input)) {
+    } else if (looksLikeImageBase64(input)) {
       ({ data, mimeType } = loadFromBase64(input));
     } else if (isFilePath(input)) {
       ({ data, mimeType } = await loadFromFilePath(input));
@@ -1138,6 +1241,435 @@ async function normalizeImage(input) {
   };
 }
 
+// src/core/debug-frames.ts
+import { randomBytes } from "crypto";
+import { mkdir, writeFile } from "fs/promises";
+import { join, resolve } from "path";
+var DEBUG_FRAMES_ENV = "VISUAL_AI_DEBUG_FRAMES";
+var DEBUG_FRAMES_DIR_ENV = "VISUAL_AI_DEBUG_FRAMES_DIR";
+var DEFAULT_DIR_NAME = "visual-ai-debug-frames";
+function isEnabled(env) {
+  const raw = env[DEBUG_FRAMES_ENV];
+  if (raw === void 0 || raw === "") return false;
+  const lower = raw.toLowerCase();
+  return lower === "true" || lower === "1";
+}
+function timestampSlug(date) {
+  return date.toISOString().replace(/[:.]/g, "-");
+}
+function paddedIndex(value, total) {
+  const width = Math.max(2, String(total - 1).length);
+  return String(value).padStart(width, "0");
+}
+function extensionFromMimeType(mimeType) {
+  if (mimeType === "image/png") return ".png";
+  if (mimeType === "image/webp") return ".webp";
+  return ".jpg";
+}
+async function saveDebugFrames(frames, env = process.env) {
+  if (!isEnabled(env)) return void 0;
+  if (frames.length === 0) return void 0;
+  const baseDir = env[DEBUG_FRAMES_DIR_ENV]?.trim() || DEFAULT_DIR_NAME;
+  const runDir = resolve(baseDir, `${timestampSlug(/* @__PURE__ */ new Date())}-${randomBytes(3).toString("hex")}`);
+  try {
+    await mkdir(runDir, { recursive: true });
+    await Promise.all(
+      frames.map((frame) => {
+        const idx = paddedIndex(frame.index, frames.length);
+        const ts = frame.timestampSeconds.toFixed(2);
+        const ext = extensionFromMimeType(frame.mimeType);
+        const filename = `frame-${idx}-t${ts}s${ext}`;
+        return writeFile(join(runDir, filename), frame.data);
+      })
+    );
+  } catch (err) {
+    process.stderr.write(
+      `[visual-ai-assertions] warning: failed to save debug frames to ${runDir}: ${err instanceof Error ? err.message : String(err)}
+`
+    );
+    return void 0;
+  }
+  process.stderr.write(
+    `[visual-ai-assertions] Saved ${frames.length} debug frame(s) to ${runDir}
+`
+  );
+  return runDir;
+}
+
+// src/core/video.ts
+import { mkdtemp, readFile as readFile2, readdir, rm, writeFile as writeFile2 } from "fs/promises";
+import { tmpdir } from "os";
+import { extname as extname2, join as join2 } from "path";
+var FRAME_MAX_DIMENSION = 1568;
+var DEFAULT_FPS = 1;
+var DEFAULT_MAX_FRAMES = 10;
+var DEFAULT_MAX_DURATION_SECONDS = 10;
+var MAX_FRAMES_HARD_CAP = 60;
+var FFPROBE_TIMEOUT_MS = 15e3;
+var FFMPEG_RUN_TIMEOUT_MS = 6e4;
+var VIDEO_EXTENSIONS = {
+  ".mp4": "video/mp4",
+  ".m4v": "video/mp4",
+  ".webm": "video/webm",
+  ".mov": "video/quicktime",
+  ".qt": "video/quicktime",
+  ".mkv": "video/x-matroska"
+};
+var VIDEO_MIME_TYPES = /* @__PURE__ */ new Set([
+  "video/mp4",
+  "video/webm",
+  "video/quicktime",
+  "video/x-matroska"
+]);
+function isSupportedVideoMimeType(value) {
+  return VIDEO_MIME_TYPES.has(value);
+}
+function getVideoMimeFromExtension(filePath) {
+  const ext = extname2(filePath).toLowerCase();
+  return VIDEO_EXTENSIONS[ext];
+}
+function detectVideoMimeType(data) {
+  if (data.length < 12) return null;
+  if (data[4] === 102 && data[5] === 116 && data[6] === 121 && data[7] === 112) {
+    if (data[8] === 113 && data[9] === 116 && data[10] === 32 && data[11] === 32) {
+      return "video/quicktime";
+    }
+    return "video/mp4";
+  }
+  if (data[0] === 26 && data[1] === 69 && data[2] === 223 && data[3] === 163) {
+    return "video/webm";
+  }
+  return null;
+}
+async function resolveVideoToPath(input) {
+  if (Buffer.isBuffer(input) || input instanceof Uint8Array) {
+    const buf2 = Buffer.isBuffer(input) ? input : Buffer.from(input);
+    const mimeType2 = detectVideoMimeType(buf2);
+    if (!mimeType2) {
+      throw new VisualAIVideoError("Unable to detect video format from buffer contents");
+    }
+    return writeBufferToTemp(buf2, mimeType2);
+  }
+  if (typeof input !== "string") {
+    throw new VisualAIVideoError(
+      "Invalid video input: expected Buffer, Uint8Array, file path, data URL, or base64 string"
+    );
+  }
+  if (isDataUrl(input)) {
+    const parsed = parseDataUrl(input);
+    if (!parsed) {
+      throw new VisualAIVideoError("Invalid data URL format");
+    }
+    if (!isSupportedVideoMimeType(parsed.mimeType)) {
+      throw new VisualAIVideoError(`Unsupported video format: ${parsed.mimeType}`);
+    }
+    let buf2;
+    try {
+      buf2 = decodeBase64(parsed.base64Payload);
+    } catch {
+      throw new VisualAIVideoError("Invalid base64 payload in data URL");
+    }
+    return writeBufferToTemp(buf2, parsed.mimeType);
+  }
+  if (isFilePath(input)) {
+    const mimeType2 = getVideoMimeFromExtension(input);
+    if (!mimeType2) {
+      throw new VisualAIVideoError(
+        `Unsupported video file extension: ${input}. Supported: .mp4, .webm, .mov, .mkv`
+      );
+    }
+    return { path: input, mimeType: mimeType2, cleanup: async () => {
+    } };
+  }
+  let buf;
+  try {
+    buf = decodeBase64(input);
+  } catch {
+    throw new VisualAIVideoError(
+      `Unrecognized video input: "${input.slice(0, 80)}". Expected a file path, data URL, or base64-encoded video string.`
+    );
+  }
+  const mimeType = detectVideoMimeType(buf);
+  if (!mimeType) {
+    throw new VisualAIVideoError(
+      `Unrecognized video input: "${input.slice(0, 80)}". Expected a file path, data URL, or base64-encoded video string.`
+    );
+  }
+  return writeBufferToTemp(buf, mimeType);
+}
+async function writeBufferToTemp(data, mimeType) {
+  const dir = await mkdtemp(join2(tmpdir(), "visual-ai-video-"));
+  try {
+    const ext = extensionFor(mimeType);
+    const path = join2(dir, `input${ext}`);
+    await writeFile2(path, data);
+    return {
+      path,
+      mimeType,
+      cleanup: async () => {
+        try {
+          await rm(dir, { recursive: true, force: true });
+        } catch {
+        }
+      }
+    };
+  } catch (err) {
+    try {
+      await rm(dir, { recursive: true, force: true });
+    } catch {
+    }
+    throw err;
+  }
+}
+function extensionFor(mimeType) {
+  switch (mimeType) {
+    case "video/mp4":
+      return ".mp4";
+    case "video/webm":
+      return ".webm";
+    case "video/quicktime":
+      return ".mov";
+    case "video/x-matroska":
+      return ".mkv";
+  }
+}
+var cachedFactoryPromise;
+async function loadFfmpegFactory() {
+  if (cachedFactoryPromise) return cachedFactoryPromise;
+  cachedFactoryPromise = (async () => {
+    let ffmpegModule;
+    try {
+      ffmpegModule = await import("fluent-ffmpeg");
+    } catch (err) {
+      const code = err?.code;
+      if (code === "ERR_MODULE_NOT_FOUND" || code === "MODULE_NOT_FOUND") {
+        throw new VisualAIVideoError(
+          "Could not load fluent-ffmpeg. It ships as a dependency of visual-ai-assertions, so this usually means the install was pruned or the platform-specific binary is unavailable. Reinstall the package or run: pnpm add fluent-ffmpeg @ffmpeg-installer/ffmpeg @ffprobe-installer/ffprobe"
+        );
+      }
+      throw new VisualAIVideoError(
+        `Failed to load fluent-ffmpeg: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+    const factory = ffmpegModule.default ?? ffmpegModule;
+    try {
+      const installer = await import("@ffmpeg-installer/ffmpeg");
+      const path = (installer.default ?? installer).path;
+      if (path) factory.setFfmpegPath(path);
+    } catch (err) {
+      const code = err?.code;
+      if (code !== "ERR_MODULE_NOT_FOUND" && code !== "MODULE_NOT_FOUND") {
+        process.stderr.write(
+          `[visual-ai-assertions] warning: @ffmpeg-installer/ffmpeg failed to load: ${err instanceof Error ? err.message : String(err)}
+`
+        );
+      }
+    }
+    try {
+      const installer = await import("@ffprobe-installer/ffprobe");
+      const path = (installer.default ?? installer).path;
+      if (path) factory.setFfprobePath(path);
+    } catch (err) {
+      const code = err?.code;
+      if (code !== "ERR_MODULE_NOT_FOUND" && code !== "MODULE_NOT_FOUND") {
+        process.stderr.write(
+          `[visual-ai-assertions] warning: @ffprobe-installer/ffprobe failed to load: ${err instanceof Error ? err.message : String(err)}
+`
+        );
+      }
+    }
+    return factory;
+  })();
+  try {
+    return await cachedFactoryPromise;
+  } catch (err) {
+    cachedFactoryPromise = void 0;
+    throw err;
+  }
+}
+async function probeDurationSeconds(videoPath) {
+  const ffmpeg = await loadFfmpegFactory();
+  return new Promise((resolve2, reject) => {
+    let settled = false;
+    const finish = (fn) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      fn();
+    };
+    const timer = setTimeout(() => {
+      finish(() => {
+        reject(
+          new VisualAIVideoError(
+            `ffprobe timed out after ${FFPROBE_TIMEOUT_MS}ms while probing ${videoPath}`
+          )
+        );
+      });
+    }, FFPROBE_TIMEOUT_MS);
+    ffmpeg.ffprobe(videoPath, (err, data) => {
+      if (err) {
+        finish(() => {
+          reject(
+            new VisualAIVideoError(
+              `Failed to probe video metadata: ${err.message}. Ensure ffprobe is installed (e.g. via @ffprobe-installer/ffprobe).`
+            )
+          );
+        });
+        return;
+      }
+      const raw = data.format?.duration;
+      const duration = typeof raw === "string" ? Number(raw) : raw;
+      if (!duration || !Number.isFinite(duration) || duration <= 0) {
+        finish(() => {
+          reject(new VisualAIVideoError("Video duration could not be determined"));
+        });
+        return;
+      }
+      finish(() => {
+        resolve2(duration);
+      });
+    });
+  });
+}
+async function extractFrames(videoPath, options = {}) {
+  const fps = options.fps ?? DEFAULT_FPS;
+  const maxFrames = options.maxFrames ?? DEFAULT_MAX_FRAMES;
+  const maxDurationSeconds = options.maxDurationSeconds ?? DEFAULT_MAX_DURATION_SECONDS;
+  if (!Number.isFinite(fps) || fps <= 0) {
+    throw new VisualAIVideoError(`Invalid fps: ${fps}. Must be a finite number > 0.`);
+  }
+  if (!Number.isFinite(maxFrames) || maxFrames <= 0) {
+    throw new VisualAIVideoError(`Invalid maxFrames: ${maxFrames}. Must be a finite number > 0.`);
+  }
+  if (maxFrames > MAX_FRAMES_HARD_CAP) {
+    throw new VisualAIVideoError(
+      `maxFrames ${maxFrames} exceeds the hard cap of ${MAX_FRAMES_HARD_CAP}. Lower maxFrames or open an issue if you need a larger limit.`
+    );
+  }
+  if (!Number.isFinite(maxDurationSeconds) || maxDurationSeconds <= 0) {
+    throw new VisualAIVideoError(
+      `Invalid maxDurationSeconds: ${maxDurationSeconds}. Must be a finite number > 0.`
+    );
+  }
+  const ffmpeg = await loadFfmpegFactory();
+  const durationSeconds = await probeDurationSeconds(videoPath);
+  if (durationSeconds > maxDurationSeconds) {
+    throw new VisualAIVideoError(
+      `Video duration ${durationSeconds.toFixed(2)}s exceeds limit of ${maxDurationSeconds}s. Pass { maxDurationSeconds: N } to override, or trim the source video.`
+    );
+  }
+  const outputDir = await mkdtemp(join2(tmpdir(), "visual-ai-frames-"));
+  try {
+    const filter = `fps=${fps},scale='if(gt(iw,ih),min(${FRAME_MAX_DIMENSION},iw),-2)':'if(gt(iw,ih),-2,min(${FRAME_MAX_DIMENSION},ih))':flags=area`;
+    await new Promise((resolve2, reject) => {
+      let settled = false;
+      const cmd = ffmpeg(videoPath);
+      const finish = (fn) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        fn();
+      };
+      const timer = setTimeout(() => {
+        try {
+          cmd.kill("SIGKILL");
+        } catch {
+        }
+        finish(() => {
+          reject(
+            new VisualAIVideoError(
+              `ffmpeg frame extraction timed out after ${FFMPEG_RUN_TIMEOUT_MS}ms`
+            )
+          );
+        });
+      }, FFMPEG_RUN_TIMEOUT_MS);
+      cmd.outputOptions(["-vf", filter, "-vframes", String(maxFrames), "-q:v", "3"]).output(join2(outputDir, "frame-%04d.jpg")).on("end", () => {
+        finish(() => {
+          resolve2();
+        });
+      }).on("error", (err) => {
+        finish(() => {
+          reject(new VisualAIVideoError(`ffmpeg frame extraction failed: ${err.message}`));
+        });
+      }).run();
+    });
+    const files = (await readdir(outputDir)).filter((name) => name.endsWith(".jpg")).sort();
+    if (files.length === 0) {
+      throw new VisualAIVideoError(
+        "No frames could be extracted from the video. The source may be corrupt or empty."
+      );
+    }
+    const frames = await Promise.all(
+      files.map(async (name, index) => {
+        const data = await readFile2(join2(outputDir, name));
+        const timestampSeconds = Math.min(durationSeconds, (index + 0.5) / fps);
+        let cachedBase64;
+        return {
+          data,
+          mimeType: "image/jpeg",
+          get base64() {
+            if (cachedBase64 === void 0) {
+              cachedBase64 = data.toString("base64");
+            }
+            return cachedBase64;
+          },
+          timestampSeconds,
+          index
+        };
+      })
+    );
+    return { frames, durationSeconds };
+  } finally {
+    try {
+      await rm(outputDir, { recursive: true, force: true });
+    } catch {
+    }
+  }
+}
+
+// src/core/media.ts
+var VIDEO_MAGIC_BYTE_PREFIX_LEN = 16;
+function isVideoInput(input) {
+  if (Buffer.isBuffer(input) || input instanceof Uint8Array) {
+    const buf = Buffer.isBuffer(input) ? input : Buffer.from(input);
+    return detectVideoMimeType(buf) !== null;
+  }
+  if (typeof input !== "string") return false;
+  if (isDataUrl(input)) {
+    const parsed = parseDataUrl(input);
+    return parsed?.mimeType.startsWith("video/") ?? false;
+  }
+  if (isFilePath(input)) {
+    return getVideoMimeFromExtension(input) !== void 0;
+  }
+  if (looksLikeVideoBase64(input)) {
+    try {
+      const buf = decodeBase64(input.slice(0, VIDEO_MAGIC_BYTE_PREFIX_LEN));
+      return detectVideoMimeType(buf) !== null;
+    } catch {
+      return false;
+    }
+  }
+  return false;
+}
+async function normalizeMedia(input, videoOptions) {
+  if (isVideoInput(input)) {
+    const { path, cleanup } = await resolveVideoToPath(input);
+    try {
+      const { frames, durationSeconds } = await extractFrames(path, videoOptions);
+      await saveDebugFrames(frames);
+      return { kind: "video", frames, durationSeconds };
+    } finally {
+      try {
+        await cleanup();
+      } catch {
+      }
+    }
+  }
+  const image = await normalizeImage(input);
+  return { kind: "image", image };
+}
+
 // src/types.ts
 import { z } from "zod";
 var IssuePrioritySchema = z.enum(["critical", "major", "minor"]);
@@ -1162,7 +1694,13 @@ var StatementResultSchema = z.object({
   statement: z.string(),
   pass: z.boolean(),
   reasoning: z.string(),
-  confidence: ConfidenceSchema.optional()
+  confidence: ConfidenceSchema.optional(),
+  /**
+   * For video inputs, the approximate timestamp (in seconds, from the start of the clip)
+   * of the frame that most clearly demonstrates the statement. `null` when the statement
+   * fails or applies across the whole clip. Always omitted for image inputs.
+   */
+  timestampSeconds: z.number().nonnegative().nullable().optional()
 });
 var UsageInfoSchema = z.object({
   inputTokens: z.number(),
@@ -1191,6 +1729,11 @@ var CompareResultSchema = BaseResultSchema.extend({
 var AskResultSchema = z.object({
   summary: z.string(),
   issues: z.array(IssueSchema),
+  /**
+   * For video inputs, the indices of frames the model relied on to answer.
+   * Indices are 0-based and refer to entries in `frames.timestampsSeconds`.
+   */
+  frameReferences: z.array(z.number().int().nonnegative()).optional(),
   usage: UsageInfoSchema.optional()
 });
 
@@ -1266,6 +1809,29 @@ function createDriver(provider, config) {
 var checkSchemaOptions = toSchemaOptions(CheckResponseSchema);
 var askSchemaOptions = toSchemaOptions(AskResponseSchema);
 var compareSchemaOptions = toSchemaOptions(CompareResponseSchema);
+function mediaToProviderInputs(media) {
+  if (media.kind === "image") {
+    return {
+      images: [media.image],
+      mediaContext: { kind: "image" },
+      framesMetadata: void 0
+    };
+  }
+  const timestamps = media.frames.map((f) => f.timestampSeconds);
+  return {
+    images: media.frames,
+    mediaContext: {
+      kind: "video",
+      frameTimestamps: timestamps,
+      durationSeconds: media.durationSeconds
+    },
+    framesMetadata: {
+      count: media.frames.length,
+      timestampsSeconds: timestamps,
+      durationSeconds: media.durationSeconds
+    }
+  };
+}
 function visualAI(config = {}) {
   const resolvedConfig = resolveConfig(config);
   const driverConfig = {
@@ -1294,34 +1860,44 @@ function visualAI(config = {}) {
     });
   }
   return {
-    async check(image, statements, options) {
+    async check(input, statements, options) {
       const stmts = Array.isArray(statements) ? statements : [statements];
       if (stmts.length === 0) {
         throw new VisualAIConfigError("At least one statement is required for check()");
       }
       return withErrorDebug(resolvedConfig, "check", async () => {
-        const img = await normalizeImage(image);
-        const prompt = buildCheckPrompt(stmts, { instructions: options?.instructions });
+        const media = await normalizeMedia(input, options?.video);
+        const { images, mediaContext, framesMetadata } = mediaToProviderInputs(media);
+        const prompt = buildCheckPrompt(stmts, {
+          instructions: options?.instructions,
+          media: mediaContext
+        });
         debugLog(resolvedConfig, "check prompt", prompt, "prompt");
-        const response = await timedSendMessage(driver, [img], prompt, checkSchemaOptions);
+        const response = await timedSendMessage(driver, images, prompt, checkSchemaOptions);
         debugLog(resolvedConfig, "check response", response.text, "response");
         const result = parseCheckResponse(response.text);
         return {
           ...result,
+          ...framesMetadata ? { frames: framesMetadata } : {},
           usage: processUsage("check", response.usage, response.durationSeconds, resolvedConfig)
         };
       });
     },
-    async ask(image, userPrompt, options) {
+    async ask(input, userPrompt, options) {
       return withErrorDebug(resolvedConfig, "ask", async () => {
-        const img = await normalizeImage(image);
-        const prompt = buildAskPrompt(userPrompt, { instructions: options?.instructions });
+        const media = await normalizeMedia(input, options?.video);
+        const { images, mediaContext, framesMetadata } = mediaToProviderInputs(media);
+        const prompt = buildAskPrompt(userPrompt, {
+          instructions: options?.instructions,
+          media: mediaContext
+        });
         debugLog(resolvedConfig, "ask prompt", prompt, "prompt");
-        const response = await timedSendMessage(driver, [img], prompt, askSchemaOptions);
+        const response = await timedSendMessage(driver, images, prompt, askSchemaOptions);
         debugLog(resolvedConfig, "ask response", response.text, "response");
         const result = parseAskResponse(response.text);
         return {
           ...result,
+          ...framesMetadata ? { frames: framesMetadata } : {},
           usage: processUsage("ask", response.usage, response.durationSeconds, resolvedConfig)
         };
       });
@@ -1509,6 +2085,7 @@ export {
   VisualAIRateLimitError,
   VisualAIResponseParseError,
   VisualAITruncationError,
+  VisualAIVideoError,
   assertVisualCompareResult,
   assertVisualResult,
   formatCheckResult,