visual-ai-assertions 0.23.0 → 0.25.0

package/dist/index.js

@@ -258,10 +258,15 @@ Example for a failing check:
   ]
 }
 ${JSON_INSTRUCTIONS}`;
-var CHECK_OUTPUT_SCHEMA_VIDEO = `IMPORTANT: Follow this evaluation order:
+function buildCheckOutputSchemaVideo(unit) {
+  const anyPoint = unit === "frame" ? "ANY frame of the timeline" : "ANY moment of the video";
+  const bestPoint = unit === "frame" ? "the timestamp of the frame that most clearly demonstrates it" : "the timestamp of the moment that most clearly demonstrates it";
+  const citing = unit === "frame" ? "citing frame timestamps" : "citing timestamps";
+  const exampleWhere = unit === "frame" ? "at the 3.5s frame" : "at 3.5s";
+  return `IMPORTANT: Follow this evaluation order:
 1. First, evaluate EACH statement independently across the entire timeline and populate the "statements" array
-2. A statement passes if it is true at ANY frame of the timeline, unless the wording explicitly says otherwise (e.g. "throughout", "at all times")
-3. For each statement that passes, set "timestampSeconds" to the timestamp of the frame that most clearly demonstrates it (or where it first becomes true). Use null when the statement fails or applies across the whole clip.
+2. A statement passes if it is true at ${anyPoint}, unless the wording explicitly says otherwise (e.g. "throughout", "at all times")
+3. For each statement that passes, set "timestampSeconds" to ${bestPoint} (or where it first becomes true). Use null when the statement fails or applies across the whole clip.
 4. Then, set "pass" to true ONLY if every statement passed (logical AND of all statement results)
 5. Write "reasoning" as a brief overall summary of the evaluation
 6. Include "issues" only for statements that failed
@@ -275,7 +280,7 @@ Respond with a JSON object matching this exact structure:
     {
       "statement": string,  // the original statement text
       "pass": boolean,      // whether this statement is true at any point in the timeline
-      "reasoning": string,  // explanation for this statement, citing frame timestamps where relevant
+      "reasoning": string,  // explanation for this statement, ${citing} where relevant
       "confidence": "high" | "medium" | "low",
       "timestampSeconds": number | null
         // seconds from the start of the clip where the statement is most clearly true,
@@ -293,10 +298,13 @@ Example for a passing video check:
   "reasoning": "The success toast appeared briefly around 3.5s.",
   "issues": [],
   "statements": [
-    { "statement": "A success toast with text 'Saved' appears", "pass": true, "reasoning": "A green toast labeled 'Saved' is visible in the bottom-right at the 3.5s frame", "confidence": "high", "timestampSeconds": 3.5 }
+    { "statement": "A success toast with text 'Saved' appears", "pass": true, "reasoning": "A green toast labeled 'Saved' is visible in the bottom-right ${exampleWhere}", "confidence": "high", "timestampSeconds": 3.5 }
   ]
 }
 ${JSON_INSTRUCTIONS}`;
+}
+var CHECK_OUTPUT_SCHEMA_VIDEO = buildCheckOutputSchemaVideo("frame");
+var CHECK_OUTPUT_SCHEMA_NATIVE_VIDEO = buildCheckOutputSchemaVideo("moment");
 var ASK_OUTPUT_SCHEMA_IMAGE = `Respond with a JSON object matching this exact structure:
 {
   "summary": string,        // high-level analysis summary
@@ -329,6 +337,17 @@ ${ISSUE_SCHEMA_INSTRUCTIONS}
 Prioritize issues by severity (critical / major / minor) as for image input.
 Cite frame indices in "frameReferences" so the user can locate the moments you describe.
 ${JSON_INSTRUCTIONS}`;
+var ASK_OUTPUT_SCHEMA_NATIVE_VIDEO = `Respond with a JSON object matching this exact structure:
+{
+  "summary": string,                // high-level summary of what happens across the video
+  "issues": [...],                  // list of issues/findings, can be empty
+  "timestampReferences": number[]   // seconds from the start of the clip of the moments the answer relies on (in order)
+}
+${ISSUE_SCHEMA_INSTRUCTIONS}
+
+Prioritize issues by severity (critical / major / minor) as for image input.
+Cite timestamps in "timestampReferences" so the user can locate the moments you describe.
+${JSON_INSTRUCTIONS}`;
 var COMPARE_OUTPUT_SCHEMA = `Respond with a JSON object matching this exact structure:
 {
   "pass": boolean,          // true if no critical or major changes found
@@ -350,15 +369,26 @@ var DEFAULT_CHECK_ROLE = "You are a visual QA assistant. Evaluate the provided i
 var DEFAULT_CHECK_ROLE_VIDEO = "You are a visual QA assistant. Evaluate the provided sequence of video frames precisely and objectively, treating them as a chronological timeline.";
 var DEFAULT_ASK_ROLE = "You are a visual QA assistant. Analyze the provided image based on the user's request.";
 var DEFAULT_ASK_ROLE_VIDEO = "You are a visual QA assistant. Analyze the provided sequence of video frames as a chronological timeline based on the user's request.";
-function buildVideoTimelineSection(frameTimestamps, durationSeconds) {
+var DEFAULT_CHECK_ROLE_NATIVE_VIDEO = "You are a visual QA assistant. Evaluate the provided video recording precisely and objectively, treating it as a chronological timeline.";
+var DEFAULT_ASK_ROLE_NATIVE_VIDEO = "You are a visual QA assistant. Analyze the provided video recording as a chronological timeline based on the user's request.";
+function buildNativeVideoSection(durationSeconds) {
+  return `Video recording:
+- Total duration: ${durationSeconds.toFixed(2)}s
+
+The attached file is the complete video recording. Treat it as a chronological timeline and refer to moments by timestamp (seconds from the start of the clip) where helpful.`;
+}
+function buildVideoTimelineSection(frameTimestamps, durationSeconds, droppedUnchanged = 0) {
   const formatted = frameTimestamps.map((t, i) => `  ${i}: ${t.toFixed(2)}s`).join("\n");
+  const attached = frameTimestamps.length;
+  const sampledLine = droppedUnchanged > 0 ? `- ${attached + droppedUnchanged} frames sampled (in chronological order); ${droppedUnchanged} ${droppedUnchanged === 1 ? "was" : "were"} dropped because ${droppedUnchanged === 1 ? "it" : "they"} did not visibly change from the preceding kept frame, so ${attached} ${attached === 1 ? "image is" : "images are"} attached` : `- ${attached} frames sampled (in chronological order)`;
+  const droppedGuidance = droppedUnchanged > 0 ? ` Frames sampled between two consecutive listed timestamps looked the same as the earlier listed frame, and frames sampled after the last listed timestamp looked the same as the last attached image until the clip ended at ${durationSeconds.toFixed(2)}s.` : "";
   return `Video timeline:
 - Total duration: ${durationSeconds.toFixed(2)}s
-- ${frameTimestamps.length} frames sampled (in chronological order)
+${sampledLine}
 - Frame index \u2192 timestamp:
 ${formatted}
 
-Treat the attached images as a chronological timeline. The first image is the earliest frame, the last is the latest. Refer to frames by timestamp where helpful.`;
+Treat the attached images as a chronological timeline. The first image is the earliest frame, the last is the latest. Refer to frames by timestamp where helpful.${droppedGuidance}`;
 }
 var COMPARE_ROLE = "You are performing a visual regression test. Compare the BEFORE image (baseline) to the AFTER image (current) and identify all visual differences. Flag changes that appear unintentional or problematic.";
 var COMPARE_EDGE_RULES = [
@@ -373,30 +403,52 @@ function buildCheckPrompt(statements, options) {
   const stmts = Array.isArray(statements) ? statements : [statements];
   const statementsBlock = stmts.map((s, i) => `${i + 1}. "${s}"`).join("\n");
   const media = options?.media;
-  const defaultRole = media?.kind === "video" ? DEFAULT_CHECK_ROLE_VIDEO : DEFAULT_CHECK_ROLE;
+  const defaultRole = media?.kind === "video" ? DEFAULT_CHECK_ROLE_VIDEO : media?.kind === "native-video" ? DEFAULT_CHECK_ROLE_NATIVE_VIDEO : DEFAULT_CHECK_ROLE;
   const sections = [options?.role ?? defaultRole];
   if (media?.kind === "video") {
-    sections.push(buildVideoTimelineSection(media.frameTimestamps, media.durationSeconds));
+    sections.push(
+      buildVideoTimelineSection(
+        media.frameTimestamps,
+        media.durationSeconds,
+        media.droppedUnchanged
+      )
+    );
+  } else if (media?.kind === "native-video") {
+    sections.push(buildNativeVideoSection(media.durationSeconds));
   }
   if (options?.instructions && options.instructions.length > 0) {
     sections.push(buildInstructionsSection(options.instructions));
   }
   sections.push(`Statements to evaluate:
 ${statementsBlock}`);
-  sections.push(media?.kind === "video" ? CHECK_OUTPUT_SCHEMA_VIDEO : CHECK_OUTPUT_SCHEMA_IMAGE);
+  sections.push(
+    media?.kind === "video" ? CHECK_OUTPUT_SCHEMA_VIDEO : media?.kind === "native-video" ? CHECK_OUTPUT_SCHEMA_NATIVE_VIDEO : CHECK_OUTPUT_SCHEMA_IMAGE
+  );
   return sections.join("\n\n");
 }
 function buildAskPrompt(userPrompt, options) {
   const media = options?.media;
-  const sections = [media?.kind === "video" ? DEFAULT_ASK_ROLE_VIDEO : DEFAULT_ASK_ROLE];
+  const sections = [
+    media?.kind === "video" ? DEFAULT_ASK_ROLE_VIDEO : media?.kind === "native-video" ? DEFAULT_ASK_ROLE_NATIVE_VIDEO : DEFAULT_ASK_ROLE
+  ];
   if (media?.kind === "video") {
-    sections.push(buildVideoTimelineSection(media.frameTimestamps, media.durationSeconds));
+    sections.push(
+      buildVideoTimelineSection(
+        media.frameTimestamps,
+        media.durationSeconds,
+        media.droppedUnchanged
+      )
+    );
+  } else if (media?.kind === "native-video") {
+    sections.push(buildNativeVideoSection(media.durationSeconds));
   }
   if (options?.instructions && options.instructions.length > 0) {
     sections.push(buildInstructionsSection(options.instructions));
   }
   sections.push(`User request: ${userPrompt}`);
-  sections.push(media?.kind === "video" ? ASK_OUTPUT_SCHEMA_VIDEO : ASK_OUTPUT_SCHEMA_IMAGE);
+  sections.push(
+    media?.kind === "video" ? ASK_OUTPUT_SCHEMA_VIDEO : media?.kind === "native-video" ? ASK_OUTPUT_SCHEMA_NATIVE_VIDEO : ASK_OUTPUT_SCHEMA_IMAGE
+  );
   return sections.join("\n\n");
 }
 function buildAiDiffPrompt() {
@@ -697,10 +749,16 @@ var AnthropicDriver = class {
 
 // src/providers/google.ts
 var DEFAULT_IMAGE_GEN_MODEL = "gemini-2.5-flash-image";
+var GEMINI_INLINE_VIDEO_LIMIT_BYTES = 19 * 1024 * 1024;
+var GEMINI_FILE_POLL_INTERVAL_MS = 2e3;
+var GEMINI_FILE_POLL_TIMEOUT_MS = 5 * 6e4;
 function needsCodeExecution(model) {
   const match = model.match(/^gemini-(\d+)/);
   return match !== null && match[1] !== void 0 && parseInt(match[1], 10) >= 3;
 }
+function sleep(ms) {
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
+}
 var GOOGLE_THINKING_LEVEL = {
   low: "low",
   medium: "medium",
@@ -768,24 +826,28 @@ var GoogleDriver = class {
     });
     return this.client;
   }
-  async sendMessage(images, prompt, _options) {
-    const client = await this.getClient();
+  /** Request config shared by image and video messages. */
+  generationConfig() {
+    return {
+      responseMimeType: "application/json",
+      maxOutputTokens: this.maxTokens,
+      ...this.reasoningEffort && {
+        thinkingConfig: {
+          thinkingLevel: GOOGLE_THINKING_LEVEL[this.reasoningEffort]
+        }
+      },
+      ...this.imageDetail && GOOGLE_MEDIA_RESOLUTION[this.imageDetail] && {
+        mediaResolution: GOOGLE_MEDIA_RESOLUTION[this.imageDetail]
+      }
+    };
+  }
+  /** Runs one generateContent call and normalizes finish reasons, text, and usage. */
+  async generate(client, contents) {
     try {
       const response = await client.models.generateContent({
         model: this.model,
-        contents: [...this.toGeminiParts(images), prompt],
-        config: {
-          responseMimeType: "application/json",
-          maxOutputTokens: this.maxTokens,
-          ...this.reasoningEffort && {
-            thinkingConfig: {
-              thinkingLevel: GOOGLE_THINKING_LEVEL[this.reasoningEffort]
-            }
-          },
-          ...this.imageDetail && GOOGLE_MEDIA_RESOLUTION[this.imageDetail] && {
-            mediaResolution: GOOGLE_MEDIA_RESOLUTION[this.imageDetail]
-          }
-        }
+        contents,
+        config: this.generationConfig()
       });
       const finishReason = response.candidates?.[0]?.finishReason;
       if (finishReason === "MAX_TOKENS") {
@@ -800,9 +862,8 @@ var GoogleDriver = class {
           `Response blocked: Google returned finishReason "${finishReason}".`
         );
       }
-      const text = response.text ?? "";
       return {
-        text,
+        text: response.text ?? "",
         usage: toGeminiUsage(response.usageMetadata)
       };
     } catch (err) {
@@ -810,6 +871,80 @@ var GoogleDriver = class {
       throw mapProviderError(err);
     }
   }
+  async sendMessage(images, prompt, _options) {
+    const client = await this.getClient();
+    return this.generate(client, [...this.toGeminiParts(images), prompt]);
+  }
+  /**
+   * Uploads a video through the Files API and waits until Gemini has finished
+   * processing it. Returns the ACTIVE file record.
+   */
+  async uploadVideo(client, video) {
+    try {
+      const uploaded = await client.files.upload({
+        file: new Blob([new Uint8Array(video.data)], { type: video.mimeType }),
+        config: { mimeType: video.mimeType }
+      });
+      const name = uploaded.name;
+      if (!name) {
+        throw new VisualAIProviderError("Gemini Files API returned a file without a name.");
+      }
+      const deadline = Date.now() + GEMINI_FILE_POLL_TIMEOUT_MS;
+      let current = uploaded;
+      while (current.state === "PROCESSING") {
+        if (Date.now() > deadline) {
+          throw new VisualAIProviderError(
+            `Gemini file ${name} was still processing after ${GEMINI_FILE_POLL_TIMEOUT_MS}ms.`
+          );
+        }
+        await sleep(GEMINI_FILE_POLL_INTERVAL_MS);
+        current = await client.files.get({ name });
+      }
+      if (current.state !== "ACTIVE") {
+        throw new VisualAIProviderError(
+          `Gemini file ${name} ended in state ${current.state ?? "unknown"}: ${current.error?.message ?? "no error message"}`
+        );
+      }
+      if (!current.uri) {
+        throw new VisualAIProviderError("Gemini Files API returned an ACTIVE file without a URI.");
+      }
+      return current;
+    } catch (err) {
+      if (err instanceof VisualAIProviderError) throw err;
+      throw mapProviderError(err);
+    }
+  }
+  /**
+   * Sends the video bytes themselves. Gemini samples the clip server-side at
+   * `video.fps` and, unlike sampled frames, also hears the audio track. Small
+   * videos go inline; larger ones are uploaded via the Files API and deleted
+   * again afterwards (they would expire on their own after 48 h).
+   */
+  async sendVideoMessage(video, prompt, _options) {
+    const client = await this.getClient();
+    const videoMetadata = { fps: video.fps };
+    if (video.data.byteLength <= GEMINI_INLINE_VIDEO_LIMIT_BYTES) {
+      const part = {
+        inlineData: { data: video.data.toString("base64"), mimeType: video.mimeType },
+        videoMetadata
+      };
+      const response = await this.generate(client, [part, prompt]);
+      return { ...response, delivery: "inline" };
+    }
+    const file = await this.uploadVideo(client, video);
+    try {
+      const part = {
+        fileData: { fileUri: file.uri, mimeType: file.mimeType ?? video.mimeType },
+        videoMetadata
+      };
+      const response = await this.generate(client, [part, prompt]);
+      return { ...response, delivery: "file" };
+    } finally {
+      if (file.name) {
+        await client.files.delete({ name: file.name }).catch(() => void 0);
+      }
+    }
+  }
   async generateImage(images, prompt, options) {
     const client = await this.getClient();
     const imageModel = options?.model ?? DEFAULT_IMAGE_GEN_MODEL;
@@ -1402,6 +1537,15 @@ async function timedSendMessage(driver, images, prompt, options) {
   const durationSeconds = (performance.now() - start) / 1e3;
   return { ...response, durationSeconds };
 }
+async function timedSendVideoMessage(driver, video, prompt, options) {
+  if (!driver.sendVideoMessage) {
+    throw new VisualAIError("Provider driver does not support native video delivery");
+  }
+  const start = performance.now();
+  const response = await driver.sendVideoMessage(video, prompt, options);
+  const durationSeconds = (performance.now() - start) / 1e3;
+  return { ...response, durationSeconds };
+}
 
 // src/core/diff.ts
 import sharp from "sharp";
@@ -1641,6 +1785,9 @@ async function normalizeImage(input, maxDimension = DEFAULT_MAX_IMAGE_DIMENSION)
   };
 }
 
+// src/core/media.ts
+import { readFile as readFile3 } from "fs/promises";
+
 // src/core/debug-frames.ts
 import { randomBytes } from "crypto";
 import { mkdir, writeFile } from "fs/promises";
@@ -1696,6 +1843,92 @@ async function saveDebugFrames(frames, env = process.env) {
   return runDir;
 }
 
+// src/core/frame-dedupe.ts
+import sharp3 from "sharp";
+var DEDUPE_THUMBNAIL_EDGE = 256;
+var DEDUPE_PIXEL_TOLERANCE = 24;
+var DEFAULT_DEDUPE_THRESHOLD = 1e-3;
+function resolveDedupeOptions(raw) {
+  if (raw === void 0 || raw === true) {
+    return { enabled: true, threshold: DEFAULT_DEDUPE_THRESHOLD };
+  }
+  if (raw === false) {
+    return { enabled: false, threshold: DEFAULT_DEDUPE_THRESHOLD };
+  }
+  const threshold = raw.threshold ?? DEFAULT_DEDUPE_THRESHOLD;
+  if (!Number.isFinite(threshold) || threshold <= 0 || threshold > 1) {
+    throw new VisualAIVideoError(
+      `Invalid dedupe threshold: ${String(threshold)}. Must be a finite number in (0, 1].`
+    );
+  }
+  return { enabled: true, threshold };
+}
+async function frameSignature(frame) {
+  try {
+    const { data, info } = await sharp3(frame.data).flatten({ background: { r: 255, g: 255, b: 255 } }).greyscale().resize(DEDUPE_THUMBNAIL_EDGE, DEDUPE_THUMBNAIL_EDGE, {
+      fit: "inside",
+      withoutEnlargement: true
+    }).raw().toBuffer({ resolveWithObject: true });
+    return { width: info.width, height: info.height, pixels: data };
+  } catch (err) {
+    const reason = err instanceof Error ? err.message : String(err);
+    throw new VisualAIVideoError(
+      `Failed to decode frame ${frame.index} (${frame.timestampSeconds.toFixed(2)}s) for change detection: ${reason}`
+    );
+  }
+}
+function changedFraction(a, b) {
+  if (a.width !== b.width || a.height !== b.height) {
+    return 1;
+  }
+  let changed = 0;
+  for (let i = 0; i < a.pixels.length; i++) {
+    if (Math.abs((a.pixels[i] ?? 0) - (b.pixels[i] ?? 0)) > DEDUPE_PIXEL_TOLERANCE) {
+      changed++;
+    }
+  }
+  return changed / a.pixels.length;
+}
+function reindex(frame, index) {
+  if (frame.index === index) {
+    return frame;
+  }
+  return {
+    data: frame.data,
+    mimeType: frame.mimeType,
+    get base64() {
+      return frame.base64;
+    },
+    timestampSeconds: frame.timestampSeconds,
+    index
+  };
+}
+async function dedupeFrames(frames, options) {
+  const { enabled, threshold } = resolveDedupeOptions(options);
+  if (!enabled || frames.length < 2) {
+    return { frames: [...frames], dropped: 0 };
+  }
+  const signed = await Promise.all(
+    frames.map(async (frame) => ({ frame, signature: await frameSignature(frame) }))
+  );
+  const [first, ...rest] = signed;
+  if (first === void 0) {
+    return { frames: [], dropped: 0 };
+  }
+  const kept = [first.frame];
+  let lastKept = first.signature;
+  for (const { frame, signature } of rest) {
+    if (changedFraction(lastKept, signature) >= threshold) {
+      kept.push(frame);
+      lastKept = signature;
+    }
+  }
+  return {
+    frames: kept.map((frame, index) => reindex(frame, index)),
+    dropped: frames.length - kept.length
+  };
+}
+
 // src/core/video.ts
 import { mkdtemp, readFile as readFile2, readdir, rm, writeFile as writeFile2 } from "fs/promises";
 import { tmpdir } from "os";
@@ -1931,7 +2164,7 @@ async function probeDurationSeconds(videoPath) {
     });
   });
 }
-async function extractFrames(videoPath, options = {}, maxDimension = FRAME_MAX_DIMENSION) {
+function resolveVideoSamplingOptions(options = {}) {
   const fps = options.fps ?? DEFAULT_FPS;
   const maxFrames = options.maxFrames ?? DEFAULT_MAX_FRAMES;
   const maxDurationSeconds = options.maxDurationSeconds ?? DEFAULT_MAX_DURATION_SECONDS;
@@ -1951,13 +2184,20 @@ async function extractFrames(videoPath, options = {}, maxDimension = FRAME_MAX_D
       `Invalid maxDurationSeconds: ${maxDurationSeconds}. Must be a finite number > 0.`
     );
   }
-  const ffmpeg = await loadFfmpegFactory();
-  const durationSeconds = await probeDurationSeconds(videoPath);
+  return { fps, maxFrames, maxDurationSeconds };
+}
+function assertDurationWithinLimit(durationSeconds, maxDurationSeconds) {
   if (durationSeconds > maxDurationSeconds) {
     throw new VisualAIVideoError(
       `Video duration ${durationSeconds.toFixed(2)}s exceeds limit of ${maxDurationSeconds}s. Pass { maxDurationSeconds: N } to override, or trim the source video.`
     );
   }
+}
+async function extractFrames(videoPath, options = {}, maxDimension = FRAME_MAX_DIMENSION) {
+  const { fps, maxFrames, maxDurationSeconds } = resolveVideoSamplingOptions(options);
+  const ffmpeg = await loadFfmpegFactory();
+  const durationSeconds = await probeDurationSeconds(videoPath);
+  assertDurationWithinLimit(durationSeconds, maxDurationSeconds);
   const outputDir = await mkdtemp(join2(tmpdir(), "visual-ai-frames-"));
   try {
     const filter = `fps=${fps},scale='if(gt(iw,ih),min(${maxDimension},iw),-2)':'if(gt(iw,ih),-2,min(${maxDimension},ih))':flags=area`;
@@ -2061,6 +2301,7 @@ function isTimestampedFrameInput(frame) {
 async function normalizeFrames(input, maxDimension = DEFAULT_MAX_IMAGE_DIMENSION) {
   const rawFrames = input.frames;
   const fps = input.fps ?? DEFAULT_FPS;
+  resolveDedupeOptions(input.dedupe);
   if (rawFrames.length === 0) {
     throw new VisualAIVideoError("frames must be a non-empty array of image inputs");
   }
@@ -2072,7 +2313,7 @@ async function normalizeFrames(input, maxDimension = DEFAULT_MAX_IMAGE_DIMENSION
   if (!Number.isFinite(fps) || fps <= 0) {
     throw new VisualAIVideoError(`Invalid fps: ${fps}. Must be a finite number > 0.`);
   }
-  const frames = await Promise.all(
+  const sampled = await Promise.all(
     rawFrames.map(async (raw, index) => {
       const timestamped = isTimestampedFrameInput(raw);
       const imageInput = timestamped ? raw.image : raw;
@@ -2095,20 +2336,45 @@ async function normalizeFrames(input, maxDimension = DEFAULT_MAX_IMAGE_DIMENSION
       };
     })
   );
-  const durationSeconds = frames.reduce((max, f) => Math.max(max, f.timestampSeconds), 0);
+  const durationSeconds = sampled.reduce((max, f) => Math.max(max, f.timestampSeconds), 0);
+  const { frames, dropped } = await dedupeFrames(sampled, input.dedupe);
   await saveDebugFrames(frames);
-  return { kind: "video", frames, durationSeconds };
+  return { kind: "video", frames, durationSeconds, droppedUnchanged: dropped };
+}
+async function normalizeNativeVideo(input, videoOptions) {
+  const { fps, maxDurationSeconds } = resolveVideoSamplingOptions(videoOptions);
+  const { path, mimeType, cleanup } = await resolveVideoToPath(input);
+  try {
+    const durationSeconds = await probeDurationSeconds(path);
+    assertDurationWithinLimit(durationSeconds, maxDurationSeconds);
+    const data = await readFile3(path);
+    return { kind: "native-video", video: { data, mimeType, durationSeconds, fps } };
+  } finally {
+    try {
+      await cleanup();
+    } catch {
+    }
+  }
 }
-async function normalizeMedia(input, videoOptions, maxDimension = DEFAULT_MAX_IMAGE_DIMENSION) {
+async function normalizeMedia(input, videoOptions, maxDimension = DEFAULT_MAX_IMAGE_DIMENSION, nativeVideo = false) {
   if (isFramesInput(input)) {
     return normalizeFrames(input, maxDimension);
   }
   if (isVideoInput(input)) {
+    if (nativeVideo) {
+      return normalizeNativeVideo(input, videoOptions);
+    }
+    resolveDedupeOptions(videoOptions?.dedupe);
     const { path, cleanup } = await resolveVideoToPath(input);
     try {
-      const { frames, durationSeconds } = await extractFrames(path, videoOptions, maxDimension);
+      const { frames: sampled, durationSeconds } = await extractFrames(
+        path,
+        videoOptions,
+        maxDimension
+      );
+      const { frames, dropped } = await dedupeFrames(sampled, videoOptions?.dedupe);
       await saveDebugFrames(frames);
-      return { kind: "video", frames, durationSeconds };
+      return { kind: "video", frames, durationSeconds, droppedUnchanged: dropped };
     } finally {
       try {
         await cleanup();
@@ -2198,6 +2464,12 @@ var AskResultSchema = z.object({
    * omitting the key, even for image inputs that were never asked to populate it.
    */
   frameReferences: z.array(z.number().int().nonnegative()).nullable().optional(),
+  /**
+   * For natively delivered video, the timestamps (seconds from the start of
+   * the clip) the model relied on to answer. The native counterpart of
+   * `frameReferences`. Nullable for the same strict-schema reason.
+   */
+  timestampReferences: z.array(z.number().nonnegative()).nullable().optional(),
   usage: UsageInfoSchema.optional()
 });
 
@@ -2276,7 +2548,8 @@ function parseAskResponse(raw) {
   const result = parseResponse(raw, AskResponseSchema);
   return {
     ...result,
-    frameReferences: result.frameReferences ?? void 0
+    frameReferences: result.frameReferences ?? void 0,
+    timestampReferences: result.timestampReferences ?? void 0
   };
 }
 function parseCompareResponse(raw) {
@@ -2305,26 +2578,66 @@ var compareSchemaOptions = toSchemaOptions(CompareResponseSchema);
 function mediaToProviderInputs(media) {
   if (media.kind === "image") {
     return {
+      kind: "images",
       images: [media.image],
       mediaContext: { kind: "image" },
-      framesMetadata: void 0
+      frames: void 0
+    };
+  }
+  if (media.kind === "native-video") {
+    return {
+      kind: "native-video",
+      video: media.video,
+      mediaContext: { kind: "native-video", durationSeconds: media.video.durationSeconds }
     };
   }
   const timestamps = media.frames.map((f) => f.timestampSeconds);
   return {
+    kind: "images",
     images: media.frames,
     mediaContext: {
       kind: "video",
       frameTimestamps: timestamps,
-      durationSeconds: media.durationSeconds
+      durationSeconds: media.durationSeconds,
+      droppedUnchanged: media.droppedUnchanged
     },
-    framesMetadata: {
+    frames: {
       count: media.frames.length,
       timestampsSeconds: timestamps,
-      durationSeconds: media.durationSeconds
+      durationSeconds: media.durationSeconds,
+      droppedUnchanged: media.droppedUnchanged
     }
   };
 }
+async function sendMedia(driver, dispatch, prompt, options) {
+  if (dispatch.kind === "native-video") {
+    const response2 = await timedSendVideoMessage(driver, dispatch.video, prompt, options);
+    const { durationSeconds, fps, mimeType } = dispatch.video;
+    return {
+      response: response2,
+      metadata: { video: { durationSeconds, fps, mimeType, delivery: response2.delivery } }
+    };
+  }
+  const response = await timedSendMessage(driver, dispatch.images, prompt, options);
+  return { response, metadata: dispatch.frames ? { frames: dispatch.frames } : {} };
+}
+function resolveNativeVideo(input, videoOptions, driver, provider) {
+  const mode = videoOptions?.mode ?? "auto";
+  if (mode !== "auto" && mode !== "native" && mode !== "frames") {
+    throw new VisualAIConfigError(
+      `Invalid video mode: ${mode}. Expected "auto", "native", or "frames".`
+    );
+  }
+  const supported = typeof driver.sendVideoMessage === "function";
+  if (mode === "frames") return false;
+  if (mode === "auto") return supported;
+  if (!supported && !isFramesInput(input) && isVideoInput(input)) {
+    throw new VisualAIConfigError(
+      `Native video delivery is not supported by the "${provider}" provider. Use a Google model, or set video: { mode: "frames" } to sample frames instead.`
+    );
+  }
+  return true;
+}
 function visualAI(config = {}) {
   const resolvedConfig = resolveConfig(config);
   const driverConfig = {
@@ -2362,38 +2675,55 @@ function visualAI(config = {}) {
         throw new VisualAIConfigError("At least one statement is required for check()");
       }
       return withErrorDebug(resolvedConfig, "check", async () => {
-        const media = await normalizeMedia(input, options?.video, maxImageDimension);
-        const { images, mediaContext, framesMetadata } = mediaToProviderInputs(media);
+        const nativeVideo = resolveNativeVideo(
+          input,
+          options?.video,
+          driver,
+          resolvedConfig.provider
+        );
+        const media = await normalizeMedia(input, options?.video, maxImageDimension, nativeVideo);
+        const dispatch = mediaToProviderInputs(media);
         const prompt = buildCheckPrompt(stmts, {
           instructions: options?.instructions,
-          media: mediaContext
+          media: dispatch.mediaContext
         });
         debugLog(resolvedConfig, "check prompt", prompt, "prompt");
-        const response = await timedSendMessage(driver, images, prompt, checkSchemaOptions);
+        const { response, metadata } = await sendMedia(
+          driver,
+          dispatch,
+          prompt,
+          checkSchemaOptions
+        );
         debugLog(resolvedConfig, "check response", response.text, "response");
         const result = parseCheckResponse(response.text);
         return {
           ...result,
-          ...framesMetadata ? { frames: framesMetadata } : {},
+          ...metadata,
           usage: processUsage("check", response.usage, response.durationSeconds, resolvedConfig)
         };
       });
     },
     async ask(input, userPrompt, options) {
       return withErrorDebug(resolvedConfig, "ask", async () => {
-        const media = await normalizeMedia(input, options?.video, maxImageDimension);
-        const { images, mediaContext, framesMetadata } = mediaToProviderInputs(media);
+        const nativeVideo = resolveNativeVideo(
+          input,
+          options?.video,
+          driver,
+          resolvedConfig.provider
+        );
+        const media = await normalizeMedia(input, options?.video, maxImageDimension, nativeVideo);
+        const dispatch = mediaToProviderInputs(media);
         const prompt = buildAskPrompt(userPrompt, {
           instructions: options?.instructions,
-          media: mediaContext
+          media: dispatch.mediaContext
         });
         debugLog(resolvedConfig, "ask prompt", prompt, "prompt");
-        const response = await timedSendMessage(driver, images, prompt, askSchemaOptions);
+        const { response, metadata } = await sendMedia(driver, dispatch, prompt, askSchemaOptions);
         debugLog(resolvedConfig, "ask response", response.text, "response");
         const result = parseAskResponse(response.text);
         return {
           ...result,
-          ...framesMetadata ? { frames: framesMetadata } : {},
+          ...metadata,
           usage: processUsage("ask", response.usage, response.durationSeconds, resolvedConfig)
         };
       });