visual-ai-assertions 0.13.0 → 0.14.0

package/README.md

@@ -346,6 +346,31 @@ How it works: the library samples frames with ffmpeg and sends them to the provi
 
 **ffmpeg setup.** Video support works out of the box — `fluent-ffmpeg`, `@ffmpeg-installer/ffmpeg`, and `@ffprobe-installer/ffprobe` ship as regular dependencies and bundle platform-specific ffmpeg/ffprobe binaries. If you ran `npm install` you already have everything you need. On platforms where the prebuilt binary is unavailable (or if you've pruned dependencies), `check()` and `ask()` throw `VisualAIVideoError` (import from `visual-ai-assertions` to `instanceof`-narrow it) when called with video input.
 
+### Pre-sampled Frames
+
+When you already have an array of screenshots — or can't pass a video file — hand `check()` and `ask()` a `FramesInput` (`{ frames, fps? }`) instead of a `MediaInput`. Each frame is a plain `ImageInput` (buffer, data URL, base64, file path, or URL) or a `{ image, timestampSeconds }` pair. Frames are treated as the same chronological timeline a sampled video produces — no ffmpeg is loaded on this path.
+
+```typescript
+const frames = [await page.screenshot(), await page.screenshot()];
+
+const result = await ai.check({ frames }, ['A success toast with text "Saved" appears']);
+console.log(result.frames);
+// { count: 2, timestampsSeconds: [0, 1], durationSeconds: 1 }
+
+// Control timestamps: give an explicit fps, or per-frame timestampSeconds
+await ai.ask(
+  {
+    frames: [
+      { image: before, timestampSeconds: 0 },
+      { image: after, timestampSeconds: 2.5 },
+    ],
+  },
+  "What changed between the frames?",
+);
+```
+
+Timestamps for bare frames are derived as `index / fps` (default `fps` is `1`); a per-frame `timestampSeconds` overrides that. The frame count is subject to the same 60-frame hard cap as video sampling, and the `video` sampling option is ignored for this path.
+
 ### Formatting & Assertion Helpers
 
 ```typescript

package/dist/index.cjs

@@ -1773,7 +1773,57 @@ function isVideoInput(input) {
   }
   return false;
 }
+function isFramesInput(input) {
+  return typeof input === "object" && input !== null && !Buffer.isBuffer(input) && !(input instanceof Uint8Array) && Array.isArray(input.frames);
+}
+function isTimestampedFrameInput(frame) {
+  return typeof frame === "object" && !Buffer.isBuffer(frame) && !(frame instanceof Uint8Array) && "image" in frame;
+}
+async function normalizeFrames(input) {
+  const rawFrames = input.frames;
+  const fps = input.fps ?? DEFAULT_FPS;
+  if (rawFrames.length === 0) {
+    throw new VisualAIVideoError("frames must be a non-empty array of image inputs");
+  }
+  if (rawFrames.length > MAX_FRAMES_HARD_CAP) {
+    throw new VisualAIVideoError(
+      `frames length ${rawFrames.length} exceeds the hard cap of ${MAX_FRAMES_HARD_CAP}. Pass fewer frames or open an issue if you need a larger limit.`
+    );
+  }
+  if (!Number.isFinite(fps) || fps <= 0) {
+    throw new VisualAIVideoError(`Invalid fps: ${fps}. Must be a finite number > 0.`);
+  }
+  const frames = await Promise.all(
+    rawFrames.map(async (raw, index) => {
+      const timestamped = isTimestampedFrameInput(raw);
+      const imageInput = timestamped ? raw.image : raw;
+      const explicit = timestamped ? raw.timestampSeconds : void 0;
+      const timestampSeconds = explicit ?? index / fps;
+      if (!Number.isFinite(timestampSeconds) || timestampSeconds < 0) {
+        throw new VisualAIVideoError(
+          `Invalid timestampSeconds for frame ${index}: ${String(timestampSeconds)}. Must be a finite number >= 0.`
+        );
+      }
+      const image = await normalizeImage(imageInput);
+      return {
+        data: image.data,
+        mimeType: image.mimeType,
+        get base64() {
+          return image.base64;
+        },
+        timestampSeconds,
+        index
+      };
+    })
+  );
+  const durationSeconds = frames.reduce((max, f) => Math.max(max, f.timestampSeconds), 0);
+  await saveDebugFrames(frames);
+  return { kind: "video", frames, durationSeconds };
+}
 async function normalizeMedia(input, videoOptions) {
+  if (isFramesInput(input)) {
+    return normalizeFrames(input);
+  }
   if (isVideoInput(input)) {
     const { path, cleanup } = await resolveVideoToPath(input);
     try {
@@ -1853,8 +1903,12 @@ var AskResultSchema = import_zod.z.object({
   /**
    * For video inputs, the indices of frames the model relied on to answer.
    * Indices are 0-based and refer to entries in `frames.timestampsSeconds`.
+   *
+   * Nullable because providers with strict structured-output schemas (e.g. OpenAI)
+   * must mark every field required and represent "no value" as `null` rather than
+   * omitting the key, even for image inputs that were never asked to populate it.
    */
-  frameReferences: import_zod.z.array(import_zod.z.number().int().nonnegative()).optional(),
+  frameReferences: import_zod.z.array(import_zod.z.number().int().nonnegative()).nullable().optional(),
   usage: UsageInfoSchema.optional()
 });
 
@@ -1906,7 +1960,11 @@ function parseCheckResponse(raw) {
   return reconcileCheckResult(result);
 }
 function parseAskResponse(raw) {
-  return parseResponse(raw, AskResponseSchema);
+  const result = parseResponse(raw, AskResponseSchema);
+  return {
+    ...result,
+    frameReferences: result.frameReferences ?? void 0
+  };
 }
 function parseCompareResponse(raw) {
   return parseResponse(raw, CompareResponseSchema);