visual-ai-assertions 0.13.0 → 0.14.0

package/dist/index.d.cts

@@ -420,8 +420,12 @@ declare const AskResultSchema: z.ZodObject<{
     /**
      * For video inputs, the indices of frames the model relied on to answer.
      * Indices are 0-based and refer to entries in `frames.timestampsSeconds`.
+     *
+     * Nullable because providers with strict structured-output schemas (e.g. OpenAI)
+     * must mark every field required and represent "no value" as `null` rather than
+     * omitting the key, even for image inputs that were never asked to populate it.
      */
-    frameReferences: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
+    frameReferences: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodNumber, "many">>>;
     usage: z.ZodOptional<z.ZodObject<{
         inputTokens: z.ZodNumber;
         outputTokens: z.ZodNumber;
@@ -457,7 +461,7 @@ declare const AskResultSchema: z.ZodObject<{
         estimatedCost?: number | undefined;
         durationSeconds?: number | undefined;
     } | undefined;
-    frameReferences?: number[] | undefined;
+    frameReferences?: number[] | null | undefined;
 }, {
     issues: {
         priority: "critical" | "major" | "minor";
@@ -473,11 +477,12 @@ declare const AskResultSchema: z.ZodObject<{
         estimatedCost?: number | undefined;
         durationSeconds?: number | undefined;
     } | undefined;
-    frameReferences?: number[] | undefined;
+    frameReferences?: number[] | null | undefined;
 }>;
 /** Result returned by `ask()`. */
-type AskResult = z.infer<typeof AskResultSchema> & {
+type AskResult = Omit<z.infer<typeof AskResultSchema>, "frameReferences"> & {
     /** Present only when the input was a video. Describes which frames the model saw. */
+    frameReferences?: number[];
     frames?: VideoFramesMetadata;
 };
 /** Supported input shapes for image arguments accepted by the client. */
@@ -488,6 +493,36 @@ type ImageInput = Buffer | Uint8Array | string;
  * an image or a video.
  */
 type MediaInput = ImageInput;
+/**
+ * A single pre-sampled frame with an explicit timestamp. Use this shape when
+ * you want to control the timestamp the model sees; otherwise pass a bare
+ * `ImageInput` and the timestamp is derived from `fps` and the frame's position.
+ */
+interface TimestampedFrameInput {
+    /** Image source for the frame (buffer, data URL, base64, file path, or URL). */
+    image: ImageInput;
+    /** Timestamp in seconds, from the start of the sequence. Derived from `fps` when omitted. */
+    timestampSeconds?: number;
+}
+/**
+ * Pre-sampled frames supplied directly instead of a video file. Useful when you
+ * already have an array of screenshots and cannot (or would rather not) pass a
+ * video — this path never loads ffmpeg. Passed to `check()` / `ask()` in place
+ * of a `MediaInput`; downstream handling is identical to a sampled video.
+ */
+interface FramesInput {
+    /**
+     * Ordered frames, earliest first. Each entry is an image source or a
+     * `{ image, timestampSeconds }` pair. Must be non-empty and is capped at the
+     * same per-request frame limit as video sampling (60).
+     */
+    frames: ReadonlyArray<ImageInput | TimestampedFrameInput>;
+    /**
+     * Frame rate used to derive timestamps for frames that don't carry their own
+     * `timestampSeconds` (frame `i` maps to `i / fps` seconds). Default `1`.
+     */
+    fps?: number;
+}
 /** Supported image MIME types accepted by all providers. */
 type SupportedMimeType = "image/jpeg" | "image/png" | "image/webp" | "image/gif";
 /** Supported video MIME types the client can accept and sample frames from. */
@@ -628,9 +663,11 @@ interface VisualAIClient {
      * frames automatically; statements pass if they are true at any sampled
      * frame, and each statement result includes the timestamp where it
      * matched. The `frames` metadata on the result reports which timestamps
-     * the model saw.
+     * the model saw. Pass a `FramesInput` (`{ frames, fps? }`) to supply
+     * pre-sampled frames directly — handled identically to a video timeline but
+     * without loading ffmpeg.
      *
-     * @param input Image or video source as a buffer, URL, file path, or base64 string.
+     * @param input Image or video source as a buffer, URL, file path, or base64 string, or a `FramesInput` of pre-sampled frames.
      * @param statements One or more statements to validate against the input.
      * @param options Optional additional instructions and video sampling overrides.
      * @returns A structured result describing pass/fail, issues, and statement reasoning.
@@ -653,15 +690,17 @@ interface VisualAIClient {
      * console.log(result.statements[0].timestampSeconds); // e.g. 3.5
      * ```
      */
-    check(input: MediaInput, statements: string | string[], options?: CheckOptions): Promise<CheckResult>;
+    check(input: MediaInput | FramesInput, statements: string | string[], options?: CheckOptions): Promise<CheckResult>;
     /**
      * Asks an open-ended question about an image or video and returns a structured summary.
      *
      * Video inputs are sampled into frames and analyzed as a chronological
      * timeline. The result's `frameReferences` array surfaces which frames the
-     * model relied on for its answer.
+     * model relied on for its answer. Pass a `FramesInput` (`{ frames, fps? }`)
+     * to supply pre-sampled frames directly — handled identically to a video
+     * timeline but without loading ffmpeg.
      *
-     * @param input Image or video source as a buffer, URL, file path, or base64 string.
+     * @param input Image or video source as a buffer, URL, file path, or base64 string, or a `FramesInput` of pre-sampled frames.
      * @param prompt Prompt describing what to inspect in the input.
      * @param options Optional additional instructions and video sampling overrides.
      * @returns A summary with any detected issues.
@@ -673,7 +712,7 @@ interface VisualAIClient {
      * const result = await client.ask(screenshot, "What looks visually broken on this page?");
      * ```
      */
-    ask(input: MediaInput, prompt: string, options?: AskOptions): Promise<AskResult>;
+    ask(input: MediaInput | FramesInput, prompt: string, options?: AskOptions): Promise<AskResult>;
     /**
      * Compares two images and reports meaningful visual differences.
      *
@@ -1050,4 +1089,4 @@ declare function assertVisualResult(result: CheckResult, label?: string): void;
  */
 declare function assertVisualCompareResult(result: CompareResult, label?: string): void;
 
-export { Accessibility, type AccessibilityCheckName, type AccessibilityOptions, type AskOptions, type AskResult, AskResultSchema, type ChangeEntry, ChangeEntrySchema, type CheckOptions, type CheckResult, CheckResultSchema, type CompareOptions, type CompareResult, CompareResultSchema, type Confidence, ConfidenceSchema, Content, type ContentCheckName, type ContentOptions, DEFAULT_MODELS, type DiffImageResult, type ElementsVisibilityOptions, type Frame, type ImageInput, type Issue, type IssueCategory, IssueCategorySchema, type IssuePriority, IssuePrioritySchema, IssueSchema, type KnownModelName, Layout, type LayoutCheckName, type LayoutOptions, type MediaInput, Model, type PageLoadOptions, Provider, type ProviderName, ReasoningEffort, type ReasoningEffortLevel, type StatementResult, StatementResultSchema, type SupportedMimeType, type SupportedVideoMimeType, type UsageInfo, UsageInfoSchema, type VideoFramesMetadata, type VideoSamplingOptions, VisualAIAssertionError, VisualAIAuthError, type VisualAIClient, type VisualAIConfig, VisualAIConfigError, VisualAIError, type VisualAIErrorCode, VisualAIImageError, type VisualAIKnownError, VisualAIProviderError, VisualAIRateLimitError, VisualAIResponseParseError, VisualAITruncationError, VisualAIVideoError, assertVisualCompareResult, assertVisualResult, formatCheckResult, formatCompareResult, isVisualAIKnownError, visualAI };
+export { Accessibility, type AccessibilityCheckName, type AccessibilityOptions, type AskOptions, type AskResult, AskResultSchema, type ChangeEntry, ChangeEntrySchema, type CheckOptions, type CheckResult, CheckResultSchema, type CompareOptions, type CompareResult, CompareResultSchema, type Confidence, ConfidenceSchema, Content, type ContentCheckName, type ContentOptions, DEFAULT_MODELS, type DiffImageResult, type ElementsVisibilityOptions, type Frame, type FramesInput, type ImageInput, type Issue, type IssueCategory, IssueCategorySchema, type IssuePriority, IssuePrioritySchema, IssueSchema, type KnownModelName, Layout, type LayoutCheckName, type LayoutOptions, type MediaInput, Model, type PageLoadOptions, Provider, type ProviderName, ReasoningEffort, type ReasoningEffortLevel, type StatementResult, StatementResultSchema, type SupportedMimeType, type SupportedVideoMimeType, type TimestampedFrameInput, type UsageInfo, UsageInfoSchema, type VideoFramesMetadata, type VideoSamplingOptions, VisualAIAssertionError, VisualAIAuthError, type VisualAIClient, type VisualAIConfig, VisualAIConfigError, VisualAIError, type VisualAIErrorCode, VisualAIImageError, type VisualAIKnownError, VisualAIProviderError, VisualAIRateLimitError, VisualAIResponseParseError, VisualAITruncationError, VisualAIVideoError, assertVisualCompareResult, assertVisualResult, formatCheckResult, formatCompareResult, isVisualAIKnownError, visualAI };

package/dist/index.d.ts

@@ -420,8 +420,12 @@ declare const AskResultSchema: z.ZodObject<{
     /**
      * For video inputs, the indices of frames the model relied on to answer.
      * Indices are 0-based and refer to entries in `frames.timestampsSeconds`.
+     *
+     * Nullable because providers with strict structured-output schemas (e.g. OpenAI)
+     * must mark every field required and represent "no value" as `null` rather than
+     * omitting the key, even for image inputs that were never asked to populate it.
      */
-    frameReferences: z.ZodOptional<z.ZodArray<z.ZodNumber, "many">>;
+    frameReferences: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodNumber, "many">>>;
     usage: z.ZodOptional<z.ZodObject<{
         inputTokens: z.ZodNumber;
         outputTokens: z.ZodNumber;
@@ -457,7 +461,7 @@ declare const AskResultSchema: z.ZodObject<{
         estimatedCost?: number | undefined;
         durationSeconds?: number | undefined;
     } | undefined;
-    frameReferences?: number[] | undefined;
+    frameReferences?: number[] | null | undefined;
 }, {
     issues: {
         priority: "critical" | "major" | "minor";
@@ -473,11 +477,12 @@ declare const AskResultSchema: z.ZodObject<{
         estimatedCost?: number | undefined;
         durationSeconds?: number | undefined;
     } | undefined;
-    frameReferences?: number[] | undefined;
+    frameReferences?: number[] | null | undefined;
 }>;
 /** Result returned by `ask()`. */
-type AskResult = z.infer<typeof AskResultSchema> & {
+type AskResult = Omit<z.infer<typeof AskResultSchema>, "frameReferences"> & {
     /** Present only when the input was a video. Describes which frames the model saw. */
+    frameReferences?: number[];
     frames?: VideoFramesMetadata;
 };
 /** Supported input shapes for image arguments accepted by the client. */
@@ -488,6 +493,36 @@ type ImageInput = Buffer | Uint8Array | string;
  * an image or a video.
  */
 type MediaInput = ImageInput;
+/**
+ * A single pre-sampled frame with an explicit timestamp. Use this shape when
+ * you want to control the timestamp the model sees; otherwise pass a bare
+ * `ImageInput` and the timestamp is derived from `fps` and the frame's position.
+ */
+interface TimestampedFrameInput {
+    /** Image source for the frame (buffer, data URL, base64, file path, or URL). */
+    image: ImageInput;
+    /** Timestamp in seconds, from the start of the sequence. Derived from `fps` when omitted. */
+    timestampSeconds?: number;
+}
+/**
+ * Pre-sampled frames supplied directly instead of a video file. Useful when you
+ * already have an array of screenshots and cannot (or would rather not) pass a
+ * video — this path never loads ffmpeg. Passed to `check()` / `ask()` in place
+ * of a `MediaInput`; downstream handling is identical to a sampled video.
+ */
+interface FramesInput {
+    /**
+     * Ordered frames, earliest first. Each entry is an image source or a
+     * `{ image, timestampSeconds }` pair. Must be non-empty and is capped at the
+     * same per-request frame limit as video sampling (60).
+     */
+    frames: ReadonlyArray<ImageInput | TimestampedFrameInput>;
+    /**
+     * Frame rate used to derive timestamps for frames that don't carry their own
+     * `timestampSeconds` (frame `i` maps to `i / fps` seconds). Default `1`.
+     */
+    fps?: number;
+}
 /** Supported image MIME types accepted by all providers. */
 type SupportedMimeType = "image/jpeg" | "image/png" | "image/webp" | "image/gif";
 /** Supported video MIME types the client can accept and sample frames from. */
@@ -628,9 +663,11 @@ interface VisualAIClient {
      * frames automatically; statements pass if they are true at any sampled
      * frame, and each statement result includes the timestamp where it
      * matched. The `frames` metadata on the result reports which timestamps
-     * the model saw.
+     * the model saw. Pass a `FramesInput` (`{ frames, fps? }`) to supply
+     * pre-sampled frames directly — handled identically to a video timeline but
+     * without loading ffmpeg.
      *
-     * @param input Image or video source as a buffer, URL, file path, or base64 string.
+     * @param input Image or video source as a buffer, URL, file path, or base64 string, or a `FramesInput` of pre-sampled frames.
      * @param statements One or more statements to validate against the input.
      * @param options Optional additional instructions and video sampling overrides.
      * @returns A structured result describing pass/fail, issues, and statement reasoning.
@@ -653,15 +690,17 @@ interface VisualAIClient {
      * console.log(result.statements[0].timestampSeconds); // e.g. 3.5
      * ```
      */
-    check(input: MediaInput, statements: string | string[], options?: CheckOptions): Promise<CheckResult>;
+    check(input: MediaInput | FramesInput, statements: string | string[], options?: CheckOptions): Promise<CheckResult>;
     /**
      * Asks an open-ended question about an image or video and returns a structured summary.
      *
      * Video inputs are sampled into frames and analyzed as a chronological
      * timeline. The result's `frameReferences` array surfaces which frames the
-     * model relied on for its answer.
+     * model relied on for its answer. Pass a `FramesInput` (`{ frames, fps? }`)
+     * to supply pre-sampled frames directly — handled identically to a video
+     * timeline but without loading ffmpeg.
      *
-     * @param input Image or video source as a buffer, URL, file path, or base64 string.
+     * @param input Image or video source as a buffer, URL, file path, or base64 string, or a `FramesInput` of pre-sampled frames.
      * @param prompt Prompt describing what to inspect in the input.
      * @param options Optional additional instructions and video sampling overrides.
      * @returns A summary with any detected issues.
@@ -673,7 +712,7 @@ interface VisualAIClient {
      * const result = await client.ask(screenshot, "What looks visually broken on this page?");
      * ```
      */
-    ask(input: MediaInput, prompt: string, options?: AskOptions): Promise<AskResult>;
+    ask(input: MediaInput | FramesInput, prompt: string, options?: AskOptions): Promise<AskResult>;
     /**
      * Compares two images and reports meaningful visual differences.
      *
@@ -1050,4 +1089,4 @@ declare function assertVisualResult(result: CheckResult, label?: string): void;
  */
 declare function assertVisualCompareResult(result: CompareResult, label?: string): void;
 
-export { Accessibility, type AccessibilityCheckName, type AccessibilityOptions, type AskOptions, type AskResult, AskResultSchema, type ChangeEntry, ChangeEntrySchema, type CheckOptions, type CheckResult, CheckResultSchema, type CompareOptions, type CompareResult, CompareResultSchema, type Confidence, ConfidenceSchema, Content, type ContentCheckName, type ContentOptions, DEFAULT_MODELS, type DiffImageResult, type ElementsVisibilityOptions, type Frame, type ImageInput, type Issue, type IssueCategory, IssueCategorySchema, type IssuePriority, IssuePrioritySchema, IssueSchema, type KnownModelName, Layout, type LayoutCheckName, type LayoutOptions, type MediaInput, Model, type PageLoadOptions, Provider, type ProviderName, ReasoningEffort, type ReasoningEffortLevel, type StatementResult, StatementResultSchema, type SupportedMimeType, type SupportedVideoMimeType, type UsageInfo, UsageInfoSchema, type VideoFramesMetadata, type VideoSamplingOptions, VisualAIAssertionError, VisualAIAuthError, type VisualAIClient, type VisualAIConfig, VisualAIConfigError, VisualAIError, type VisualAIErrorCode, VisualAIImageError, type VisualAIKnownError, VisualAIProviderError, VisualAIRateLimitError, VisualAIResponseParseError, VisualAITruncationError, VisualAIVideoError, assertVisualCompareResult, assertVisualResult, formatCheckResult, formatCompareResult, isVisualAIKnownError, visualAI };
+export { Accessibility, type AccessibilityCheckName, type AccessibilityOptions, type AskOptions, type AskResult, AskResultSchema, type ChangeEntry, ChangeEntrySchema, type CheckOptions, type CheckResult, CheckResultSchema, type CompareOptions, type CompareResult, CompareResultSchema, type Confidence, ConfidenceSchema, Content, type ContentCheckName, type ContentOptions, DEFAULT_MODELS, type DiffImageResult, type ElementsVisibilityOptions, type Frame, type FramesInput, type ImageInput, type Issue, type IssueCategory, IssueCategorySchema, type IssuePriority, IssuePrioritySchema, IssueSchema, type KnownModelName, Layout, type LayoutCheckName, type LayoutOptions, type MediaInput, Model, type PageLoadOptions, Provider, type ProviderName, ReasoningEffort, type ReasoningEffortLevel, type StatementResult, StatementResultSchema, type SupportedMimeType, type SupportedVideoMimeType, type TimestampedFrameInput, type UsageInfo, UsageInfoSchema, type VideoFramesMetadata, type VideoSamplingOptions, VisualAIAssertionError, VisualAIAuthError, type VisualAIClient, type VisualAIConfig, VisualAIConfigError, VisualAIError, type VisualAIErrorCode, VisualAIImageError, type VisualAIKnownError, VisualAIProviderError, VisualAIRateLimitError, VisualAIResponseParseError, VisualAITruncationError, VisualAIVideoError, assertVisualCompareResult, assertVisualResult, formatCheckResult, formatCompareResult, isVisualAIKnownError, visualAI };

package/dist/index.js

@@ -1705,7 +1705,57 @@ function isVideoInput(input) {
   }
   return false;
 }
+function isFramesInput(input) {
+  return typeof input === "object" && input !== null && !Buffer.isBuffer(input) && !(input instanceof Uint8Array) && Array.isArray(input.frames);
+}
+function isTimestampedFrameInput(frame) {
+  return typeof frame === "object" && !Buffer.isBuffer(frame) && !(frame instanceof Uint8Array) && "image" in frame;
+}
+async function normalizeFrames(input) {
+  const rawFrames = input.frames;
+  const fps = input.fps ?? DEFAULT_FPS;
+  if (rawFrames.length === 0) {
+    throw new VisualAIVideoError("frames must be a non-empty array of image inputs");
+  }
+  if (rawFrames.length > MAX_FRAMES_HARD_CAP) {
+    throw new VisualAIVideoError(
+      `frames length ${rawFrames.length} exceeds the hard cap of ${MAX_FRAMES_HARD_CAP}. Pass fewer frames or open an issue if you need a larger limit.`
+    );
+  }
+  if (!Number.isFinite(fps) || fps <= 0) {
+    throw new VisualAIVideoError(`Invalid fps: ${fps}. Must be a finite number > 0.`);
+  }
+  const frames = await Promise.all(
+    rawFrames.map(async (raw, index) => {
+      const timestamped = isTimestampedFrameInput(raw);
+      const imageInput = timestamped ? raw.image : raw;
+      const explicit = timestamped ? raw.timestampSeconds : void 0;
+      const timestampSeconds = explicit ?? index / fps;
+      if (!Number.isFinite(timestampSeconds) || timestampSeconds < 0) {
+        throw new VisualAIVideoError(
+          `Invalid timestampSeconds for frame ${index}: ${String(timestampSeconds)}. Must be a finite number >= 0.`
+        );
+      }
+      const image = await normalizeImage(imageInput);
+      return {
+        data: image.data,
+        mimeType: image.mimeType,
+        get base64() {
+          return image.base64;
+        },
+        timestampSeconds,
+        index
+      };
+    })
+  );
+  const durationSeconds = frames.reduce((max, f) => Math.max(max, f.timestampSeconds), 0);
+  await saveDebugFrames(frames);
+  return { kind: "video", frames, durationSeconds };
+}
 async function normalizeMedia(input, videoOptions) {
+  if (isFramesInput(input)) {
+    return normalizeFrames(input);
+  }
   if (isVideoInput(input)) {
     const { path, cleanup } = await resolveVideoToPath(input);
     try {
@@ -1785,8 +1835,12 @@ var AskResultSchema = z.object({
   /**
    * For video inputs, the indices of frames the model relied on to answer.
    * Indices are 0-based and refer to entries in `frames.timestampsSeconds`.
+   *
+   * Nullable because providers with strict structured-output schemas (e.g. OpenAI)
+   * must mark every field required and represent "no value" as `null` rather than
+   * omitting the key, even for image inputs that were never asked to populate it.
    */
-  frameReferences: z.array(z.number().int().nonnegative()).optional(),
+  frameReferences: z.array(z.number().int().nonnegative()).nullable().optional(),
   usage: UsageInfoSchema.optional()
 });
 
@@ -1838,7 +1892,11 @@ function parseCheckResponse(raw) {
   return reconcileCheckResult(result);
 }
 function parseAskResponse(raw) {
-  return parseResponse(raw, AskResponseSchema);
+  const result = parseResponse(raw, AskResponseSchema);
+  return {
+    ...result,
+    frameReferences: result.frameReferences ?? void 0
+  };
 }
 function parseCompareResponse(raw) {
   return parseResponse(raw, CompareResponseSchema);