visual-ai-assertions 0.5.0 → 0.6.0

package/README.md

@@ -387,11 +387,14 @@ The `VisualAIKnownError` union and `isVisualAIKnownError()` helper are useful wh
 
 ### Optional Configuration
 
-| Variable                | Description                                                                                    |
-| ----------------------- | ---------------------------------------------------------------------------------------------- |
-| `VISUAL_AI_MODEL`       | Default model when `model` is not set in config. Overrides the provider's default model.       |
-| `VISUAL_AI_DEBUG`       | Enable debug logging when `debug` is not set in config. Use `"true"` or `"1"` to enable.       |
-| `VISUAL_AI_TRACK_USAGE` | Enable usage tracking when `trackUsage` is not set in config. Use `"true"` or `"1"` to enable. |
+| Variable                     | Description                                                                                                            |
+| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------- |
+| `VISUAL_AI_MODEL`            | Default model when `model` is not set in config. Overrides the provider's default model.                               |
+| `VISUAL_AI_DEBUG`            | Enable error diagnostic logging to stderr. Does **not** enable prompt/response logging. Use `"true"` or `"1"`.         |
+| `VISUAL_AI_DEBUG_PROMPT`     | Enable prompt-only debug logging to stderr. Use `"true"` or `"1"`.                                                     |
+| `VISUAL_AI_DEBUG_RESPONSE`   | Enable response-only debug logging to stderr. Use `"true"` or `"1"`.                                                   |
+| `VISUAL_AI_REASONING_EFFORT` | Default reasoning effort when `reasoningEffort` is not set in config. Use `"low"`, `"medium"`, `"high"`, or `"xhigh"`. |
+| `VISUAL_AI_TRACK_USAGE`      | Enable usage tracking (token counts and cost) to stderr. Use `"true"` or `"1"`.                                        |
 
 ## Configuration
 
@@ -399,7 +402,9 @@ The `VisualAIKnownError` union and `isVisualAIKnownError()` helper are useful wh
 | ----------------- | ------- | ---------------- | ----------------------------------------------------------------------------- |
 | `apiKey`          | string  | env var          | API key for the provider                                                      |
 | `model`           | string  | provider default | Model to use                                                                  |
-| `debug`           | boolean | `false`          | Log prompts/responses to stderr                                               |
+| `debug`           | boolean | `false`          | Enable error diagnostic logging to stderr                                     |
+| `debugPrompt`     | boolean | `false`          | Log prompts to stderr                                                         |
+| `debugResponse`   | boolean | `false`          | Log responses to stderr                                                       |
 | `maxTokens`       | number  | `4096`           | Max tokens for AI response                                                    |
 | `reasoningEffort` | string  | `undefined`      | `"low"` `"medium"` `"high"` `"xhigh"` — controls how deeply the model reasons |
 | `trackUsage`      | boolean | `false`          | Log token usage and estimated cost to stderr                                  |

package/dist/index.cjs

@@ -100,6 +100,11 @@ var MODEL_TO_PROVIDER = new Map([
   ...Object.values(Model.Google).map((m) => [m, Provider.GOOGLE])
 ]);
 var VALID_PROVIDERS = Object.values(Provider);
+var PROVIDER_DEFAULT_REASONING = {
+  openai: "medium",
+  anthropic: "off",
+  google: "off"
+};
 var Content = {
   /** Detects Lorem ipsum, TODO, TBD, and similar placeholder text */
   PLACEHOLDER_TEXT: "placeholder-text",
@@ -752,16 +757,38 @@ function parseBooleanEnv(envName, value) {
     `Invalid ${envName} value: "${value}". Use "true", "1", "false", or "0".`
   );
 }
+var VALID_REASONING_EFFORTS = ["low", "medium", "high", "xhigh"];
+function parseReasoningEffortEnv(envName, value) {
+  if (value === void 0 || value === "") return void 0;
+  const lower = value.toLowerCase();
+  if (VALID_REASONING_EFFORTS.includes(lower)) return lower;
+  throw new VisualAIConfigError(
+    `Invalid ${envName} value: "${value}". Use "low", "medium", "high", or "xhigh".`
+  );
+}
+var debugDeprecationWarned = false;
 function resolveConfig(config) {
   const provider = resolveProvider(config);
   const model = config.model ?? process.env.VISUAL_AI_MODEL ?? DEFAULT_MODELS[provider];
+  const debug = config.debug ?? parseBooleanEnv("VISUAL_AI_DEBUG", process.env.VISUAL_AI_DEBUG) ?? false;
+  const debugPrompt = config.debugPrompt ?? parseBooleanEnv("VISUAL_AI_DEBUG_PROMPT", process.env.VISUAL_AI_DEBUG_PROMPT) ?? false;
+  const debugResponse = config.debugResponse ?? parseBooleanEnv("VISUAL_AI_DEBUG_RESPONSE", process.env.VISUAL_AI_DEBUG_RESPONSE) ?? false;
+  if (debug && !debugPrompt && !debugResponse && !debugDeprecationWarned) {
+    debugDeprecationWarned = true;
+    process.stderr.write(
+      `[visual-ai-assertions] Warning: VISUAL_AI_DEBUG no longer enables prompt/response logging. Use VISUAL_AI_DEBUG_PROMPT=true and/or VISUAL_AI_DEBUG_RESPONSE=true instead.
+`
+    );
+  }
   return {
     provider,
     apiKey: config.apiKey,
     model,
     maxTokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,
-    reasoningEffort: config.reasoningEffort,
-    debug: config.debug ?? parseBooleanEnv("VISUAL_AI_DEBUG", process.env.VISUAL_AI_DEBUG) ?? false,
+    reasoningEffort: config.reasoningEffort ?? parseReasoningEffortEnv("VISUAL_AI_REASONING_EFFORT", process.env.VISUAL_AI_REASONING_EFFORT),
+    debug,
+    debugPrompt,
+    debugResponse,
     trackUsage: config.trackUsage ?? parseBooleanEnv("VISUAL_AI_TRACK_USAGE", process.env.VISUAL_AI_TRACK_USAGE) ?? false
   };
 }
@@ -822,8 +849,9 @@ function calculateCost(provider, model, inputTokens, outputTokens) {
 }
 
 // src/core/debug.ts
-function debugLog(config, label, data) {
-  if (config.debug) {
+function debugLog(config, label, data, kind = "error") {
+  const enabled = kind === "prompt" ? config.debugPrompt : kind === "response" ? config.debugResponse : config.debug;
+  if (enabled) {
     process.stderr.write(`[visual-ai-assertions] ${label}: ${data}
 `);
   }
@@ -831,8 +859,9 @@ function debugLog(config, label, data) {
 function usageLog(config, method, usage) {
   if (!config.trackUsage) return;
   const costStr = usage.estimatedCost !== void 0 ? `$${usage.estimatedCost.toFixed(6)}` : "unknown";
+  const reasoningStr = config.reasoningEffort ? `reasoning: ${config.reasoningEffort}` : `reasoning: ${PROVIDER_DEFAULT_REASONING[config.provider]} (provider default)`;
   process.stderr.write(
-    `[visual-ai-assertions] ${method} usage: ${usage.inputTokens} input + ${usage.outputTokens} output tokens (${costStr}) in ${usage.durationSeconds?.toFixed(3) ?? "0.000"}s [${config.model}]
+    `[visual-ai-assertions] ${method} usage: ${usage.inputTokens} input + ${usage.outputTokens} output tokens (${costStr}) in ${usage.durationSeconds?.toFixed(3) ?? "0.000"}s [${config.model}, ${reasoningStr}]
 `
   );
 }
@@ -848,6 +877,28 @@ function processUsage(method, rawUsage, durationSeconds, config) {
   usageLog(config, method, usage);
   return usage;
 }
+var MAX_RAW_RESPONSE_PREVIEW = 500;
+function formatError(error) {
+  if (error instanceof VisualAIResponseParseError) {
+    const truncated = error.rawResponse.length > MAX_RAW_RESPONSE_PREVIEW ? error.rawResponse.slice(0, MAX_RAW_RESPONSE_PREVIEW) + "..." : error.rawResponse;
+    return `${error.name} (${error.code}): ${error.message}. Raw (truncated): ${truncated}`;
+  }
+  if (error instanceof VisualAIError) {
+    return `${error.name} (${error.code}): ${error.message}`;
+  }
+  if (error instanceof Error) {
+    return `${error.name}: ${error.message}`;
+  }
+  return String(error);
+}
+async function withErrorDebug(config, method, fn) {
+  try {
+    return await fn();
+  } catch (error) {
+    debugLog(config, `${method} error`, formatError(error), "error");
+    throw error;
+  }
+}
 async function timedSendMessage(driver, images, prompt) {
   const start = performance.now();
   const response = await driver.sendMessage(images, prompt);
@@ -1176,16 +1227,18 @@ function visualAI(config = {}) {
     if (elements.length === 0) {
       throw new VisualAIConfigError(`At least one element is required for ${methodName}()`);
     }
-    const img = await normalizeImage(image);
-    const prompt = buildElementsVisibilityPrompt(elements, visible, options);
-    debugLog(resolvedConfig, `${methodName} prompt`, prompt);
-    const response = await timedSendMessage(driver, [img], prompt);
-    debugLog(resolvedConfig, `${methodName} response`, response.text);
-    const result = parseCheckResponse(response.text);
-    return {
-      ...result,
-      usage: processUsage(methodName, response.usage, response.durationSeconds, resolvedConfig)
-    };
+    return withErrorDebug(resolvedConfig, methodName, async () => {
+      const img = await normalizeImage(image);
+      const prompt = buildElementsVisibilityPrompt(elements, visible, options);
+      debugLog(resolvedConfig, `${methodName} prompt`, prompt, "prompt");
+      const response = await timedSendMessage(driver, [img], prompt);
+      debugLog(resolvedConfig, `${methodName} response`, response.text, "response");
+      const result = parseCheckResponse(response.text);
+      return {
+        ...result,
+        usage: processUsage(methodName, response.usage, response.durationSeconds, resolvedConfig)
+      };
+    });
   }
   return {
     async check(image, statements, options) {
@@ -1193,61 +1246,64 @@ function visualAI(config = {}) {
       if (stmts.length === 0) {
         throw new VisualAIConfigError("At least one statement is required for check()");
       }
-      const img = await normalizeImage(image);
-      const prompt = buildCheckPrompt(stmts, { instructions: options?.instructions });
-      debugLog(resolvedConfig, "check prompt", prompt);
-      const response = await timedSendMessage(driver, [img], prompt);
-      debugLog(resolvedConfig, "check response", response.text);
-      const result = parseCheckResponse(response.text);
-      return {
-        ...result,
-        usage: processUsage("check", response.usage, response.durationSeconds, resolvedConfig)
-      };
+      return withErrorDebug(resolvedConfig, "check", async () => {
+        const img = await normalizeImage(image);
+        const prompt = buildCheckPrompt(stmts, { instructions: options?.instructions });
+        debugLog(resolvedConfig, "check prompt", prompt, "prompt");
+        const response = await timedSendMessage(driver, [img], prompt);
+        debugLog(resolvedConfig, "check response", response.text, "response");
+        const result = parseCheckResponse(response.text);
+        return {
+          ...result,
+          usage: processUsage("check", response.usage, response.durationSeconds, resolvedConfig)
+        };
+      });
     },
     async ask(image, userPrompt, options) {
-      const img = await normalizeImage(image);
-      const prompt = buildAskPrompt(userPrompt, { instructions: options?.instructions });
-      debugLog(resolvedConfig, "ask prompt", prompt);
-      const response = await timedSendMessage(driver, [img], prompt);
-      debugLog(resolvedConfig, "ask response", response.text);
-      const result = parseAskResponse(response.text);
-      return {
-        ...result,
-        usage: processUsage("ask", response.usage, response.durationSeconds, resolvedConfig)
-      };
+      return withErrorDebug(resolvedConfig, "ask", async () => {
+        const img = await normalizeImage(image);
+        const prompt = buildAskPrompt(userPrompt, { instructions: options?.instructions });
+        debugLog(resolvedConfig, "ask prompt", prompt, "prompt");
+        const response = await timedSendMessage(driver, [img], prompt);
+        debugLog(resolvedConfig, "ask response", response.text, "response");
+        const result = parseAskResponse(response.text);
+        return {
+          ...result,
+          usage: processUsage("ask", response.usage, response.durationSeconds, resolvedConfig)
+        };
+      });
     },
     async compare(imageA, imageB, options) {
-      const [imgA, imgB] = await Promise.all([normalizeImage(imageA), normalizeImage(imageB)]);
-      const prompt = buildComparePrompt({
-        userPrompt: options?.prompt,
-        instructions: options?.instructions
-      });
-      debugLog(resolvedConfig, "compare prompt", prompt);
-      const response = await timedSendMessage(driver, [imgA, imgB], prompt);
-      debugLog(resolvedConfig, "compare response", response.text);
-      const supportsAnnotatedDiff = resolvedConfig.provider === "google" && resolvedConfig.model === Model.Google.GEMINI_3_FLASH_PREVIEW;
-      const effectiveDiffImage = options?.diffImage ?? (supportsAnnotatedDiff ? true : false);
-      let diffImage;
-      if (effectiveDiffImage) {
-        try {
-          diffImage = await generateAiDiff(imgA, imgB, resolvedConfig.model, driver);
-        } catch (err) {
-          const msg = err instanceof Error ? err.message : String(err);
-          debugLog(resolvedConfig, "ai diff error", msg);
-          if (!resolvedConfig.debug) {
+      return withErrorDebug(resolvedConfig, "compare", async () => {
+        const [imgA, imgB] = await Promise.all([normalizeImage(imageA), normalizeImage(imageB)]);
+        const prompt = buildComparePrompt({
+          userPrompt: options?.prompt,
+          instructions: options?.instructions
+        });
+        debugLog(resolvedConfig, "compare prompt", prompt, "prompt");
+        const response = await timedSendMessage(driver, [imgA, imgB], prompt);
+        debugLog(resolvedConfig, "compare response", response.text, "response");
+        const supportsAnnotatedDiff = resolvedConfig.provider === "google" && resolvedConfig.model === Model.Google.GEMINI_3_FLASH_PREVIEW;
+        const effectiveDiffImage = options?.diffImage ?? (supportsAnnotatedDiff ? true : false);
+        let diffImage;
+        if (effectiveDiffImage) {
+          try {
+            diffImage = await generateAiDiff(imgA, imgB, resolvedConfig.model, driver);
+          } catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
             process.stderr.write(
               `[visual-ai-assertions] warning: diff generation failed: ${msg}
 `
             );
           }
         }
-      }
-      const result = parseCompareResponse(response.text);
-      return {
-        ...result,
-        ...diffImage ? { diffImage } : {},
-        usage: processUsage("compare", response.usage, response.durationSeconds, resolvedConfig)
-      };
+        const result = parseCompareResponse(response.text);
+        return {
+          ...result,
+          ...diffImage ? { diffImage } : {},
+          usage: processUsage("compare", response.usage, response.durationSeconds, resolvedConfig)
+        };
+      });
     },
     elementsVisible(image, elements, options) {
       return checkElementsVisibility(image, elements, true, options);
@@ -1256,57 +1312,65 @@ function visualAI(config = {}) {
       return checkElementsVisibility(image, elements, false, options);
     },
     async accessibility(image, options) {
-      const img = await normalizeImage(image);
-      const prompt = buildAccessibilityPrompt(options);
-      debugLog(resolvedConfig, "accessibility prompt", prompt);
-      const response = await timedSendMessage(driver, [img], prompt);
-      debugLog(resolvedConfig, "accessibility response", response.text);
-      const result = parseCheckResponse(response.text);
-      return {
-        ...result,
-        usage: processUsage(
-          "accessibility",
-          response.usage,
-          response.durationSeconds,
-          resolvedConfig
-        )
-      };
+      return withErrorDebug(resolvedConfig, "accessibility", async () => {
+        const img = await normalizeImage(image);
+        const prompt = buildAccessibilityPrompt(options);
+        debugLog(resolvedConfig, "accessibility prompt", prompt, "prompt");
+        const response = await timedSendMessage(driver, [img], prompt);
+        debugLog(resolvedConfig, "accessibility response", response.text, "response");
+        const result = parseCheckResponse(response.text);
+        return {
+          ...result,
+          usage: processUsage(
+            "accessibility",
+            response.usage,
+            response.durationSeconds,
+            resolvedConfig
+          )
+        };
+      });
     },
     async layout(image, options) {
-      const img = await normalizeImage(image);
-      const prompt = buildLayoutPrompt(options);
-      debugLog(resolvedConfig, "layout prompt", prompt);
-      const response = await timedSendMessage(driver, [img], prompt);
-      debugLog(resolvedConfig, "layout response", response.text);
-      const result = parseCheckResponse(response.text);
-      return {
-        ...result,
-        usage: processUsage("layout", response.usage, response.durationSeconds, resolvedConfig)
-      };
+      return withErrorDebug(resolvedConfig, "layout", async () => {
+        const img = await normalizeImage(image);
+        const prompt = buildLayoutPrompt(options);
+        debugLog(resolvedConfig, "layout prompt", prompt, "prompt");
+        const response = await timedSendMessage(driver, [img], prompt);
+        debugLog(resolvedConfig, "layout response", response.text, "response");
+        const result = parseCheckResponse(response.text);
+        return {
+          ...result,
+          usage: processUsage("layout", response.usage, response.durationSeconds, resolvedConfig)
+        };
+      });
     },
     async pageLoad(image, options) {
-      const img = await normalizeImage(image);
-      const prompt = buildPageLoadPrompt(options);
-      debugLog(resolvedConfig, "pageLoad prompt", prompt);
-      const response = await timedSendMessage(driver, [img], prompt);
-      debugLog(resolvedConfig, "pageLoad response", response.text);
-      const result = parseCheckResponse(response.text);
-      return {
-        ...result,
-        usage: processUsage("pageLoad", response.usage, response.durationSeconds, resolvedConfig)
-      };
+      return withErrorDebug(resolvedConfig, "pageLoad", async () => {
+        const img = await normalizeImage(image);
+        const prompt = buildPageLoadPrompt(options);
+        debugLog(resolvedConfig, "pageLoad prompt", prompt, "prompt");
+        const response = await timedSendMessage(driver, [img], prompt);
+        debugLog(resolvedConfig, "pageLoad response", response.text, "response");
+        const result = parseCheckResponse(response.text);
+        return {
+          ...result,
+          usage: processUsage("pageLoad", response.usage, response.durationSeconds, resolvedConfig)
+        };
+      });
     },
     async content(image, options) {
-      const img = await normalizeImage(image);
-      const prompt = buildContentPrompt(options);
-      debugLog(resolvedConfig, "content prompt", prompt);
-      const response = await timedSendMessage(driver, [img], prompt);
-      debugLog(resolvedConfig, "content response", response.text);
-      const result = parseCheckResponse(response.text);
-      return {
-        ...result,
-        usage: processUsage("content", response.usage, response.durationSeconds, resolvedConfig)
-      };
+      return withErrorDebug(resolvedConfig, "content", async () => {
+        const img = await normalizeImage(image);
+        const prompt = buildContentPrompt(options);
+        debugLog(resolvedConfig, "content prompt", prompt, "prompt");
+        const response = await timedSendMessage(driver, [img], prompt);
+        debugLog(resolvedConfig, "content response", response.text, "response");
+        const result = parseCheckResponse(response.text);
+        return {
+          ...result,
+          usage: processUsage("content", response.usage, response.durationSeconds, resolvedConfig)
+        };
+      });
     }
   };
 }