pi-vision-handoff 0.3.0 → 0.3.2

package/src/index.ts

@@ -38,14 +38,39 @@ export const DEFAULT_USER_PROMPT_PREFIX = "The user's request about this image:
 export const IMAGE_PLACEHOLDER_PREFIX = "[Image: ";
 export const IMAGE_PLACEHOLDER_SUFFIX = "]";
 
-/** Default max output tokens for a single image description. */
-export const DEFAULT_MAX_TOKENS = 1024;
+/** Marker appended to a description whose `complete()` call ended with
+ *  `stopReason: "length"` — i.e. the vision model hit a token limit (either the
+ *  configured `maxTokens` or the provider's hard output cap) before finishing.
+ *  A truncated description is still useful, but the agent must not mistake it
+ *  for the complete picture (it would reason as if the image contained only
+ *  what made it into the text). The marker makes the cut-off visible to both
+ *  the agent (it reads the marker in the tool result / context) and the user
+ *  (it renders inline). */
+export const DESCRIPTION_TRUNCATED_MARKER =
+  "\n\n[... description truncated — the vision model reached its output token limit. The above may be incomplete; raise maxTokens or use a higher-output vision model for the full description.]";
+
+/** Append the {@link DESCRIPTION_TRUNCATED_MARKER} to a raw description. */
+export function markDescriptionTruncated(text: string): string {
+  return text + DESCRIPTION_TRUNCATED_MARKER;
+}
 
 /** Default vision cache size (number of described images kept in memory per session). */
 export const DEFAULT_CACHE_MAX = 50;
 
-/** Per-description request timeout. */
-export const DESCRIBE_TIMEOUT_MS = 30_000;
+/** Per-description request timeout. Generous because the describer generates an
+ *  exhaustive multi-paragraph description per image; a single image commonly
+ *  takes ~20s, and a batched call over websocket transport scales with image
+ *  count. {@link describeTimeoutMs} scales this per batch size. */
+export const DESCRIBE_TIMEOUT_MS = 120_000;
+
+/** Per-batch timeout: the base timeout plus a per-image budget so a 5-image
+ *  batch isn't held to the same wall-clock budget as a single image. The
+ *  describer generates exhaustive prose per image, and websocket transport
+ *  adds latency, so the budget scales with the number of images in the call. */
+export function describeTimeoutMs(imageCount: number): number {
+  const perImage = 45_000;
+  return DESCRIBE_TIMEOUT_MS + Math.max(0, imageCount - 1) * perImage;
+}
 
 /**
  * Default cap on the number of lines kept from a description before truncation.
@@ -91,8 +116,13 @@ export interface VisionHandoffConfig {
   autoHandoff: boolean;
   /** Extra "provider/id" refs that should ALSO receive handoff (e.g. weak vision models). */
   handoffModels: string[];
-  /** Max output tokens for a single description. */
-  maxTokens: number;
+  /** Max output tokens for a single description. `undefined` (default) = use the
+   *  vision model's declared max output (`model.maxTokens`) as the cap — the
+   *  highest the model supports — so the "be exhaustive" prompt isn't defeated
+   *  by a provider's small omitted-default. Set a number only to cap lower.
+   *  A truncation is always surfaced via {@link DESCRIPTION_TRUNCATED_MARKER}
+   *  when the model hits the limit. */
+  maxTokens?: number;
   /** Max images kept in the in-memory description cache. */
   cacheMax: number;
   /**
@@ -112,7 +142,7 @@ export const DEFAULT_CONFIG: VisionHandoffConfig = {
   visionModel: null,
   autoHandoff: true,
   handoffModels: [],
-  maxTokens: DEFAULT_MAX_TOKENS,
+  maxTokens: undefined,
   cacheMax: DEFAULT_CACHE_MAX,
   maxDescriptionLines: DEFAULT_MAX_DESCRIPTION_LINES,
 };
@@ -159,6 +189,8 @@ export function normalizeConfig(raw: unknown): VisionHandoffConfig {
       .map((m) => m.trim())
       .filter((m) => m && parseModelRef(m));
   }
+  // maxTokens: optional. undefined (default) = no artificial cap. Only set when
+  // a valid positive finite number is given; any other value leaves it unset.
   if (typeof obj.maxTokens === "number" && Number.isFinite(obj.maxTokens) && obj.maxTokens > 0) {
     base.maxTokens = Math.floor(obj.maxTokens);
   }
@@ -292,6 +324,100 @@ export function truncateDescription(text: string, maxLines: number): TruncatedDe
   return { text: `${kept}\n... (${hidden} more lines)`, truncated: true, hidden };
 }
 
+/** Start marker for the k-th image section in a batched describer response. */
+export const BATCH_IMAGE_MARKER_START = "<<<IMAGE";
+/** End marker closing a single image section in a batched describer response. */
+export const BATCH_IMAGE_MARKER_END = "<<<END>>>";
+
+/** Escape a literal string for use in a RegExp. */
+function escapeRe(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// Section regexes built from the marker constants so the producer
+// (batchUserPrompt) and consumer (parseBatchedDescriptions) share one source of
+// truth for the delimiter format. A lone `<<<IMAGE k>>> … <<<END>>>` wrapper
+// for a single image; a global scan for the multi-image case that stops at the
+// next start marker, the end marker, or end-of-text.
+const SECTION_SINGLE_RE = new RegExp(
+  `${escapeRe(BATCH_IMAGE_MARKER_START)}\\s+1>>>\\s*([\\s\\S]*?)${escapeRe(BATCH_IMAGE_MARKER_END)}`,
+);
+const SECTION_MULTI_RE = new RegExp(
+  `${escapeRe(BATCH_IMAGE_MARKER_START)}\\s+(\\d+)>>>\\s*([\\s\\S]*?)(?=${escapeRe(
+    BATCH_IMAGE_MARKER_START,
+  )}\\s+\\d+>>>|${escapeRe(BATCH_IMAGE_MARKER_END)}|$)`,
+  "g",
+);
+const SECTION_END_STRIP_RE = new RegExp(`${escapeRe(BATCH_IMAGE_MARKER_END)}\\s*$`);
+
+/** Build the user-message text block for a (possibly batched) describer call.
+ *
+ * For a single image (count <= 1) this returns the same simple prompt the
+ * original per-image pipeline used (no markers), so single-image behaviour is
+ * unchanged. For count > 1 it instructs the vision model to emit one delimited
+ * `<<<IMAGE k>>> … <<<END>>>` section per image, in order, so the response can
+ * be split back into per-image descriptions by {@link parseBatchedDescriptions}.
+ *
+ * The user's turn prompt (if any) is folded in via {@link DEFAULT_USER_PROMPT_PREFIX}
+ * so every image in the batch is described in the context of the same request —
+ * one describer call covers the whole prompt's image set, dataloader-style,
+ * rather than spinning up one agent call per image. */
+export function batchUserPrompt(count: number, userPrompt: string, prefix: string): string {
+  const userLine = userPrompt && userPrompt.trim() ? `${prefix}${userPrompt}` : "";
+  if (count <= 1) {
+    return userLine || "Describe this image.";
+  }
+  return [
+    `Describe each of the ${count} images below exhaustively, in order. For image k (1 through ${count}), emit exactly:`,
+    "",
+    `${BATCH_IMAGE_MARKER_START} k>>>`,
+    "<your exhaustive description>",
+    BATCH_IMAGE_MARKER_END,
+    ...(userLine ? ["", userLine] : []),
+  ].join("\n");
+}
+
+/** Parse a batched vision-model response into one description per image, in order.
+ *
+ * Returns an array of length `count`; each slot is the trimmed description for
+ * image k (1-indexed → array index k-1), or `null` when that image's section is
+ * missing, empty, or unparseable. Callers treat `null` as "description
+ * unavailable" (graceful degradation) — one image failing to parse must not
+ * void the rest of the batch.
+ *
+ * For `count <= 1` the whole response is treated as the single image's
+ * description (a lone `<<<IMAGE 1>>> … <<<END>>>` wrapper is stripped if the
+ * model emitted one anyway), preserving the original single-image behaviour. */
+export function parseBatchedDescriptions(text: string, count: number): (string | null)[] {
+  const trimmed = (text ?? "").trim();
+  if (count <= 1) {
+    const m = SECTION_SINGLE_RE.exec(trimmed);
+    const body = (m ? m[1] : trimmed).trim();
+    return [body || null];
+  }
+  const out: (string | null)[] = new Array(count).fill(null);
+  let m: RegExpExecArray | null;
+  SECTION_MULTI_RE.lastIndex = 0;
+  while ((m = SECTION_MULTI_RE.exec(trimmed)) !== null) {
+    const idx = parseInt(m[1], 10) - 1;
+    const body = m[2].replace(SECTION_END_STRIP_RE, "").trim();
+    if (idx >= 0 && idx < count && !out[idx]) out[idx] = body || null;
+  }
+  return out;
+}
+
+/** Truncate (if configured) and wrap a raw description in the `[Image: …]`
+ *  placeholder envelope used by every insertion path. Centralises the wrap+
+ *  truncate logic shared by the read-tool and context injection paths so both
+ *  surfaces stay identical for the same image hash. */
+export function wrapDescription(description: string, cfg: VisionHandoffConfig): string {
+  const { text: final } =
+    cfg.maxDescriptionLines && cfg.maxDescriptionLines > 0
+      ? truncateDescription(description, cfg.maxDescriptionLines)
+      : { text: description };
+  return `${IMAGE_PLACEHOLDER_PREFIX}${final}${IMAGE_PLACEHOLDER_SUFFIX}`;
+}
+
 /** Outcome of {@link insertImageDescriptions}. */
 export interface ReplacedContent {
   content: (TextContent | ImageContent)[];

package/src/usage.ts

@@ -49,9 +49,19 @@ export const EMPTY_ENERGY_CAPTURE: VisionHandoffEnergyCapture = {
 };
 
 /** A single describer-call usage record. Energy fields are present only when
- *  Neuralwatt SSE energy comments were captured (omitted, not zeroed, otherwise). */
+ *  Neuralwatt SSE energy comments were captured (omitted, not zeroed, otherwise).
+ *
+ *  One record is emitted per REAL describer provider call (cache hits emit
+ *  nothing). A batched call that describes several images at once still emits a
+ *  single record: `imageHash` is the representative (first) member and
+ *  `imageHashes` lists every image the call covered, so consumers can attribute
+ *  the call's tokens/energy to each member image without double-counting. */
 export interface VisionHandoffUsageRecord {
+  /** Representative image hash (first member of the batch). */
   imageHash: string;
+  /** All image hashes covered by this describer call. Present only for batched
+   *  calls (length > 1); omitted for single-image calls. */
+  imageHashes?: string[];
   model: string;
   provider: string;
   responseModel?: string;
@@ -148,6 +158,7 @@ export function buildUsageRecord(
   capture: VisionHandoffEnergyCapture,
   visionModel: Model<Api>,
   imageHash: string,
+  imageHashes?: string[],
 ): VisionHandoffUsageRecord | null {
   const hasEnergy = !!(
     capture.energyRaw ||
@@ -169,6 +180,11 @@ export function buildUsageRecord(
     responseId: response.responseId,
     usage: response.usage,
   };
+  // Present only for genuine batched calls (more than one image). Keeps the
+  // single-image record shape unchanged for existing consumers.
+  if (imageHashes && imageHashes.length > 1) {
+    record.imageHashes = imageHashes;
+  }
   if (hasEnergy) {
     record.energyJoules = capture.energyJoules;
     record.costUsd = capture.costUsd;