pi-braid 0.43.58 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -96,6 +96,20 @@ Adding a thirteenth extension moves the counts six source claims pin to the mani
96
96
  schema's recorded line for the `braid.contrib.*` reader; `check:doc-drift` and `check:schema` demanded
97
97
  both repairs in the same commit.
98
98
 
99
+ ## Release — a missing version is absence, not a failure (2026-11-12)
100
+
101
+ The registry gate failed the first release it ran on, and it was right to be looked at: a version that had
102
+ just been published was not yet served, `npm pack` answered `ETARGET` rather than `E404`, the read counted
103
+ as unanswerable, and the verify step exited 2 while the publish itself had succeeded. Absence now covers both
104
+ codes npm uses, and only a read that answers neither is unanswerable, with the three meanings pinned by
105
+ tests that carry the real `ETARGET` body.
106
+
107
+ The fix exposed a second defect on the way in: the script ran its CLI on import, so the test that imports it
108
+ to reach one pure function packed every workspace, `npm pack` ran this package's `prepack`, the prepack
109
+ rebuilt `dist`, and for the fourteen seconds of that rebuild every fixture in the pooled run that imports
110
+ `dist` failed with a module-not-found - nine files, in a run where the same commit passed in a clean
111
+ checkout. The CLI now runs only when the file is the program.
112
+
99
113
  ## Skills — one design loop, and a gate that refuses a private literal (2026-11-12)
100
114
 
101
115
  Two changes in one release, both about a check that did not exist.
@@ -186,6 +200,32 @@ tarball did not have. All three move here, with the fixes they already hold:
186
200
  instead of throwing out of a timer and ending the process.
187
201
  - `pi-braid-contrib` 0.1.5: the git tool, argv-only and read-only by default.
188
202
 
203
+ ## Core 0.44.0, Contrib 0.2.0, Providers 1.0.8 — the ruler gains eyes, and refusals learn to speak (2026-11-12)
204
+
205
+ `design_audit` could measure a rendered page and say nothing about whether it looked cheap. The same
206
+ adapter's full-page PNGs now go to the model the session resolves for vision, one call per width, in
207
+ `visual_judge` beside the audit. The result names the model that answered, its route, the tokens and the
208
+ elapsed time, because an opinion with no author is not a report; the default rubric asks for named
209
+ observations and forbids scores, since a judge that answers 7/10 has measured nothing.
210
+
211
+ The generic half stayed in `lib/ai`, where the retry ladder, the pre-send budget and the key-ring
212
+ feedback already live. `callLlm` gained an optional image part and `callVision` is the named entry that
213
+ refuses a text-only model or a malformed part before a request is built, so an image is priced at the
214
+ budget where the text estimator had counted it as free. `resolveVisionModel` and `searchVisionModel`
215
+ share one walk, so a refusal prints every rung it passed over and why it was skipped.
216
+
217
+ A test-harness defect fell out on the way, and it was the kind that hides: contrib's fixtures import core as
218
+ `pi-braid/lib`, which in a checkout resolves through `node_modules` to whichever pi-braid is installed
219
+ rather than the tree under test, so a core export added in the same tree was invisible and the failure read
220
+ as a broken import. The shared hook maps that specifier to this tree's build, and an arm compares the
221
+ build's mtime against the newest core source, because a hook that reads an artifact can otherwise report a
222
+ green suite about code the tree no longer contains. Touching a source without rebuilding reddens it;
223
+ removing the export from the artifact reddens it naming the export; reverting the image part fails one arm
224
+ of twelve.
225
+
226
+ Contrib moves a minor because a new tool is something a caller notices, core because `pi-braid/lib` gained
227
+ five names; providers carries the rotation census at 1.0.8 and is unchanged here.
228
+
189
229
  ## Core — a prefix is ranked, never qualifying on its own (2026-11-11)
190
230
 
191
231
  The same defect as the class noun, one leg over: `improve` starts `improvement`, so the token-prefix leg
package/build-info.json CHANGED
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "name": "pi-braid",
3
- "version": "0.43.58",
4
- "sha": "99f15692c030acf8aaba4f3c42b15321489b74e2",
5
- "shaShort": "99f15692",
3
+ "version": "0.44.0",
4
+ "sha": "f0165737cba18bd9ce624ec1a4bd13f47494b7aa",
5
+ "shaShort": "f0165737",
6
6
  "branch": "detached",
7
7
  "dirty": false,
8
- "packedAt": "2026-11-12T12:05:14.037Z"
8
+ "packedAt": "2026-11-12T23:32:20.399Z"
9
9
  }
@@ -28,6 +28,11 @@
28
28
  * callLlm(model, apiKey, systemPrompt, userPrompt, options)
29
29
  * Returns { text } on success or { error } on failure (discriminated union).
30
30
  * const text = result.text;
31
+ *
32
+ * callVision(model, apiKey, systemPrompt, instruction, images, options)
33
+ * The same union with image parts attached to the user message. A model
34
+ * that declares no image input is refused before anything is sent, so a
35
+ * text-only model can never quietly answer about an image it never saw.
31
36
  */
32
37
  import type { Api, AssistantMessage, Context, Model, ModelThinkingLevel, ModelsSimpleStreamOptions } from "@earendil-works/pi-ai";
33
38
  /** One provider-neutral completion call. */
@@ -48,6 +53,32 @@ export declare function __resetRateLimitCooldownForTest(): void;
48
53
  import { setLlmCallListeners, type LlmErrorListener, type LlmSuccessListener } from "../state/global-state.js";
49
54
  export { setLlmCallListeners };
50
55
  export type { LlmErrorListener, LlmSuccessListener };
56
+ /**
57
+ * One image attached to an auxiliary request: base64 bytes and the MIME type
58
+ * they carry. `data` is the payload itself, without a `data:` prefix.
59
+ */
60
+ export interface LlmImagePart {
61
+ /** Base64 payload. */
62
+ data: string;
63
+ /** Declared MIME type, e.g. "image/png". The transport passes it through. */
64
+ mimeType: string;
65
+ }
66
+ /**
67
+ * What an auxiliary call returns: the text (with the stop reason and usage the
68
+ * provider reported) or one line naming why it failed. A vision call returns
69
+ * the same shape, so a caller never branches on which kind of call it made.
70
+ */
71
+ export type LlmCallResult = {
72
+ text: string;
73
+ stopReason?: string;
74
+ usage?: {
75
+ input?: number;
76
+ output?: number;
77
+ totalTokens?: number;
78
+ };
79
+ } | {
80
+ error: string;
81
+ };
51
82
  export interface LlmCallOptions {
52
83
  /** Temperature for generation. Default: 0.1 */
53
84
  temperature?: number;
@@ -85,6 +116,16 @@ export interface LlmCallOptions {
85
116
  sdkRetries?: number;
86
117
  /** Optional AbortSignal for cancellation */
87
118
  signal?: AbortSignal;
119
+ /**
120
+ * Image parts attached to the user message. `callVision` is the entry point
121
+ * that sets this: a text call leaves it out and its user content stays a
122
+ * plain string, so the request shape of every existing caller is unchanged.
123
+ *
124
+ * The model's declared input is checked here as well as in `callVision`,
125
+ * because the transport would otherwise drop the image silently or return a
126
+ * provider error that reads like a transient failure.
127
+ */
128
+ images?: readonly LlmImagePart[];
88
129
  }
89
130
  /**
90
131
  * Make an LLM call with two-tier retry, backoff, and rate-limit circuit breaker.
@@ -98,17 +139,25 @@ export interface LlmCallOptions {
98
139
  * @param options - Optional configuration
99
140
  * @returns `{ text }` on success or `{ error }` on failure
100
141
  */
101
- export declare function callLlm(model: Model<Api>, apiKey: string | undefined, systemPrompt: string, userPrompt: string, options?: LlmCallOptions): Promise<{
102
- text: string;
103
- stopReason?: string;
104
- usage?: {
105
- input?: number;
106
- output?: number;
107
- totalTokens?: number;
108
- };
109
- } | {
110
- error: string;
111
- }>;
142
+ export declare function callLlm(model: Model<Api>, apiKey: string | undefined, systemPrompt: string, userPrompt: string, options?: LlmCallOptions): Promise<LlmCallResult>;
143
+ /**
144
+ * Make an auxiliary call that carries images: the same dispatch, retry ladder,
145
+ * circuit breaker, pre-send budget and key-ring feedback as `callLlm`, with the
146
+ * images attached to the user message.
147
+ *
148
+ * Why this is a wrapper and not a second dispatcher: every policy the text
149
+ * path carries (per-model concurrency, transient retry with the backoff
150
+ * ceiling, the 413 emergency offload, the listener and key-ring seams) applies
151
+ * to a vision call exactly as it applies to a text one. A copy would drift from
152
+ * that policy at the first fix made in one place, and a vision call that does
153
+ * not feed the key ring back is a route that keeps failing without rotating.
154
+ * The image-specific decisions - is this model image-capable, is this part a
155
+ * real image - are the only ones the wrapper owns.
156
+ *
157
+ * `instruction` is the user turn; `images` are attached after it, in order, so
158
+ * a caller can refer to "the first image" and mean what it sent.
159
+ */
160
+ export declare function callVision(model: Model<Api>, apiKey: string | undefined, systemPrompt: string, instruction: string, images: readonly LlmImagePart[], options?: Omit<LlmCallOptions, "images">): Promise<LlmCallResult>;
112
161
  /**
113
162
  * The one yes/no question core asks the providers seam before re-using a
114
163
  * (provider, model) pair for an auxiliary call: can this pair serve a request
@@ -28,6 +28,11 @@
28
28
  * callLlm(model, apiKey, systemPrompt, userPrompt, options)
29
29
  * Returns { text } on success or { error } on failure (discriminated union).
30
30
  * const text = result.text;
31
+ *
32
+ * callVision(model, apiKey, systemPrompt, instruction, images, options)
33
+ * The same union with image parts attached to the user message. A model
34
+ * that declares no image input is refused before anything is sent, so a
35
+ * text-only model can never quietly answer about an image it never saw.
31
36
  */
32
37
  // Degraded transport only: see completeAuxCall below. Compat resolves the
33
38
  // credential from the model and the apiKey argument and nothing else, which is
@@ -231,6 +236,13 @@ export async function callLlm(model, apiKey, systemPrompt, userPrompt, options =
231
236
  log.debug(`Skipping LLM call - rate-limit cooldown ${remaining}ms remaining`);
232
237
  return { error: "Rate-limit cooldown active" };
233
238
  }
239
+ // Images: a request the model cannot read is refused before it is built,
240
+ // not sent and reported as a provider error (or, worse, answered from the
241
+ // text alone with no complaint about the missing picture).
242
+ const images = options.images ?? [];
243
+ const imageRefusal = refuseImages(model, images);
244
+ if (imageRefusal !== null)
245
+ return { error: imageRefusal };
234
246
  const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
235
247
  const maxTokens = options.maxTokens ?? Math.min(Math.max(1000, Math.floor(userPrompt.length / 8)), 4000);
236
248
  // 10-minute default (was 30s). Aborts should be signal-driven (ESC,
@@ -254,7 +266,9 @@ export async function callLlm(model, apiKey, systemPrompt, userPrompt, options =
254
266
  }
255
267
  const context = {
256
268
  systemPrompt: systemSafe.text,
257
- messages: [{ role: "user", content: userSafe.text, timestamp: Date.now() }],
269
+ messages: [
270
+ { role: "user", content: userContent(userSafe.text, images), timestamp: Date.now() },
271
+ ],
258
272
  };
259
273
  // ── pre-send budget ──
260
274
  // A request over the window cannot succeed, and a provider 400 is a late,
@@ -265,7 +279,8 @@ export async function callLlm(model, apiKey, systemPrompt, userPrompt, options =
265
279
  // provider will refuse what the cap already refused.
266
280
  const resolvedWindow = resolveWindow(model.contextWindow);
267
281
  const budget = await guardAgainstDoomedSend({
268
- measure: () => calibrateEstimate(estimatePayloadTokens({ systemPrompt: context.systemPrompt, messages: context.messages })).tokens,
282
+ measure: () => calibrateEstimate(estimatePayloadTokens({ systemPrompt: context.systemPrompt, messages: context.messages })).tokens +
283
+ images.length * IMAGE_TOKEN_ALLOWANCE,
269
284
  outputTokens: maxTokens,
270
285
  contextWindow: resolvedWindow.contextWindow,
271
286
  // The only compaction available to an auxiliary call is the published
@@ -544,7 +559,94 @@ export async function callLlm(model, apiKey, systemPrompt, userPrompt, options =
544
559
  releaseModelSlot(model.id);
545
560
  }
546
561
  }
562
+ // ── Vision ────────────────────────────────────────────────────────────────────
563
+ /**
564
+ * Make an auxiliary call that carries images: the same dispatch, retry ladder,
565
+ * circuit breaker, pre-send budget and key-ring feedback as `callLlm`, with the
566
+ * images attached to the user message.
567
+ *
568
+ * Why this is a wrapper and not a second dispatcher: every policy the text
569
+ * path carries (per-model concurrency, transient retry with the backoff
570
+ * ceiling, the 413 emergency offload, the listener and key-ring seams) applies
571
+ * to a vision call exactly as it applies to a text one. A copy would drift from
572
+ * that policy at the first fix made in one place, and a vision call that does
573
+ * not feed the key ring back is a route that keeps failing without rotating.
574
+ * The image-specific decisions - is this model image-capable, is this part a
575
+ * real image - are the only ones the wrapper owns.
576
+ *
577
+ * `instruction` is the user turn; `images` are attached after it, in order, so
578
+ * a caller can refer to "the first image" and mean what it sent.
579
+ */
580
+ export async function callVision(model, apiKey, systemPrompt, instruction, images, options = {}) {
581
+ if (images.length === 0) {
582
+ return {
583
+ error: "Image input refused: a vision call needs at least one image part. " +
584
+ "Use callLlm for a text-only request.",
585
+ };
586
+ }
587
+ return callLlm(model, apiKey, systemPrompt, instruction, { ...options, images });
588
+ }
547
589
  // ── Helpers ────────────────────────────────────────────────────────────────────
590
+ /**
591
+ * Tokens the pre-send budget counts per attached image.
592
+ *
593
+ * The payload estimator reads text only: an image block contributes its
594
+ * `mimeType` and nothing of its `data` (see estimateMessageTokens in
595
+ * detection/context-usage.ts), so without this allowance an image-heavy
596
+ * request looks free and the guard waves through a payload the provider then
597
+ * rejects. An image is priced by its pixels, not its bytes, and a full-page
598
+ * capture is a few thousand tokens on every provider that accepts one: OpenAI
599
+ * at high detail tiles a 1440x3000 capture into 18 tiles (~3.2K tokens),
600
+ * Anthropic's pixel rule puts the same image near 1.6K after its own downscale.
601
+ * This constant is the conservative ceiling of that range, so the estimate errs
602
+ * toward refusing rather than toward a doomed send.
603
+ */
604
+ const IMAGE_TOKEN_ALLOWANCE = 4_000;
605
+ /**
606
+ * The one place that decides whether a request carrying images may be built.
607
+ * Null means yes. A refusal names the model, what it declares, and the offending
608
+ * part, because the caller's next move (pick another model, re-encode the
609
+ * image) depends on which of those failed.
610
+ */
611
+ function refuseImages(model, images) {
612
+ if (images.length === 0)
613
+ return null;
614
+ const declared = model.input ?? [];
615
+ if (!declared.includes("image")) {
616
+ return (`Image input refused: ${model.provider ?? "unknown"}/${model.id} declares input ` +
617
+ `[${declared.join(", ") || "text"}] and does not accept images. A vision call needs an ` +
618
+ `image-capable model; resolveVisionModel names one for the current session.`);
619
+ }
620
+ for (let index = 0; index < images.length; index++) {
621
+ const image = images[index];
622
+ const position = `Image part ${index + 1} of ${images.length}`;
623
+ if (typeof image.data !== "string" || image.data.length === 0) {
624
+ return `${position} carries no data. Expected base64 image bytes, not an empty string.`;
625
+ }
626
+ if (typeof image.mimeType !== "string" || !/^image\/[a-z0-9.+-]+$/i.test(image.mimeType)) {
627
+ return (`${position} declares no image MIME type (got ${JSON.stringify(image.mimeType)}). ` +
628
+ `Expected a type such as "image/png".`);
629
+ }
630
+ }
631
+ return null;
632
+ }
633
+ /**
634
+ * The user message content: a plain string for a text call, text-then-images for
635
+ * a vision call. The text block comes first so an instruction that says "the
636
+ * image below" reads in the order it was written.
637
+ */
638
+ function userContent(text, images) {
639
+ if (images.length === 0)
640
+ return text;
641
+ return [
642
+ { type: "text", text },
643
+ ...images.map((image) => ({
644
+ type: "image",
645
+ data: image.data,
646
+ mimeType: image.mimeType,
647
+ })),
648
+ ];
649
+ }
548
650
  /**
549
651
  * Run the published emergency offloader (context-manager's
550
652
  * context-emergency.ts) against the current session context. Lossy-safe:
@@ -27,7 +27,23 @@ export declare function normaliseRoleSetting(raw: unknown): string[];
27
27
  */
28
28
  export declare function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[];
29
29
  /**
30
- * The model a vision call should use.
30
+ * What a vision-model resolution found and what it passed over.
31
+ *
32
+ * `resolveVisionModel` answers "which model"; a caller that reports to a human
33
+ * also has to answer "and then what should I change", which needs the rungs
34
+ * that produced nothing. Both come from one walk, so the model a report names
35
+ * and the trace beside it cannot disagree.
36
+ */
37
+ export interface VisionModelSearch {
38
+ /** The model a vision call should use, or undefined when the walk found none. */
39
+ model?: Model<Api>;
40
+ /** Which rung supplied `model`; null when none did. */
41
+ source: "current" | "utilityModel" | "available" | null;
42
+ /** One line per rung, in the order tried, saying what it found or why not. */
43
+ tried: string[];
44
+ }
45
+ /**
46
+ * The model a vision call should use, with the walk that produced it.
31
47
  *
32
48
  * Priority:
33
49
  * 1. The current model when it accepts images: the image stays in the call
@@ -38,6 +54,10 @@ export declare function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[
38
54
  *
39
55
  * A text-only entry is never selected for an image call.
40
56
  */
57
+ export declare function searchVisionModel(ctx: ExtensionContext): VisionModelSearch;
58
+ /**
59
+ * The model a vision call should use: the `model` of the walk above.
60
+ */
41
61
  export declare function resolveVisionModel(ctx: ExtensionContext): Model<Api> | undefined;
42
62
  /**
43
63
  * Get the configured utility model for classification tasks.
@@ -42,6 +42,26 @@ export function normaliseRoleSetting(raw) {
42
42
  }
43
43
  return patterns;
44
44
  }
45
+ /**
46
+ * A model named the way a report names it: provider then id, the pair a reader
47
+ * can paste into a model flag.
48
+ */
49
+ function modelLabel(model) {
50
+ return `${model.provider ?? "unknown"}/${model.id}`;
51
+ }
52
+ /**
53
+ * Why a resolved model was not accepted for a role, in words, for a caller that
54
+ * has to explain an empty result. Null when the model is acceptable.
55
+ */
56
+ function rejectionReason(model, requireImage) {
57
+ if (!model)
58
+ return "no provider serves this pattern";
59
+ if (requireImage && !model.input?.includes("image")) {
60
+ const declared = model.input ?? [];
61
+ return `declares input [${declared.join(", ") || "text"}], no image input`;
62
+ }
63
+ return null;
64
+ }
45
65
  /**
46
66
  * Resolve a role setting to a single model.
47
67
  *
@@ -55,19 +75,40 @@ export function normaliseRoleSetting(raw) {
55
75
  *
56
76
  * @param requireImage - vision role only: entries must accept image input, so
57
77
  * a list may hold a text-only entry without an image being sent to it.
78
+ * @param trace - optional sink for one line per entry, in the order tried, so a
79
+ * caller that gets nothing back can print what was tried and why each entry
80
+ * was passed over. Passed in rather than returned because the successful path
81
+ * must stay allocation-free for the classification callers.
58
82
  */
59
- function resolveRoleModel(ctx, key, requireImage = false) {
83
+ function resolveRoleModel(ctx, key, requireImage = false, trace) {
60
84
  const raw = getCustomModelKey(key);
61
85
  const patterns = normaliseRoleSetting(raw);
62
- if (patterns.length === 0)
86
+ if (patterns.length === 0) {
87
+ trace?.push(`${key}: not configured`);
63
88
  return undefined;
89
+ }
64
90
  const acceptable = (model) => model && (!requireImage || model.input?.includes("image")) ? model : undefined;
65
- if (!Array.isArray(raw))
66
- return acceptable(resolveFromPattern(ctx, patterns[0]));
91
+ if (!Array.isArray(raw)) {
92
+ const resolved = resolveFromPattern(ctx, patterns[0]);
93
+ const model = acceptable(resolved);
94
+ if (!trace)
95
+ return model;
96
+ const reason = rejectionReason(resolved, requireImage);
97
+ trace.push(model && reason === null
98
+ ? `${key} "${patterns[0]}": using ${modelLabel(model)}, accepts image input`
99
+ : `${key} "${patterns[0]}": ${reason}`);
100
+ return model;
101
+ }
67
102
  for (const pattern of patterns) {
68
- const model = acceptable(resolveFromPattern(ctx, pattern));
69
- if (model && isModelRouteUsable(model))
103
+ const resolved = resolveFromPattern(ctx, pattern);
104
+ const model = acceptable(resolved);
105
+ if (model && isModelRouteUsable(model)) {
106
+ trace?.push(`${key} "${pattern}": using ${modelLabel(model)}, the provider reports the route usable`);
70
107
  return model;
108
+ }
109
+ trace?.push(model
110
+ ? `${key} "${pattern}": resolved ${modelLabel(model)}, but the provider reports the route unusable`
111
+ : `${key} "${pattern}": ${rejectionReason(resolved, requireImage)}`);
71
112
  }
72
113
  return undefined;
73
114
  }
@@ -83,7 +124,7 @@ export function resolveUtilityModels(ctx) {
83
124
  .filter((model) => model !== undefined);
84
125
  }
85
126
  /**
86
- * The model a vision call should use.
127
+ * The model a vision call should use, with the walk that produced it.
87
128
  *
88
129
  * Priority:
89
130
  * 1. The current model when it accepts images: the image stays in the call
@@ -94,18 +135,38 @@ export function resolveUtilityModels(ctx) {
94
135
  *
95
136
  * A text-only entry is never selected for an image call.
96
137
  */
97
- export function resolveVisionModel(ctx) {
138
+ export function searchVisionModel(ctx) {
139
+ const tried = [];
98
140
  // 1. Current model if vision-capable
99
- if (ctx.model?.input?.includes("image"))
100
- return ctx.model;
141
+ const current = ctx.model;
142
+ if (current?.input?.includes("image")) {
143
+ tried.push(`current model ${modelLabel(current)}: accepts image input`);
144
+ return { model: current, source: "current", tried };
145
+ }
146
+ tried.push(current
147
+ ? `current model ${modelLabel(current)}: ${rejectionReason(current, true)}`
148
+ : "current model: none selected");
101
149
  // 2. The merged role supplies an image-capable entry; a text-only entry in
102
150
  // the list is skipped rather than sent an image.
103
- const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true);
151
+ const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true, tried);
104
152
  if (role)
105
- return role;
153
+ return { model: role, source: "utilityModel", tried };
106
154
  // 3. First available vision model
107
155
  const available = ctx.modelRegistry.getAvailable();
108
- return available.find((m) => m?.input?.includes("image"));
156
+ const imageCapable = available.filter((m) => m?.input?.includes("image"));
157
+ tried.push(`available models: ${available.length}, ${imageCapable.length} accept image input`);
158
+ const fallback = imageCapable[0];
159
+ if (fallback) {
160
+ tried.push(`using ${modelLabel(fallback)}: first available model that accepts images`);
161
+ return { model: fallback, source: "available", tried };
162
+ }
163
+ return { source: null, tried };
164
+ }
165
+ /**
166
+ * The model a vision call should use: the `model` of the walk above.
167
+ */
168
+ export function resolveVisionModel(ctx) {
169
+ return searchVisionModel(ctx).model;
109
170
  }
110
171
  /**
111
172
  * Get the configured utility model for classification tasks.
@@ -8,7 +8,10 @@
8
8
  */
9
9
  export { classify } from "./ai/text-classifier.js";
10
10
  export type { ClassifierCriteria, ClassificationResult, ClassifierOptions, } from "./ai/text-classifier.js";
11
- export { resolveVisionModel } from "./ai/model-resolver.js";
11
+ export { resolveVisionModel, searchVisionModel } from "./ai/model-resolver.js";
12
+ export type { VisionModelSearch } from "./ai/model-resolver.js";
13
+ export { callVision } from "./ai/llm-call.js";
14
+ export type { LlmCallOptions, LlmCallResult, LlmImagePart } from "./ai/llm-call.js";
12
15
  export { healthcheckDir, HEALTHCHECK_DIR } from "./config/braid-paths.js";
13
16
  export { onEvent } from "./events/event-types.js";
14
17
  export { getExtensionState } from "./state/extension-state.js";
@@ -7,7 +7,8 @@
7
7
  * `./lib`.
8
8
  */
9
9
  export { classify } from "./ai/text-classifier.js";
10
- export { resolveVisionModel } from "./ai/model-resolver.js";
10
+ export { resolveVisionModel, searchVisionModel } from "./ai/model-resolver.js";
11
+ export { callVision } from "./ai/llm-call.js";
11
12
  export { healthcheckDir, HEALTHCHECK_DIR } from "./config/braid-paths.js";
12
13
  export { onEvent } from "./events/event-types.js";
13
14
  export { getExtensionState } from "./state/extension-state.js";
package/docs/LOCAL_DEV.md CHANGED
@@ -138,7 +138,7 @@ before and after a full run. A non-zero delta is the leak this rule prevents.
138
138
  ### New fixtures: use `node:test`
139
139
 
140
140
  A new fixture is written on `node:test`. The runner does not care which style a
141
- file uses - 121 of the core suite's 357 fixtures already use `node:test` - so the
141
+ file uses - 122 of the core suite's 358 fixtures already use `node:test` - so the
142
142
  reason is the failure the other style produces. A hand-rolled helper that calls
143
143
  `fn()` without awaiting it discards the case's promise, so an `async` case runs
144
144
  detached and the file can print its green tally before that case settles, with
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pi-braid",
3
- "version": "0.43.58",
3
+ "version": "0.44.0",
4
4
  "description": "🥨 Braid - Metacognitive extension suite for pi coding agent: Adds memory, plan tracking, structured cognitive signals and completion checks on top of pi core loop, so long sessions keep their context and stop claiming unfinished work.",
5
5
  "type": "module",
6
6
  "main": "./dist/index.js",
@@ -28,6 +28,11 @@
28
28
  * callLlm(model, apiKey, systemPrompt, userPrompt, options)
29
29
  * Returns { text } on success or { error } on failure (discriminated union).
30
30
  * const text = result.text;
31
+ *
32
+ * callVision(model, apiKey, systemPrompt, instruction, images, options)
33
+ * The same union with image parts attached to the user message. A model
34
+ * that declares no image input is refused before anything is sent, so a
35
+ * text-only model can never quietly answer about an image it never saw.
31
36
  */
32
37
 
33
38
  import type {
@@ -320,6 +325,30 @@ function releaseModelSlot(modelId: string): void {
320
325
 
321
326
  // ── Options ────────────────────────────────────────────────────────────────────
322
327
 
328
+ /**
329
+ * One image attached to an auxiliary request: base64 bytes and the MIME type
330
+ * they carry. `data` is the payload itself, without a `data:` prefix.
331
+ */
332
+ export interface LlmImagePart {
333
+ /** Base64 payload. */
334
+ data: string;
335
+ /** Declared MIME type, e.g. "image/png". The transport passes it through. */
336
+ mimeType: string;
337
+ }
338
+
339
+ /**
340
+ * What an auxiliary call returns: the text (with the stop reason and usage the
341
+ * provider reported) or one line naming why it failed. A vision call returns
342
+ * the same shape, so a caller never branches on which kind of call it made.
343
+ */
344
+ export type LlmCallResult =
345
+ | {
346
+ text: string;
347
+ stopReason?: string;
348
+ usage?: { input?: number; output?: number; totalTokens?: number };
349
+ }
350
+ | { error: string };
351
+
323
352
  export interface LlmCallOptions {
324
353
  /** Temperature for generation. Default: 0.1 */
325
354
  temperature?: number;
@@ -357,6 +386,16 @@ export interface LlmCallOptions {
357
386
  sdkRetries?: number;
358
387
  /** Optional AbortSignal for cancellation */
359
388
  signal?: AbortSignal;
389
+ /**
390
+ * Image parts attached to the user message. `callVision` is the entry point
391
+ * that sets this: a text call leaves it out and its user content stays a
392
+ * plain string, so the request shape of every existing caller is unchanged.
393
+ *
394
+ * The model's declared input is checked here as well as in `callVision`,
395
+ * because the transport would otherwise drop the image silently or return a
396
+ * provider error that reads like a transient failure.
397
+ */
398
+ images?: readonly LlmImagePart[];
360
399
  }
361
400
 
362
401
  // ── Public API ─────────────────────────────────────────────────────────────────
@@ -379,14 +418,7 @@ export async function callLlm(
379
418
  systemPrompt: string,
380
419
  userPrompt: string,
381
420
  options: LlmCallOptions = {},
382
- ): Promise<
383
- | {
384
- text: string;
385
- stopReason?: string;
386
- usage?: { input?: number; output?: number; totalTokens?: number };
387
- }
388
- | { error: string }
389
- > {
421
+ ): Promise<LlmCallResult> {
390
422
  // Circuit breaker: skip while a recent 429 cooldown is active ───────
391
423
  const now = Date.now();
392
424
  if (rateLimitCooldownUntil > 0 && now < rateLimitCooldownUntil) {
@@ -395,6 +427,13 @@ export async function callLlm(
395
427
  return { error: "Rate-limit cooldown active" };
396
428
  }
397
429
 
430
+ // Images: a request the model cannot read is refused before it is built,
431
+ // not sent and reported as a provider error (or, worse, answered from the
432
+ // text alone with no complaint about the missing picture).
433
+ const images = options.images ?? [];
434
+ const imageRefusal = refuseImages(model, images);
435
+ if (imageRefusal !== null) return { error: imageRefusal };
436
+
398
437
  const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
399
438
 
400
439
  const maxTokens =
@@ -424,7 +463,9 @@ export async function callLlm(
424
463
 
425
464
  const context: Context = {
426
465
  systemPrompt: systemSafe.text,
427
- messages: [{ role: "user", content: userSafe.text, timestamp: Date.now() }],
466
+ messages: [
467
+ { role: "user", content: userContent(userSafe.text, images), timestamp: Date.now() },
468
+ ],
428
469
  };
429
470
 
430
471
  // ── pre-send budget ──
@@ -439,7 +480,8 @@ export async function callLlm(
439
480
  measure: () =>
440
481
  calibrateEstimate(
441
482
  estimatePayloadTokens({ systemPrompt: context.systemPrompt, messages: context.messages }),
442
- ).tokens,
483
+ ).tokens +
484
+ images.length * IMAGE_TOKEN_ALLOWANCE,
443
485
  outputTokens: maxTokens,
444
486
  contextWindow: resolvedWindow.contextWindow,
445
487
  // The only compaction available to an auxiliary call is the published
@@ -763,8 +805,115 @@ export async function callLlm(
763
805
  }
764
806
  }
765
807
 
808
+ // ── Vision ────────────────────────────────────────────────────────────────────
809
+
810
+ /**
811
+ * Make an auxiliary call that carries images: the same dispatch, retry ladder,
812
+ * circuit breaker, pre-send budget and key-ring feedback as `callLlm`, with the
813
+ * images attached to the user message.
814
+ *
815
+ * Why this is a wrapper and not a second dispatcher: every policy the text
816
+ * path carries (per-model concurrency, transient retry with the backoff
817
+ * ceiling, the 413 emergency offload, the listener and key-ring seams) applies
818
+ * to a vision call exactly as it applies to a text one. A copy would drift from
819
+ * that policy at the first fix made in one place, and a vision call that does
820
+ * not feed the key ring back is a route that keeps failing without rotating.
821
+ * The image-specific decisions - is this model image-capable, is this part a
822
+ * real image - are the only ones the wrapper owns.
823
+ *
824
+ * `instruction` is the user turn; `images` are attached after it, in order, so
825
+ * a caller can refer to "the first image" and mean what it sent.
826
+ */
827
+ export async function callVision(
828
+ model: Model<Api>,
829
+ apiKey: string | undefined,
830
+ systemPrompt: string,
831
+ instruction: string,
832
+ images: readonly LlmImagePart[],
833
+ options: Omit<LlmCallOptions, "images"> = {},
834
+ ): Promise<LlmCallResult> {
835
+ if (images.length === 0) {
836
+ return {
837
+ error:
838
+ "Image input refused: a vision call needs at least one image part. " +
839
+ "Use callLlm for a text-only request.",
840
+ };
841
+ }
842
+ return callLlm(model, apiKey, systemPrompt, instruction, { ...options, images });
843
+ }
844
+
766
845
  // ── Helpers ────────────────────────────────────────────────────────────────────
767
846
 
847
+ /**
848
+ * Tokens the pre-send budget counts per attached image.
849
+ *
850
+ * The payload estimator reads text only: an image block contributes its
851
+ * `mimeType` and nothing of its `data` (see estimateMessageTokens in
852
+ * detection/context-usage.ts), so without this allowance an image-heavy
853
+ * request looks free and the guard waves through a payload the provider then
854
+ * rejects. An image is priced by its pixels, not its bytes, and a full-page
855
+ * capture is a few thousand tokens on every provider that accepts one: OpenAI
856
+ * at high detail tiles a 1440x3000 capture into 18 tiles (~3.2K tokens),
857
+ * Anthropic's pixel rule puts the same image near 1.6K after its own downscale.
858
+ * This constant is the conservative ceiling of that range, so the estimate errs
859
+ * toward refusing rather than toward a doomed send.
860
+ */
861
+ const IMAGE_TOKEN_ALLOWANCE = 4_000;
862
+
863
+ /**
864
+ * The one place that decides whether a request carrying images may be built.
865
+ * Null means yes. A refusal names the model, what it declares, and the offending
866
+ * part, because the caller's next move (pick another model, re-encode the
867
+ * image) depends on which of those failed.
868
+ */
869
+ function refuseImages(model: Model<Api>, images: readonly LlmImagePart[]): string | null {
870
+ if (images.length === 0) return null;
871
+ const declared = model.input ?? [];
872
+ if (!declared.includes("image")) {
873
+ return (
874
+ `Image input refused: ${model.provider ?? "unknown"}/${model.id} declares input ` +
875
+ `[${declared.join(", ") || "text"}] and does not accept images. A vision call needs an ` +
876
+ `image-capable model; resolveVisionModel names one for the current session.`
877
+ );
878
+ }
879
+ for (let index = 0; index < images.length; index++) {
880
+ const image = images[index];
881
+ const position = `Image part ${index + 1} of ${images.length}`;
882
+ if (typeof image.data !== "string" || image.data.length === 0) {
883
+ return `${position} carries no data. Expected base64 image bytes, not an empty string.`;
884
+ }
885
+ if (typeof image.mimeType !== "string" || !/^image\/[a-z0-9.+-]+$/i.test(image.mimeType)) {
886
+ return (
887
+ `${position} declares no image MIME type (got ${JSON.stringify(image.mimeType)}). ` +
888
+ `Expected a type such as "image/png".`
889
+ );
890
+ }
891
+ }
892
+ return null;
893
+ }
894
+
895
+ /**
896
+ * The user message content: a plain string for a text call, text-then-images for
897
+ * a vision call. The text block comes first so an instruction that says "the
898
+ * image below" reads in the order it was written.
899
+ */
900
+ function userContent(
901
+ text: string,
902
+ images: readonly LlmImagePart[],
903
+ ):
904
+ | string
905
+ | Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
906
+ if (images.length === 0) return text;
907
+ return [
908
+ { type: "text", text },
909
+ ...images.map((image) => ({
910
+ type: "image" as const,
911
+ data: image.data,
912
+ mimeType: image.mimeType,
913
+ })),
914
+ ];
915
+ }
916
+
768
917
  /**
769
918
  * Run the published emergency offloader (context-manager's
770
919
  * context-emergency.ts) against the current session context. Lossy-safe:
@@ -47,6 +47,27 @@ export function normaliseRoleSetting(raw: unknown): string[] {
47
47
  return patterns;
48
48
  }
49
49
 
50
+ /**
51
+ * A model named the way a report names it: provider then id, the pair a reader
52
+ * can paste into a model flag.
53
+ */
54
+ function modelLabel(model: Model<Api>): string {
55
+ return `${model.provider ?? "unknown"}/${model.id}`;
56
+ }
57
+
58
+ /**
59
+ * Why a resolved model was not accepted for a role, in words, for a caller that
60
+ * has to explain an empty result. Null when the model is acceptable.
61
+ */
62
+ function rejectionReason(model: Model<Api> | undefined, requireImage: boolean): string | null {
63
+ if (!model) return "no provider serves this pattern";
64
+ if (requireImage && !model.input?.includes("image")) {
65
+ const declared = model.input ?? [];
66
+ return `declares input [${declared.join(", ") || "text"}], no image input`;
67
+ }
68
+ return null;
69
+ }
70
+
50
71
  /**
51
72
  * Resolve a role setting to a single model.
52
73
  *
@@ -60,24 +81,54 @@ export function normaliseRoleSetting(raw: unknown): string[] {
60
81
  *
61
82
  * @param requireImage - vision role only: entries must accept image input, so
62
83
  * a list may hold a text-only entry without an image being sent to it.
84
+ * @param trace - optional sink for one line per entry, in the order tried, so a
85
+ * caller that gets nothing back can print what was tried and why each entry
86
+ * was passed over. Passed in rather than returned because the successful path
87
+ * must stay allocation-free for the classification callers.
63
88
  */
64
89
  function resolveRoleModel(
65
90
  ctx: ExtensionContext,
66
91
  key: string,
67
92
  requireImage = false,
93
+ trace?: string[],
68
94
  ): Model<Api> | undefined {
69
95
  const raw: unknown = getCustomModelKey(key);
70
96
  const patterns = normaliseRoleSetting(raw);
71
- if (patterns.length === 0) return undefined;
97
+ if (patterns.length === 0) {
98
+ trace?.push(`${key}: not configured`);
99
+ return undefined;
100
+ }
72
101
 
73
102
  const acceptable = (model: Model<Api> | undefined): Model<Api> | undefined =>
74
103
  model && (!requireImage || model.input?.includes("image")) ? model : undefined;
75
104
 
76
- if (!Array.isArray(raw)) return acceptable(resolveFromPattern(ctx, patterns[0]));
105
+ if (!Array.isArray(raw)) {
106
+ const resolved = resolveFromPattern(ctx, patterns[0]);
107
+ const model = acceptable(resolved);
108
+ if (!trace) return model;
109
+ const reason = rejectionReason(resolved, requireImage);
110
+ trace.push(
111
+ model && reason === null
112
+ ? `${key} "${patterns[0]}": using ${modelLabel(model)}, accepts image input`
113
+ : `${key} "${patterns[0]}": ${reason}`,
114
+ );
115
+ return model;
116
+ }
77
117
 
78
118
  for (const pattern of patterns) {
79
- const model = acceptable(resolveFromPattern(ctx, pattern));
80
- if (model && isModelRouteUsable(model)) return model;
119
+ const resolved = resolveFromPattern(ctx, pattern);
120
+ const model = acceptable(resolved);
121
+ if (model && isModelRouteUsable(model)) {
122
+ trace?.push(
123
+ `${key} "${pattern}": using ${modelLabel(model)}, the provider reports the route usable`,
124
+ );
125
+ return model;
126
+ }
127
+ trace?.push(
128
+ model
129
+ ? `${key} "${pattern}": resolved ${modelLabel(model)}, but the provider reports the route unusable`
130
+ : `${key} "${pattern}": ${rejectionReason(resolved, requireImage)}`,
131
+ );
81
132
  }
82
133
  return undefined;
83
134
  }
@@ -95,7 +146,24 @@ export function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[] {
95
146
  }
96
147
 
97
148
  /**
98
- * The model a vision call should use.
149
+ * What a vision-model resolution found and what it passed over.
150
+ *
151
+ * `resolveVisionModel` answers "which model"; a caller that reports to a human
152
+ * also has to answer "and then what should I change", which needs the rungs
153
+ * that produced nothing. Both come from one walk, so the model a report names
154
+ * and the trace beside it cannot disagree.
155
+ */
156
+ export interface VisionModelSearch {
157
+ /** The model a vision call should use, or undefined when the walk found none. */
158
+ model?: Model<Api>;
159
+ /** Which rung supplied `model`; null when none did. */
160
+ source: "current" | "utilityModel" | "available" | null;
161
+ /** One line per rung, in the order tried, saying what it found or why not. */
162
+ tried: string[];
163
+ }
164
+
165
+ /**
166
+ * The model a vision call should use, with the walk that produced it.
99
167
  *
100
168
  * Priority:
101
169
  * 1. The current model when it accepts images: the image stays in the call
@@ -106,18 +174,43 @@ export function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[] {
106
174
  *
107
175
  * A text-only entry is never selected for an image call.
108
176
  */
109
- export function resolveVisionModel(ctx: ExtensionContext): Model<Api> | undefined {
177
+ export function searchVisionModel(ctx: ExtensionContext): VisionModelSearch {
178
+ const tried: string[] = [];
179
+
110
180
  // 1. Current model if vision-capable
111
- if (ctx.model?.input?.includes("image")) return ctx.model;
181
+ const current = ctx.model;
182
+ if (current?.input?.includes("image")) {
183
+ tried.push(`current model ${modelLabel(current)}: accepts image input`);
184
+ return { model: current, source: "current", tried };
185
+ }
186
+ tried.push(
187
+ current
188
+ ? `current model ${modelLabel(current)}: ${rejectionReason(current, true)}`
189
+ : "current model: none selected",
190
+ );
112
191
 
113
192
  // 2. The merged role supplies an image-capable entry; a text-only entry in
114
193
  // the list is skipped rather than sent an image.
115
- const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true);
116
- if (role) return role;
194
+ const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true, tried);
195
+ if (role) return { model: role, source: "utilityModel", tried };
117
196
 
118
197
  // 3. First available vision model
119
198
  const available = ctx.modelRegistry.getAvailable();
120
- return available.find((m) => m?.input?.includes("image"));
199
+ const imageCapable = available.filter((m) => m?.input?.includes("image"));
200
+ tried.push(`available models: ${available.length}, ${imageCapable.length} accept image input`);
201
+ const fallback = imageCapable[0];
202
+ if (fallback) {
203
+ tried.push(`using ${modelLabel(fallback)}: first available model that accepts images`);
204
+ return { model: fallback, source: "available", tried };
205
+ }
206
+ return { source: null, tried };
207
+ }
208
+
209
+ /**
210
+ * The model a vision call should use: the `model` of the walk above.
211
+ */
212
+ export function resolveVisionModel(ctx: ExtensionContext): Model<Api> | undefined {
213
+ return searchVisionModel(ctx).model;
121
214
  }
122
215
 
123
216
  /**
package/src/lib/public.ts CHANGED
@@ -13,7 +13,10 @@ export type {
13
13
  ClassificationResult,
14
14
  ClassifierOptions,
15
15
  } from "./ai/text-classifier.js";
16
- export { resolveVisionModel } from "./ai/model-resolver.js";
16
+ export { resolveVisionModel, searchVisionModel } from "./ai/model-resolver.js";
17
+ export type { VisionModelSearch } from "./ai/model-resolver.js";
18
+ export { callVision } from "./ai/llm-call.js";
19
+ export type { LlmCallOptions, LlmCallResult, LlmImagePart } from "./ai/llm-call.js";
17
20
 
18
21
  export { healthcheckDir, HEALTHCHECK_DIR } from "./config/braid-paths.js";
19
22