pi-braid 0.43.58 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/build-info.json +4 -4
- package/dist/lib/ai/llm-call.d.ts +60 -11
- package/dist/lib/ai/llm-call.js +104 -2
- package/dist/lib/ai/model-resolver.d.ts +21 -1
- package/dist/lib/ai/model-resolver.js +74 -13
- package/dist/lib/public.d.ts +4 -1
- package/dist/lib/public.js +2 -1
- package/docs/LOCAL_DEV.md +1 -1
- package/package.json +1 -1
- package/src/lib/ai/llm-call.ts +159 -10
- package/src/lib/ai/model-resolver.ts +103 -10
- package/src/lib/public.ts +4 -1
package/CHANGELOG.md
CHANGED
|
@@ -96,6 +96,20 @@ Adding a thirteenth extension moves the counts six source claims pin to the mani
|
|
|
96
96
|
schema's recorded line for the `braid.contrib.*` reader; `check:doc-drift` and `check:schema` demanded
|
|
97
97
|
both repairs in the same commit.
|
|
98
98
|
|
|
99
|
+
## Release — a missing version is absence, not a failure (2026-11-12)
|
|
100
|
+
|
|
101
|
+
The registry gate failed the first release it ran on, and it was right to be looked at: a version that had
|
|
102
|
+
just been published was not yet served, `npm pack` answered `ETARGET` rather than `E404`, the read counted
|
|
103
|
+
as unanswerable, and the verify step exited 2 while the publish itself had succeeded. Absence now covers both
|
|
104
|
+
codes npm uses, and only a read that answers neither is unanswerable, with the three meanings pinned by
|
|
105
|
+
tests that carry the real `ETARGET` body.
|
|
106
|
+
|
|
107
|
+
The fix exposed a second defect on the way in: the script ran its CLI on import, so the test that imports it
|
|
108
|
+
to reach one pure function packed every workspace, `npm pack` ran this package's `prepack`, the prepack
|
|
109
|
+
rebuilt `dist`, and for the fourteen seconds of that rebuild every fixture in the pooled run that imports
|
|
110
|
+
`dist` failed with a module-not-found - nine files, in a run where the same commit passed in a clean
|
|
111
|
+
checkout. The CLI now runs only when the file is the program.
|
|
112
|
+
|
|
99
113
|
## Skills — one design loop, and a gate that refuses a private literal (2026-11-12)
|
|
100
114
|
|
|
101
115
|
Two changes in one release, both about a check that did not exist.
|
|
@@ -186,6 +200,32 @@ tarball did not have. All three move here, with the fixes they already hold:
|
|
|
186
200
|
instead of throwing out of a timer and ending the process.
|
|
187
201
|
- `pi-braid-contrib` 0.1.5: the git tool, argv-only and read-only by default.
|
|
188
202
|
|
|
203
|
+
## Core 0.44.0, Contrib 0.2.0, Providers 1.0.8 — the ruler gains eyes, and refusals learn to speak (2026-11-12)
|
|
204
|
+
|
|
205
|
+
`design_audit` could measure a rendered page and say nothing about whether it looked cheap. The same
|
|
206
|
+
adapter's full-page PNGs now go to the model the session resolves for vision, one call per width, in
|
|
207
|
+
`visual_judge` beside the audit. The result names the model that answered, its route, the tokens and the
|
|
208
|
+
elapsed time, because an opinion with no author is not a report; the default rubric asks for named
|
|
209
|
+
observations and forbids scores, since a judge that answers 7/10 has measured nothing.
|
|
210
|
+
|
|
211
|
+
The generic half stayed in `lib/ai`, where the retry ladder, the pre-send budget and the key-ring
|
|
212
|
+
feedback already live. `callLlm` gained an optional image part and `callVision` is the named entry that
|
|
213
|
+
refuses a text-only model or a malformed part before a request is built, so an image is priced at the
|
|
214
|
+
budget where the text estimator had counted it as free. `resolveVisionModel` and `searchVisionModel`
|
|
215
|
+
share one walk, so a refusal prints every rung it passed over and why it was skipped.
|
|
216
|
+
|
|
217
|
+
A test-harness defect fell out on the way, and it was the kind that hides: contrib's fixtures import core as
|
|
218
|
+
`pi-braid/lib`, which in a checkout resolves through `node_modules` to whichever pi-braid is installed
|
|
219
|
+
rather than the tree under test, so a core export added in the same tree was invisible and the failure read
|
|
220
|
+
as a broken import. The shared hook maps that specifier to this tree's build, and an arm compares the
|
|
221
|
+
build's mtime against the newest core source, because a hook that reads an artifact can otherwise report a
|
|
222
|
+
green suite about code the tree no longer contains. Touching a source without rebuilding reddens it;
|
|
223
|
+
removing the export from the artifact reddens it naming the export; reverting the image part fails one arm
|
|
224
|
+
of twelve.
|
|
225
|
+
|
|
226
|
+
Contrib moves a minor because a new tool is something a caller notices, core because `pi-braid/lib` gained
|
|
227
|
+
five names; providers carries the rotation census at 1.0.8 and is unchanged here.
|
|
228
|
+
|
|
189
229
|
## Core — a prefix is ranked, never qualifying on its own (2026-11-11)
|
|
190
230
|
|
|
191
231
|
The same defect as the class noun, one leg over: `improve` starts `improvement`, so the token-prefix leg
|
package/build-info.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-braid",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"sha": "
|
|
5
|
-
"shaShort": "
|
|
3
|
+
"version": "0.44.0",
|
|
4
|
+
"sha": "f0165737cba18bd9ce624ec1a4bd13f47494b7aa",
|
|
5
|
+
"shaShort": "f0165737",
|
|
6
6
|
"branch": "detached",
|
|
7
7
|
"dirty": false,
|
|
8
|
-
"packedAt": "2026-11-
|
|
8
|
+
"packedAt": "2026-11-12T23:32:20.399Z"
|
|
9
9
|
}
|
|
@@ -28,6 +28,11 @@
|
|
|
28
28
|
* callLlm(model, apiKey, systemPrompt, userPrompt, options)
|
|
29
29
|
* Returns { text } on success or { error } on failure (discriminated union).
|
|
30
30
|
* const text = result.text;
|
|
31
|
+
*
|
|
32
|
+
* callVision(model, apiKey, systemPrompt, instruction, images, options)
|
|
33
|
+
* The same union with image parts attached to the user message. A model
|
|
34
|
+
* that declares no image input is refused before anything is sent, so a
|
|
35
|
+
* text-only model can never quietly answer about an image it never saw.
|
|
31
36
|
*/
|
|
32
37
|
import type { Api, AssistantMessage, Context, Model, ModelThinkingLevel, ModelsSimpleStreamOptions } from "@earendil-works/pi-ai";
|
|
33
38
|
/** One provider-neutral completion call. */
|
|
@@ -48,6 +53,32 @@ export declare function __resetRateLimitCooldownForTest(): void;
|
|
|
48
53
|
import { setLlmCallListeners, type LlmErrorListener, type LlmSuccessListener } from "../state/global-state.js";
|
|
49
54
|
export { setLlmCallListeners };
|
|
50
55
|
export type { LlmErrorListener, LlmSuccessListener };
|
|
56
|
+
/**
|
|
57
|
+
* One image attached to an auxiliary request: base64 bytes and the MIME type
|
|
58
|
+
* they carry. `data` is the payload itself, without a `data:` prefix.
|
|
59
|
+
*/
|
|
60
|
+
export interface LlmImagePart {
|
|
61
|
+
/** Base64 payload. */
|
|
62
|
+
data: string;
|
|
63
|
+
/** Declared MIME type, e.g. "image/png". The transport passes it through. */
|
|
64
|
+
mimeType: string;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* What an auxiliary call returns: the text (with the stop reason and usage the
|
|
68
|
+
* provider reported) or one line naming why it failed. A vision call returns
|
|
69
|
+
* the same shape, so a caller never branches on which kind of call it made.
|
|
70
|
+
*/
|
|
71
|
+
export type LlmCallResult = {
|
|
72
|
+
text: string;
|
|
73
|
+
stopReason?: string;
|
|
74
|
+
usage?: {
|
|
75
|
+
input?: number;
|
|
76
|
+
output?: number;
|
|
77
|
+
totalTokens?: number;
|
|
78
|
+
};
|
|
79
|
+
} | {
|
|
80
|
+
error: string;
|
|
81
|
+
};
|
|
51
82
|
export interface LlmCallOptions {
|
|
52
83
|
/** Temperature for generation. Default: 0.1 */
|
|
53
84
|
temperature?: number;
|
|
@@ -85,6 +116,16 @@ export interface LlmCallOptions {
|
|
|
85
116
|
sdkRetries?: number;
|
|
86
117
|
/** Optional AbortSignal for cancellation */
|
|
87
118
|
signal?: AbortSignal;
|
|
119
|
+
/**
|
|
120
|
+
* Image parts attached to the user message. `callVision` is the entry point
|
|
121
|
+
* that sets this: a text call leaves it out and its user content stays a
|
|
122
|
+
* plain string, so the request shape of every existing caller is unchanged.
|
|
123
|
+
*
|
|
124
|
+
* The model's declared input is checked here as well as in `callVision`,
|
|
125
|
+
* because the transport would otherwise drop the image silently or return a
|
|
126
|
+
* provider error that reads like a transient failure.
|
|
127
|
+
*/
|
|
128
|
+
images?: readonly LlmImagePart[];
|
|
88
129
|
}
|
|
89
130
|
/**
|
|
90
131
|
* Make an LLM call with two-tier retry, backoff, and rate-limit circuit breaker.
|
|
@@ -98,17 +139,25 @@ export interface LlmCallOptions {
|
|
|
98
139
|
* @param options - Optional configuration
|
|
99
140
|
* @returns `{ text }` on success or `{ error }` on failure
|
|
100
141
|
*/
|
|
101
|
-
export declare function callLlm(model: Model<Api>, apiKey: string | undefined, systemPrompt: string, userPrompt: string, options?: LlmCallOptions): Promise<
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
142
|
+
export declare function callLlm(model: Model<Api>, apiKey: string | undefined, systemPrompt: string, userPrompt: string, options?: LlmCallOptions): Promise<LlmCallResult>;
|
|
143
|
+
/**
|
|
144
|
+
* Make an auxiliary call that carries images: the same dispatch, retry ladder,
|
|
145
|
+
* circuit breaker, pre-send budget and key-ring feedback as `callLlm`, with the
|
|
146
|
+
* images attached to the user message.
|
|
147
|
+
*
|
|
148
|
+
* Why this is a wrapper and not a second dispatcher: every policy the text
|
|
149
|
+
* path carries (per-model concurrency, transient retry with the backoff
|
|
150
|
+
* ceiling, the 413 emergency offload, the listener and key-ring seams) applies
|
|
151
|
+
* to a vision call exactly as it applies to a text one. A copy would drift from
|
|
152
|
+
* that policy at the first fix made in one place, and a vision call that does
|
|
153
|
+
* not feed the key ring back is a route that keeps failing without rotating.
|
|
154
|
+
* The image-specific decisions - is this model image-capable, is this part a
|
|
155
|
+
* real image - are the only ones the wrapper owns.
|
|
156
|
+
*
|
|
157
|
+
* `instruction` is the user turn; `images` are attached after it, in order, so
|
|
158
|
+
* a caller can refer to "the first image" and mean what it sent.
|
|
159
|
+
*/
|
|
160
|
+
export declare function callVision(model: Model<Api>, apiKey: string | undefined, systemPrompt: string, instruction: string, images: readonly LlmImagePart[], options?: Omit<LlmCallOptions, "images">): Promise<LlmCallResult>;
|
|
112
161
|
/**
|
|
113
162
|
* The one yes/no question core asks the providers seam before re-using a
|
|
114
163
|
* (provider, model) pair for an auxiliary call: can this pair serve a request
|
package/dist/lib/ai/llm-call.js
CHANGED
|
@@ -28,6 +28,11 @@
|
|
|
28
28
|
* callLlm(model, apiKey, systemPrompt, userPrompt, options)
|
|
29
29
|
* Returns { text } on success or { error } on failure (discriminated union).
|
|
30
30
|
* const text = result.text;
|
|
31
|
+
*
|
|
32
|
+
* callVision(model, apiKey, systemPrompt, instruction, images, options)
|
|
33
|
+
* The same union with image parts attached to the user message. A model
|
|
34
|
+
* that declares no image input is refused before anything is sent, so a
|
|
35
|
+
* text-only model can never quietly answer about an image it never saw.
|
|
31
36
|
*/
|
|
32
37
|
// Degraded transport only: see completeAuxCall below. Compat resolves the
|
|
33
38
|
// credential from the model and the apiKey argument and nothing else, which is
|
|
@@ -231,6 +236,13 @@ export async function callLlm(model, apiKey, systemPrompt, userPrompt, options =
|
|
|
231
236
|
log.debug(`Skipping LLM call - rate-limit cooldown ${remaining}ms remaining`);
|
|
232
237
|
return { error: "Rate-limit cooldown active" };
|
|
233
238
|
}
|
|
239
|
+
// Images: a request the model cannot read is refused before it is built,
|
|
240
|
+
// not sent and reported as a provider error (or, worse, answered from the
|
|
241
|
+
// text alone with no complaint about the missing picture).
|
|
242
|
+
const images = options.images ?? [];
|
|
243
|
+
const imageRefusal = refuseImages(model, images);
|
|
244
|
+
if (imageRefusal !== null)
|
|
245
|
+
return { error: imageRefusal };
|
|
234
246
|
const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
235
247
|
const maxTokens = options.maxTokens ?? Math.min(Math.max(1000, Math.floor(userPrompt.length / 8)), 4000);
|
|
236
248
|
// 10-minute default (was 30s). Aborts should be signal-driven (ESC,
|
|
@@ -254,7 +266,9 @@ export async function callLlm(model, apiKey, systemPrompt, userPrompt, options =
|
|
|
254
266
|
}
|
|
255
267
|
const context = {
|
|
256
268
|
systemPrompt: systemSafe.text,
|
|
257
|
-
messages: [
|
|
269
|
+
messages: [
|
|
270
|
+
{ role: "user", content: userContent(userSafe.text, images), timestamp: Date.now() },
|
|
271
|
+
],
|
|
258
272
|
};
|
|
259
273
|
// ── pre-send budget ──
|
|
260
274
|
// A request over the window cannot succeed, and a provider 400 is a late,
|
|
@@ -265,7 +279,8 @@ export async function callLlm(model, apiKey, systemPrompt, userPrompt, options =
|
|
|
265
279
|
// provider will refuse what the cap already refused.
|
|
266
280
|
const resolvedWindow = resolveWindow(model.contextWindow);
|
|
267
281
|
const budget = await guardAgainstDoomedSend({
|
|
268
|
-
measure: () => calibrateEstimate(estimatePayloadTokens({ systemPrompt: context.systemPrompt, messages: context.messages })).tokens
|
|
282
|
+
measure: () => calibrateEstimate(estimatePayloadTokens({ systemPrompt: context.systemPrompt, messages: context.messages })).tokens +
|
|
283
|
+
images.length * IMAGE_TOKEN_ALLOWANCE,
|
|
269
284
|
outputTokens: maxTokens,
|
|
270
285
|
contextWindow: resolvedWindow.contextWindow,
|
|
271
286
|
// The only compaction available to an auxiliary call is the published
|
|
@@ -544,7 +559,94 @@ export async function callLlm(model, apiKey, systemPrompt, userPrompt, options =
|
|
|
544
559
|
releaseModelSlot(model.id);
|
|
545
560
|
}
|
|
546
561
|
}
|
|
562
|
+
// ── Vision ────────────────────────────────────────────────────────────────────
|
|
563
|
+
/**
|
|
564
|
+
* Make an auxiliary call that carries images: the same dispatch, retry ladder,
|
|
565
|
+
* circuit breaker, pre-send budget and key-ring feedback as `callLlm`, with the
|
|
566
|
+
* images attached to the user message.
|
|
567
|
+
*
|
|
568
|
+
* Why this is a wrapper and not a second dispatcher: every policy the text
|
|
569
|
+
* path carries (per-model concurrency, transient retry with the backoff
|
|
570
|
+
* ceiling, the 413 emergency offload, the listener and key-ring seams) applies
|
|
571
|
+
* to a vision call exactly as it applies to a text one. A copy would drift from
|
|
572
|
+
* that policy at the first fix made in one place, and a vision call that does
|
|
573
|
+
* not feed the key ring back is a route that keeps failing without rotating.
|
|
574
|
+
* The image-specific decisions - is this model image-capable, is this part a
|
|
575
|
+
* real image - are the only ones the wrapper owns.
|
|
576
|
+
*
|
|
577
|
+
* `instruction` is the user turn; `images` are attached after it, in order, so
|
|
578
|
+
* a caller can refer to "the first image" and mean what it sent.
|
|
579
|
+
*/
|
|
580
|
+
export async function callVision(model, apiKey, systemPrompt, instruction, images, options = {}) {
|
|
581
|
+
if (images.length === 0) {
|
|
582
|
+
return {
|
|
583
|
+
error: "Image input refused: a vision call needs at least one image part. " +
|
|
584
|
+
"Use callLlm for a text-only request.",
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
return callLlm(model, apiKey, systemPrompt, instruction, { ...options, images });
|
|
588
|
+
}
|
|
547
589
|
// ── Helpers ────────────────────────────────────────────────────────────────────
|
|
590
|
+
/**
|
|
591
|
+
* Tokens the pre-send budget counts per attached image.
|
|
592
|
+
*
|
|
593
|
+
* The payload estimator reads text only: an image block contributes its
|
|
594
|
+
* `mimeType` and nothing of its `data` (see estimateMessageTokens in
|
|
595
|
+
* detection/context-usage.ts), so without this allowance an image-heavy
|
|
596
|
+
* request looks free and the guard waves through a payload the provider then
|
|
597
|
+
* rejects. An image is priced by its pixels, not its bytes, and a full-page
|
|
598
|
+
* capture is a few thousand tokens on every provider that accepts one: OpenAI
|
|
599
|
+
* at high detail tiles a 1440x3000 capture into 18 tiles (~3.2K tokens),
|
|
600
|
+
* Anthropic's pixel rule puts the same image near 1.6K after its own downscale.
|
|
601
|
+
* This constant is the conservative ceiling of that range, so the estimate errs
|
|
602
|
+
* toward refusing rather than toward a doomed send.
|
|
603
|
+
*/
|
|
604
|
+
const IMAGE_TOKEN_ALLOWANCE = 4_000;
|
|
605
|
+
/**
|
|
606
|
+
* The one place that decides whether a request carrying images may be built.
|
|
607
|
+
* Null means yes. A refusal names the model, what it declares, and the offending
|
|
608
|
+
* part, because the caller's next move (pick another model, re-encode the
|
|
609
|
+
* image) depends on which of those failed.
|
|
610
|
+
*/
|
|
611
|
+
function refuseImages(model, images) {
|
|
612
|
+
if (images.length === 0)
|
|
613
|
+
return null;
|
|
614
|
+
const declared = model.input ?? [];
|
|
615
|
+
if (!declared.includes("image")) {
|
|
616
|
+
return (`Image input refused: ${model.provider ?? "unknown"}/${model.id} declares input ` +
|
|
617
|
+
`[${declared.join(", ") || "text"}] and does not accept images. A vision call needs an ` +
|
|
618
|
+
`image-capable model; resolveVisionModel names one for the current session.`);
|
|
619
|
+
}
|
|
620
|
+
for (let index = 0; index < images.length; index++) {
|
|
621
|
+
const image = images[index];
|
|
622
|
+
const position = `Image part ${index + 1} of ${images.length}`;
|
|
623
|
+
if (typeof image.data !== "string" || image.data.length === 0) {
|
|
624
|
+
return `${position} carries no data. Expected base64 image bytes, not an empty string.`;
|
|
625
|
+
}
|
|
626
|
+
if (typeof image.mimeType !== "string" || !/^image\/[a-z0-9.+-]+$/i.test(image.mimeType)) {
|
|
627
|
+
return (`${position} declares no image MIME type (got ${JSON.stringify(image.mimeType)}). ` +
|
|
628
|
+
`Expected a type such as "image/png".`);
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
return null;
|
|
632
|
+
}
|
|
633
|
+
/**
|
|
634
|
+
* The user message content: a plain string for a text call, text-then-images for
|
|
635
|
+
* a vision call. The text block comes first so an instruction that says "the
|
|
636
|
+
* image below" reads in the order it was written.
|
|
637
|
+
*/
|
|
638
|
+
function userContent(text, images) {
|
|
639
|
+
if (images.length === 0)
|
|
640
|
+
return text;
|
|
641
|
+
return [
|
|
642
|
+
{ type: "text", text },
|
|
643
|
+
...images.map((image) => ({
|
|
644
|
+
type: "image",
|
|
645
|
+
data: image.data,
|
|
646
|
+
mimeType: image.mimeType,
|
|
647
|
+
})),
|
|
648
|
+
];
|
|
649
|
+
}
|
|
548
650
|
/**
|
|
549
651
|
* Run the published emergency offloader (context-manager's
|
|
550
652
|
* context-emergency.ts) against the current session context. Lossy-safe:
|
|
@@ -27,7 +27,23 @@ export declare function normaliseRoleSetting(raw: unknown): string[];
|
|
|
27
27
|
*/
|
|
28
28
|
export declare function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[];
|
|
29
29
|
/**
|
|
30
|
-
*
|
|
30
|
+
* What a vision-model resolution found and what it passed over.
|
|
31
|
+
*
|
|
32
|
+
* `resolveVisionModel` answers "which model"; a caller that reports to a human
|
|
33
|
+
* also has to answer "and then what should I change", which needs the rungs
|
|
34
|
+
* that produced nothing. Both come from one walk, so the model a report names
|
|
35
|
+
* and the trace beside it cannot disagree.
|
|
36
|
+
*/
|
|
37
|
+
export interface VisionModelSearch {
|
|
38
|
+
/** The model a vision call should use, or undefined when the walk found none. */
|
|
39
|
+
model?: Model<Api>;
|
|
40
|
+
/** Which rung supplied `model`; null when none did. */
|
|
41
|
+
source: "current" | "utilityModel" | "available" | null;
|
|
42
|
+
/** One line per rung, in the order tried, saying what it found or why not. */
|
|
43
|
+
tried: string[];
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The model a vision call should use, with the walk that produced it.
|
|
31
47
|
*
|
|
32
48
|
* Priority:
|
|
33
49
|
* 1. The current model when it accepts images: the image stays in the call
|
|
@@ -38,6 +54,10 @@ export declare function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[
|
|
|
38
54
|
*
|
|
39
55
|
* A text-only entry is never selected for an image call.
|
|
40
56
|
*/
|
|
57
|
+
export declare function searchVisionModel(ctx: ExtensionContext): VisionModelSearch;
|
|
58
|
+
/**
|
|
59
|
+
* The model a vision call should use: the `model` of the walk above.
|
|
60
|
+
*/
|
|
41
61
|
export declare function resolveVisionModel(ctx: ExtensionContext): Model<Api> | undefined;
|
|
42
62
|
/**
|
|
43
63
|
* Get the configured utility model for classification tasks.
|
|
@@ -42,6 +42,26 @@ export function normaliseRoleSetting(raw) {
|
|
|
42
42
|
}
|
|
43
43
|
return patterns;
|
|
44
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* A model named the way a report names it: provider then id, the pair a reader
|
|
47
|
+
* can paste into a model flag.
|
|
48
|
+
*/
|
|
49
|
+
function modelLabel(model) {
|
|
50
|
+
return `${model.provider ?? "unknown"}/${model.id}`;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Why a resolved model was not accepted for a role, in words, for a caller that
|
|
54
|
+
* has to explain an empty result. Null when the model is acceptable.
|
|
55
|
+
*/
|
|
56
|
+
function rejectionReason(model, requireImage) {
|
|
57
|
+
if (!model)
|
|
58
|
+
return "no provider serves this pattern";
|
|
59
|
+
if (requireImage && !model.input?.includes("image")) {
|
|
60
|
+
const declared = model.input ?? [];
|
|
61
|
+
return `declares input [${declared.join(", ") || "text"}], no image input`;
|
|
62
|
+
}
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
45
65
|
/**
|
|
46
66
|
* Resolve a role setting to a single model.
|
|
47
67
|
*
|
|
@@ -55,19 +75,40 @@ export function normaliseRoleSetting(raw) {
|
|
|
55
75
|
*
|
|
56
76
|
* @param requireImage - vision role only: entries must accept image input, so
|
|
57
77
|
* a list may hold a text-only entry without an image being sent to it.
|
|
78
|
+
* @param trace - optional sink for one line per entry, in the order tried, so a
|
|
79
|
+
* caller that gets nothing back can print what was tried and why each entry
|
|
80
|
+
* was passed over. Passed in rather than returned because the successful path
|
|
81
|
+
* must stay allocation-free for the classification callers.
|
|
58
82
|
*/
|
|
59
|
-
function resolveRoleModel(ctx, key, requireImage = false) {
|
|
83
|
+
function resolveRoleModel(ctx, key, requireImage = false, trace) {
|
|
60
84
|
const raw = getCustomModelKey(key);
|
|
61
85
|
const patterns = normaliseRoleSetting(raw);
|
|
62
|
-
if (patterns.length === 0)
|
|
86
|
+
if (patterns.length === 0) {
|
|
87
|
+
trace?.push(`${key}: not configured`);
|
|
63
88
|
return undefined;
|
|
89
|
+
}
|
|
64
90
|
const acceptable = (model) => model && (!requireImage || model.input?.includes("image")) ? model : undefined;
|
|
65
|
-
if (!Array.isArray(raw))
|
|
66
|
-
|
|
91
|
+
if (!Array.isArray(raw)) {
|
|
92
|
+
const resolved = resolveFromPattern(ctx, patterns[0]);
|
|
93
|
+
const model = acceptable(resolved);
|
|
94
|
+
if (!trace)
|
|
95
|
+
return model;
|
|
96
|
+
const reason = rejectionReason(resolved, requireImage);
|
|
97
|
+
trace.push(model && reason === null
|
|
98
|
+
? `${key} "${patterns[0]}": using ${modelLabel(model)}, accepts image input`
|
|
99
|
+
: `${key} "${patterns[0]}": ${reason}`);
|
|
100
|
+
return model;
|
|
101
|
+
}
|
|
67
102
|
for (const pattern of patterns) {
|
|
68
|
-
const
|
|
69
|
-
|
|
103
|
+
const resolved = resolveFromPattern(ctx, pattern);
|
|
104
|
+
const model = acceptable(resolved);
|
|
105
|
+
if (model && isModelRouteUsable(model)) {
|
|
106
|
+
trace?.push(`${key} "${pattern}": using ${modelLabel(model)}, the provider reports the route usable`);
|
|
70
107
|
return model;
|
|
108
|
+
}
|
|
109
|
+
trace?.push(model
|
|
110
|
+
? `${key} "${pattern}": resolved ${modelLabel(model)}, but the provider reports the route unusable`
|
|
111
|
+
: `${key} "${pattern}": ${rejectionReason(resolved, requireImage)}`);
|
|
71
112
|
}
|
|
72
113
|
return undefined;
|
|
73
114
|
}
|
|
@@ -83,7 +124,7 @@ export function resolveUtilityModels(ctx) {
|
|
|
83
124
|
.filter((model) => model !== undefined);
|
|
84
125
|
}
|
|
85
126
|
/**
|
|
86
|
-
* The model a vision call should use.
|
|
127
|
+
* The model a vision call should use, with the walk that produced it.
|
|
87
128
|
*
|
|
88
129
|
* Priority:
|
|
89
130
|
* 1. The current model when it accepts images: the image stays in the call
|
|
@@ -94,18 +135,38 @@ export function resolveUtilityModels(ctx) {
|
|
|
94
135
|
*
|
|
95
136
|
* A text-only entry is never selected for an image call.
|
|
96
137
|
*/
|
|
97
|
-
export function
|
|
138
|
+
export function searchVisionModel(ctx) {
|
|
139
|
+
const tried = [];
|
|
98
140
|
// 1. Current model if vision-capable
|
|
99
|
-
|
|
100
|
-
|
|
141
|
+
const current = ctx.model;
|
|
142
|
+
if (current?.input?.includes("image")) {
|
|
143
|
+
tried.push(`current model ${modelLabel(current)}: accepts image input`);
|
|
144
|
+
return { model: current, source: "current", tried };
|
|
145
|
+
}
|
|
146
|
+
tried.push(current
|
|
147
|
+
? `current model ${modelLabel(current)}: ${rejectionReason(current, true)}`
|
|
148
|
+
: "current model: none selected");
|
|
101
149
|
// 2. The merged role supplies an image-capable entry; a text-only entry in
|
|
102
150
|
// the list is skipped rather than sent an image.
|
|
103
|
-
const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true);
|
|
151
|
+
const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true, tried);
|
|
104
152
|
if (role)
|
|
105
|
-
return role;
|
|
153
|
+
return { model: role, source: "utilityModel", tried };
|
|
106
154
|
// 3. First available vision model
|
|
107
155
|
const available = ctx.modelRegistry.getAvailable();
|
|
108
|
-
|
|
156
|
+
const imageCapable = available.filter((m) => m?.input?.includes("image"));
|
|
157
|
+
tried.push(`available models: ${available.length}, ${imageCapable.length} accept image input`);
|
|
158
|
+
const fallback = imageCapable[0];
|
|
159
|
+
if (fallback) {
|
|
160
|
+
tried.push(`using ${modelLabel(fallback)}: first available model that accepts images`);
|
|
161
|
+
return { model: fallback, source: "available", tried };
|
|
162
|
+
}
|
|
163
|
+
return { source: null, tried };
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* The model a vision call should use: the `model` of the walk above.
|
|
167
|
+
*/
|
|
168
|
+
export function resolveVisionModel(ctx) {
|
|
169
|
+
return searchVisionModel(ctx).model;
|
|
109
170
|
}
|
|
110
171
|
/**
|
|
111
172
|
* Get the configured utility model for classification tasks.
|
package/dist/lib/public.d.ts
CHANGED
|
@@ -8,7 +8,10 @@
|
|
|
8
8
|
*/
|
|
9
9
|
export { classify } from "./ai/text-classifier.js";
|
|
10
10
|
export type { ClassifierCriteria, ClassificationResult, ClassifierOptions, } from "./ai/text-classifier.js";
|
|
11
|
-
export { resolveVisionModel } from "./ai/model-resolver.js";
|
|
11
|
+
export { resolveVisionModel, searchVisionModel } from "./ai/model-resolver.js";
|
|
12
|
+
export type { VisionModelSearch } from "./ai/model-resolver.js";
|
|
13
|
+
export { callVision } from "./ai/llm-call.js";
|
|
14
|
+
export type { LlmCallOptions, LlmCallResult, LlmImagePart } from "./ai/llm-call.js";
|
|
12
15
|
export { healthcheckDir, HEALTHCHECK_DIR } from "./config/braid-paths.js";
|
|
13
16
|
export { onEvent } from "./events/event-types.js";
|
|
14
17
|
export { getExtensionState } from "./state/extension-state.js";
|
package/dist/lib/public.js
CHANGED
|
@@ -7,7 +7,8 @@
|
|
|
7
7
|
* `./lib`.
|
|
8
8
|
*/
|
|
9
9
|
export { classify } from "./ai/text-classifier.js";
|
|
10
|
-
export { resolveVisionModel } from "./ai/model-resolver.js";
|
|
10
|
+
export { resolveVisionModel, searchVisionModel } from "./ai/model-resolver.js";
|
|
11
|
+
export { callVision } from "./ai/llm-call.js";
|
|
11
12
|
export { healthcheckDir, HEALTHCHECK_DIR } from "./config/braid-paths.js";
|
|
12
13
|
export { onEvent } from "./events/event-types.js";
|
|
13
14
|
export { getExtensionState } from "./state/extension-state.js";
|
package/docs/LOCAL_DEV.md
CHANGED
|
@@ -138,7 +138,7 @@ before and after a full run. A non-zero delta is the leak this rule prevents.
|
|
|
138
138
|
### New fixtures: use `node:test`
|
|
139
139
|
|
|
140
140
|
A new fixture is written on `node:test`. The runner does not care which style a
|
|
141
|
-
file uses -
|
|
141
|
+
file uses - 122 of the core suite's 358 fixtures already use `node:test` - so the
|
|
142
142
|
reason is the failure the other style produces. A hand-rolled helper that calls
|
|
143
143
|
`fn()` without awaiting it discards the case's promise, so an `async` case runs
|
|
144
144
|
detached and the file can print its green tally before that case settles, with
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-braid",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.44.0",
|
|
4
4
|
"description": "🥨 Braid - Metacognitive extension suite for pi coding agent: Adds memory, plan tracking, structured cognitive signals and completion checks on top of pi core loop, so long sessions keep their context and stop claiming unfinished work.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
package/src/lib/ai/llm-call.ts
CHANGED
|
@@ -28,6 +28,11 @@
|
|
|
28
28
|
* callLlm(model, apiKey, systemPrompt, userPrompt, options)
|
|
29
29
|
* Returns { text } on success or { error } on failure (discriminated union).
|
|
30
30
|
* const text = result.text;
|
|
31
|
+
*
|
|
32
|
+
* callVision(model, apiKey, systemPrompt, instruction, images, options)
|
|
33
|
+
* The same union with image parts attached to the user message. A model
|
|
34
|
+
* that declares no image input is refused before anything is sent, so a
|
|
35
|
+
* text-only model can never quietly answer about an image it never saw.
|
|
31
36
|
*/
|
|
32
37
|
|
|
33
38
|
import type {
|
|
@@ -320,6 +325,30 @@ function releaseModelSlot(modelId: string): void {
|
|
|
320
325
|
|
|
321
326
|
// ── Options ────────────────────────────────────────────────────────────────────
|
|
322
327
|
|
|
328
|
+
/**
|
|
329
|
+
* One image attached to an auxiliary request: base64 bytes and the MIME type
|
|
330
|
+
* they carry. `data` is the payload itself, without a `data:` prefix.
|
|
331
|
+
*/
|
|
332
|
+
export interface LlmImagePart {
|
|
333
|
+
/** Base64 payload. */
|
|
334
|
+
data: string;
|
|
335
|
+
/** Declared MIME type, e.g. "image/png". The transport passes it through. */
|
|
336
|
+
mimeType: string;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* What an auxiliary call returns: the text (with the stop reason and usage the
|
|
341
|
+
* provider reported) or one line naming why it failed. A vision call returns
|
|
342
|
+
* the same shape, so a caller never branches on which kind of call it made.
|
|
343
|
+
*/
|
|
344
|
+
export type LlmCallResult =
|
|
345
|
+
| {
|
|
346
|
+
text: string;
|
|
347
|
+
stopReason?: string;
|
|
348
|
+
usage?: { input?: number; output?: number; totalTokens?: number };
|
|
349
|
+
}
|
|
350
|
+
| { error: string };
|
|
351
|
+
|
|
323
352
|
export interface LlmCallOptions {
|
|
324
353
|
/** Temperature for generation. Default: 0.1 */
|
|
325
354
|
temperature?: number;
|
|
@@ -357,6 +386,16 @@ export interface LlmCallOptions {
|
|
|
357
386
|
sdkRetries?: number;
|
|
358
387
|
/** Optional AbortSignal for cancellation */
|
|
359
388
|
signal?: AbortSignal;
|
|
389
|
+
/**
|
|
390
|
+
* Image parts attached to the user message. `callVision` is the entry point
|
|
391
|
+
* that sets this: a text call leaves it out and its user content stays a
|
|
392
|
+
* plain string, so the request shape of every existing caller is unchanged.
|
|
393
|
+
*
|
|
394
|
+
* The model's declared input is checked here as well as in `callVision`,
|
|
395
|
+
* because the transport would otherwise drop the image silently or return a
|
|
396
|
+
* provider error that reads like a transient failure.
|
|
397
|
+
*/
|
|
398
|
+
images?: readonly LlmImagePart[];
|
|
360
399
|
}
|
|
361
400
|
|
|
362
401
|
// ── Public API ─────────────────────────────────────────────────────────────────
|
|
@@ -379,14 +418,7 @@ export async function callLlm(
|
|
|
379
418
|
systemPrompt: string,
|
|
380
419
|
userPrompt: string,
|
|
381
420
|
options: LlmCallOptions = {},
|
|
382
|
-
): Promise<
|
|
383
|
-
| {
|
|
384
|
-
text: string;
|
|
385
|
-
stopReason?: string;
|
|
386
|
-
usage?: { input?: number; output?: number; totalTokens?: number };
|
|
387
|
-
}
|
|
388
|
-
| { error: string }
|
|
389
|
-
> {
|
|
421
|
+
): Promise<LlmCallResult> {
|
|
390
422
|
// Circuit breaker: skip while a recent 429 cooldown is active ───────
|
|
391
423
|
const now = Date.now();
|
|
392
424
|
if (rateLimitCooldownUntil > 0 && now < rateLimitCooldownUntil) {
|
|
@@ -395,6 +427,13 @@ export async function callLlm(
|
|
|
395
427
|
return { error: "Rate-limit cooldown active" };
|
|
396
428
|
}
|
|
397
429
|
|
|
430
|
+
// Images: a request the model cannot read is refused before it is built,
|
|
431
|
+
// not sent and reported as a provider error (or, worse, answered from the
|
|
432
|
+
// text alone with no complaint about the missing picture).
|
|
433
|
+
const images = options.images ?? [];
|
|
434
|
+
const imageRefusal = refuseImages(model, images);
|
|
435
|
+
if (imageRefusal !== null) return { error: imageRefusal };
|
|
436
|
+
|
|
398
437
|
const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
399
438
|
|
|
400
439
|
const maxTokens =
|
|
@@ -424,7 +463,9 @@ export async function callLlm(
|
|
|
424
463
|
|
|
425
464
|
const context: Context = {
|
|
426
465
|
systemPrompt: systemSafe.text,
|
|
427
|
-
messages: [
|
|
466
|
+
messages: [
|
|
467
|
+
{ role: "user", content: userContent(userSafe.text, images), timestamp: Date.now() },
|
|
468
|
+
],
|
|
428
469
|
};
|
|
429
470
|
|
|
430
471
|
// ── pre-send budget ──
|
|
@@ -439,7 +480,8 @@ export async function callLlm(
|
|
|
439
480
|
measure: () =>
|
|
440
481
|
calibrateEstimate(
|
|
441
482
|
estimatePayloadTokens({ systemPrompt: context.systemPrompt, messages: context.messages }),
|
|
442
|
-
).tokens
|
|
483
|
+
).tokens +
|
|
484
|
+
images.length * IMAGE_TOKEN_ALLOWANCE,
|
|
443
485
|
outputTokens: maxTokens,
|
|
444
486
|
contextWindow: resolvedWindow.contextWindow,
|
|
445
487
|
// The only compaction available to an auxiliary call is the published
|
|
@@ -763,8 +805,115 @@ export async function callLlm(
|
|
|
763
805
|
}
|
|
764
806
|
}
|
|
765
807
|
|
|
808
|
+
// ── Vision ────────────────────────────────────────────────────────────────────
|
|
809
|
+
|
|
810
|
+
/**
|
|
811
|
+
* Make an auxiliary call that carries images: the same dispatch, retry ladder,
|
|
812
|
+
* circuit breaker, pre-send budget and key-ring feedback as `callLlm`, with the
|
|
813
|
+
* images attached to the user message.
|
|
814
|
+
*
|
|
815
|
+
* Why this is a wrapper and not a second dispatcher: every policy the text
|
|
816
|
+
* path carries (per-model concurrency, transient retry with the backoff
|
|
817
|
+
* ceiling, the 413 emergency offload, the listener and key-ring seams) applies
|
|
818
|
+
* to a vision call exactly as it applies to a text one. A copy would drift from
|
|
819
|
+
* that policy at the first fix made in one place, and a vision call that does
|
|
820
|
+
* not feed the key ring back is a route that keeps failing without rotating.
|
|
821
|
+
* The image-specific decisions - is this model image-capable, is this part a
|
|
822
|
+
* real image - are the only ones the wrapper owns.
|
|
823
|
+
*
|
|
824
|
+
* `instruction` is the user turn; `images` are attached after it, in order, so
|
|
825
|
+
* a caller can refer to "the first image" and mean what it sent.
|
|
826
|
+
*/
|
|
827
|
+
export async function callVision(
|
|
828
|
+
model: Model<Api>,
|
|
829
|
+
apiKey: string | undefined,
|
|
830
|
+
systemPrompt: string,
|
|
831
|
+
instruction: string,
|
|
832
|
+
images: readonly LlmImagePart[],
|
|
833
|
+
options: Omit<LlmCallOptions, "images"> = {},
|
|
834
|
+
): Promise<LlmCallResult> {
|
|
835
|
+
if (images.length === 0) {
|
|
836
|
+
return {
|
|
837
|
+
error:
|
|
838
|
+
"Image input refused: a vision call needs at least one image part. " +
|
|
839
|
+
"Use callLlm for a text-only request.",
|
|
840
|
+
};
|
|
841
|
+
}
|
|
842
|
+
return callLlm(model, apiKey, systemPrompt, instruction, { ...options, images });
|
|
843
|
+
}
|
|
844
|
+
|
|
766
845
|
// ── Helpers ────────────────────────────────────────────────────────────────────
|
|
767
846
|
|
|
847
|
+
/**
|
|
848
|
+
* Tokens the pre-send budget counts per attached image.
|
|
849
|
+
*
|
|
850
|
+
* The payload estimator reads text only: an image block contributes its
|
|
851
|
+
* `mimeType` and nothing of its `data` (see estimateMessageTokens in
|
|
852
|
+
* detection/context-usage.ts), so without this allowance an image-heavy
|
|
853
|
+
* request looks free and the guard waves through a payload the provider then
|
|
854
|
+
* rejects. An image is priced by its pixels, not its bytes, and a full-page
|
|
855
|
+
* capture is a few thousand tokens on every provider that accepts one: OpenAI
|
|
856
|
+
* at high detail tiles a 1440x3000 capture into 18 tiles (~3.2K tokens),
|
|
857
|
+
* Anthropic's pixel rule puts the same image near 1.6K after its own downscale.
|
|
858
|
+
* This constant is the conservative ceiling of that range, so the estimate errs
|
|
859
|
+
* toward refusing rather than toward a doomed send.
|
|
860
|
+
*/
|
|
861
|
+
const IMAGE_TOKEN_ALLOWANCE = 4_000;
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* The one place that decides whether a request carrying images may be built.
|
|
865
|
+
* Null means yes. A refusal names the model, what it declares, and the offending
|
|
866
|
+
* part, because the caller's next move (pick another model, re-encode the
|
|
867
|
+
* image) depends on which of those failed.
|
|
868
|
+
*/
|
|
869
|
+
function refuseImages(model: Model<Api>, images: readonly LlmImagePart[]): string | null {
|
|
870
|
+
if (images.length === 0) return null;
|
|
871
|
+
const declared = model.input ?? [];
|
|
872
|
+
if (!declared.includes("image")) {
|
|
873
|
+
return (
|
|
874
|
+
`Image input refused: ${model.provider ?? "unknown"}/${model.id} declares input ` +
|
|
875
|
+
`[${declared.join(", ") || "text"}] and does not accept images. A vision call needs an ` +
|
|
876
|
+
`image-capable model; resolveVisionModel names one for the current session.`
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
for (let index = 0; index < images.length; index++) {
|
|
880
|
+
const image = images[index];
|
|
881
|
+
const position = `Image part ${index + 1} of ${images.length}`;
|
|
882
|
+
if (typeof image.data !== "string" || image.data.length === 0) {
|
|
883
|
+
return `${position} carries no data. Expected base64 image bytes, not an empty string.`;
|
|
884
|
+
}
|
|
885
|
+
if (typeof image.mimeType !== "string" || !/^image\/[a-z0-9.+-]+$/i.test(image.mimeType)) {
|
|
886
|
+
return (
|
|
887
|
+
`${position} declares no image MIME type (got ${JSON.stringify(image.mimeType)}). ` +
|
|
888
|
+
`Expected a type such as "image/png".`
|
|
889
|
+
);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
return null;
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/**
|
|
896
|
+
* The user message content: a plain string for a text call, text-then-images for
|
|
897
|
+
* a vision call. The text block comes first so an instruction that says "the
|
|
898
|
+
* image below" reads in the order it was written.
|
|
899
|
+
*/
|
|
900
|
+
function userContent(
|
|
901
|
+
text: string,
|
|
902
|
+
images: readonly LlmImagePart[],
|
|
903
|
+
):
|
|
904
|
+
| string
|
|
905
|
+
| Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
|
|
906
|
+
if (images.length === 0) return text;
|
|
907
|
+
return [
|
|
908
|
+
{ type: "text", text },
|
|
909
|
+
...images.map((image) => ({
|
|
910
|
+
type: "image" as const,
|
|
911
|
+
data: image.data,
|
|
912
|
+
mimeType: image.mimeType,
|
|
913
|
+
})),
|
|
914
|
+
];
|
|
915
|
+
}
|
|
916
|
+
|
|
768
917
|
/**
|
|
769
918
|
* Run the published emergency offloader (context-manager's
|
|
770
919
|
* context-emergency.ts) against the current session context. Lossy-safe:
|
|
@@ -47,6 +47,27 @@ export function normaliseRoleSetting(raw: unknown): string[] {
|
|
|
47
47
|
return patterns;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* A model named the way a report names it: provider then id, the pair a reader
|
|
52
|
+
* can paste into a model flag.
|
|
53
|
+
*/
|
|
54
|
+
function modelLabel(model: Model<Api>): string {
|
|
55
|
+
return `${model.provider ?? "unknown"}/${model.id}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Why a resolved model was not accepted for a role, in words, for a caller that
|
|
60
|
+
* has to explain an empty result. Null when the model is acceptable.
|
|
61
|
+
*/
|
|
62
|
+
function rejectionReason(model: Model<Api> | undefined, requireImage: boolean): string | null {
|
|
63
|
+
if (!model) return "no provider serves this pattern";
|
|
64
|
+
if (requireImage && !model.input?.includes("image")) {
|
|
65
|
+
const declared = model.input ?? [];
|
|
66
|
+
return `declares input [${declared.join(", ") || "text"}], no image input`;
|
|
67
|
+
}
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
|
|
50
71
|
/**
|
|
51
72
|
* Resolve a role setting to a single model.
|
|
52
73
|
*
|
|
@@ -60,24 +81,54 @@ export function normaliseRoleSetting(raw: unknown): string[] {
|
|
|
60
81
|
*
|
|
61
82
|
* @param requireImage - vision role only: entries must accept image input, so
|
|
62
83
|
* a list may hold a text-only entry without an image being sent to it.
|
|
84
|
+
* @param trace - optional sink for one line per entry, in the order tried, so a
|
|
85
|
+
* caller that gets nothing back can print what was tried and why each entry
|
|
86
|
+
* was passed over. Passed in rather than returned because the successful path
|
|
87
|
+
* must stay allocation-free for the classification callers.
|
|
63
88
|
*/
|
|
64
89
|
function resolveRoleModel(
|
|
65
90
|
ctx: ExtensionContext,
|
|
66
91
|
key: string,
|
|
67
92
|
requireImage = false,
|
|
93
|
+
trace?: string[],
|
|
68
94
|
): Model<Api> | undefined {
|
|
69
95
|
const raw: unknown = getCustomModelKey(key);
|
|
70
96
|
const patterns = normaliseRoleSetting(raw);
|
|
71
|
-
if (patterns.length === 0)
|
|
97
|
+
if (patterns.length === 0) {
|
|
98
|
+
trace?.push(`${key}: not configured`);
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
72
101
|
|
|
73
102
|
const acceptable = (model: Model<Api> | undefined): Model<Api> | undefined =>
|
|
74
103
|
model && (!requireImage || model.input?.includes("image")) ? model : undefined;
|
|
75
104
|
|
|
76
|
-
if (!Array.isArray(raw))
|
|
105
|
+
if (!Array.isArray(raw)) {
|
|
106
|
+
const resolved = resolveFromPattern(ctx, patterns[0]);
|
|
107
|
+
const model = acceptable(resolved);
|
|
108
|
+
if (!trace) return model;
|
|
109
|
+
const reason = rejectionReason(resolved, requireImage);
|
|
110
|
+
trace.push(
|
|
111
|
+
model && reason === null
|
|
112
|
+
? `${key} "${patterns[0]}": using ${modelLabel(model)}, accepts image input`
|
|
113
|
+
: `${key} "${patterns[0]}": ${reason}`,
|
|
114
|
+
);
|
|
115
|
+
return model;
|
|
116
|
+
}
|
|
77
117
|
|
|
78
118
|
for (const pattern of patterns) {
|
|
79
|
-
const
|
|
80
|
-
|
|
119
|
+
const resolved = resolveFromPattern(ctx, pattern);
|
|
120
|
+
const model = acceptable(resolved);
|
|
121
|
+
if (model && isModelRouteUsable(model)) {
|
|
122
|
+
trace?.push(
|
|
123
|
+
`${key} "${pattern}": using ${modelLabel(model)}, the provider reports the route usable`,
|
|
124
|
+
);
|
|
125
|
+
return model;
|
|
126
|
+
}
|
|
127
|
+
trace?.push(
|
|
128
|
+
model
|
|
129
|
+
? `${key} "${pattern}": resolved ${modelLabel(model)}, but the provider reports the route unusable`
|
|
130
|
+
: `${key} "${pattern}": ${rejectionReason(resolved, requireImage)}`,
|
|
131
|
+
);
|
|
81
132
|
}
|
|
82
133
|
return undefined;
|
|
83
134
|
}
|
|
@@ -95,7 +146,24 @@ export function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[] {
|
|
|
95
146
|
}
|
|
96
147
|
|
|
97
148
|
/**
|
|
98
|
-
*
|
|
149
|
+
* What a vision-model resolution found and what it passed over.
|
|
150
|
+
*
|
|
151
|
+
* `resolveVisionModel` answers "which model"; a caller that reports to a human
|
|
152
|
+
* also has to answer "and then what should I change", which needs the rungs
|
|
153
|
+
* that produced nothing. Both come from one walk, so the model a report names
|
|
154
|
+
* and the trace beside it cannot disagree.
|
|
155
|
+
*/
|
|
156
|
+
export interface VisionModelSearch {
|
|
157
|
+
/** The model a vision call should use, or undefined when the walk found none. */
|
|
158
|
+
model?: Model<Api>;
|
|
159
|
+
/** Which rung supplied `model`; null when none did. */
|
|
160
|
+
source: "current" | "utilityModel" | "available" | null;
|
|
161
|
+
/** One line per rung, in the order tried, saying what it found or why not. */
|
|
162
|
+
tried: string[];
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The model a vision call should use, with the walk that produced it.
|
|
99
167
|
*
|
|
100
168
|
* Priority:
|
|
101
169
|
* 1. The current model when it accepts images: the image stays in the call
|
|
@@ -106,18 +174,43 @@ export function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[] {
|
|
|
106
174
|
*
|
|
107
175
|
* A text-only entry is never selected for an image call.
|
|
108
176
|
*/
|
|
109
|
-
export function
|
|
177
|
+
export function searchVisionModel(ctx: ExtensionContext): VisionModelSearch {
|
|
178
|
+
const tried: string[] = [];
|
|
179
|
+
|
|
110
180
|
// 1. Current model if vision-capable
|
|
111
|
-
|
|
181
|
+
const current = ctx.model;
|
|
182
|
+
if (current?.input?.includes("image")) {
|
|
183
|
+
tried.push(`current model ${modelLabel(current)}: accepts image input`);
|
|
184
|
+
return { model: current, source: "current", tried };
|
|
185
|
+
}
|
|
186
|
+
tried.push(
|
|
187
|
+
current
|
|
188
|
+
? `current model ${modelLabel(current)}: ${rejectionReason(current, true)}`
|
|
189
|
+
: "current model: none selected",
|
|
190
|
+
);
|
|
112
191
|
|
|
113
192
|
// 2. The merged role supplies an image-capable entry; a text-only entry in
|
|
114
193
|
// the list is skipped rather than sent an image.
|
|
115
|
-
const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true);
|
|
116
|
-
if (role) return role;
|
|
194
|
+
const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true, tried);
|
|
195
|
+
if (role) return { model: role, source: "utilityModel", tried };
|
|
117
196
|
|
|
118
197
|
// 3. First available vision model
|
|
119
198
|
const available = ctx.modelRegistry.getAvailable();
|
|
120
|
-
|
|
199
|
+
const imageCapable = available.filter((m) => m?.input?.includes("image"));
|
|
200
|
+
tried.push(`available models: ${available.length}, ${imageCapable.length} accept image input`);
|
|
201
|
+
const fallback = imageCapable[0];
|
|
202
|
+
if (fallback) {
|
|
203
|
+
tried.push(`using ${modelLabel(fallback)}: first available model that accepts images`);
|
|
204
|
+
return { model: fallback, source: "available", tried };
|
|
205
|
+
}
|
|
206
|
+
return { source: null, tried };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* The model a vision call should use: the `model` of the walk above.
|
|
211
|
+
*/
|
|
212
|
+
export function resolveVisionModel(ctx: ExtensionContext): Model<Api> | undefined {
|
|
213
|
+
return searchVisionModel(ctx).model;
|
|
121
214
|
}
|
|
122
215
|
|
|
123
216
|
/**
|
package/src/lib/public.ts
CHANGED
|
@@ -13,7 +13,10 @@ export type {
|
|
|
13
13
|
ClassificationResult,
|
|
14
14
|
ClassifierOptions,
|
|
15
15
|
} from "./ai/text-classifier.js";
|
|
16
|
-
export { resolveVisionModel } from "./ai/model-resolver.js";
|
|
16
|
+
export { resolveVisionModel, searchVisionModel } from "./ai/model-resolver.js";
|
|
17
|
+
export type { VisionModelSearch } from "./ai/model-resolver.js";
|
|
18
|
+
export { callVision } from "./ai/llm-call.js";
|
|
19
|
+
export type { LlmCallOptions, LlmCallResult, LlmImagePart } from "./ai/llm-call.js";
|
|
17
20
|
|
|
18
21
|
export { healthcheckDir, HEALTHCHECK_DIR } from "./config/braid-paths.js";
|
|
19
22
|
|