pi-braid 0.43.60 → 0.44.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -0
- package/build-info.json +4 -4
- package/dist/extensions/enforcement/handlers/register-adherence-meter.d.ts +40 -9
- package/dist/extensions/enforcement/handlers/register-adherence-meter.js +120 -40
- package/dist/lib/ai/llm-call.d.ts +60 -11
- package/dist/lib/ai/llm-call.js +104 -2
- package/dist/lib/ai/model-resolver.d.ts +21 -1
- package/dist/lib/ai/model-resolver.js +74 -13
- package/dist/lib/config/braid-paths.d.ts +19 -0
- package/dist/lib/config/braid-paths.js +33 -1
- package/dist/lib/enforcement/adherence-meter.d.ts +49 -0
- package/dist/lib/enforcement/adherence-meter.js +49 -0
- package/dist/lib/public.d.ts +4 -1
- package/dist/lib/public.js +2 -1
- package/dist/lib/state/global-state.d.ts +14 -1
- package/docs/LOCAL_DEV.md +1 -1
- package/package.json +1 -1
- package/src/extensions/enforcement/handlers/AGENTS.md +14 -14
- package/src/extensions/enforcement/handlers/register-adherence-meter.ts +163 -49
- package/src/lib/ai/llm-call.ts +159 -10
- package/src/lib/ai/model-resolver.ts +103 -10
- package/src/lib/config/braid-paths.ts +34 -1
- package/src/lib/enforcement/AGENTS.md +4 -4
- package/src/lib/enforcement/adherence-meter.ts +90 -0
- package/src/lib/public.ts +4 -1
- package/src/lib/state/global-state.ts +18 -1
|
@@ -7,10 +7,13 @@
|
|
|
7
7
|
* family `rule` but no whisper source; there is no durable per-rule
|
|
8
8
|
* violation record, so this is the only honest count and it is
|
|
9
9
|
* current-session only.
|
|
10
|
-
* 2. On turn_end, aggregate the active branch and
|
|
11
|
-
*
|
|
12
|
-
* (`WATCHDOG_TG_SIGNAL_KEY`, the same route the watchdog page uses),
|
|
13
|
-
*
|
|
10
|
+
* 2. On turn_end, aggregate the active branch and report a rule that repeats
|
|
11
|
+
* past K to the owner over the existing TG signal seam
|
|
12
|
+
* (`WATCHDOG_TG_SIGNAL_KEY`, the same route the watchdog page uses), as at
|
|
13
|
+
* most ONE digest per hour that states how many crossings it folded. The
|
|
14
|
+
* milestone ladder and the per-session budget are unchanged: a crossing is
|
|
15
|
+
* still planned by `planEscalation` and its rung is still spent only when
|
|
16
|
+
* the digest was accepted.
|
|
14
17
|
* 3. On session shutdown, append the owner report
|
|
15
18
|
* (`~/.pi/agent/braid/adherence/report.jsonl`).
|
|
16
19
|
*
|
|
@@ -33,18 +36,22 @@ import {
|
|
|
33
36
|
safeSessionManagerTyped,
|
|
34
37
|
} from "../../../lib/state/extension-state.js";
|
|
35
38
|
import { getWatchdogTgSignalSender } from "../../../lib/state/global-state.js";
|
|
39
|
+
import { readTelegramPollerLiveness } from "../../../lib/state/telegram-poller-liveness.js";
|
|
36
40
|
import { formatLocalTime } from "../../../lib/utils/local-time.js";
|
|
37
41
|
import {
|
|
38
42
|
activeBranchEntries,
|
|
43
|
+
adherenceReportPath,
|
|
39
44
|
aggregateRuleAdherence,
|
|
45
|
+
commitDigest,
|
|
40
46
|
createEscalationState,
|
|
41
|
-
|
|
47
|
+
planDigest,
|
|
42
48
|
planEscalation,
|
|
43
49
|
readSessionEntriesFromFile,
|
|
50
|
+
recordEscalation,
|
|
44
51
|
writeAdherenceReport,
|
|
45
52
|
type AdherenceEscalationState,
|
|
46
53
|
type AdherenceWriteResult,
|
|
47
|
-
type
|
|
54
|
+
type PendingEscalation,
|
|
48
55
|
} from "../../../lib/enforcement/adherence-meter.js";
|
|
49
56
|
|
|
50
57
|
const log = createLogger("adherence-meter");
|
|
@@ -52,15 +59,49 @@ const log = createLogger("adherence-meter");
|
|
|
52
59
|
/** Extension-state bucket key for the per-session meter state. */
|
|
53
60
|
const ADHERENCE_STATE_KEY = "adherence-meter";
|
|
54
61
|
|
|
62
|
+
/** What a crossing may ask the owner channel for. */
|
|
63
|
+
export interface AdherenceOwnerSender {
|
|
64
|
+
(
|
|
65
|
+
text: string,
|
|
66
|
+
opts?: {
|
|
67
|
+
parseMode?: "HTML" | "Markdown";
|
|
68
|
+
source?: string;
|
|
69
|
+
/** Escalations folded into this message instead of sent one by one. */
|
|
70
|
+
suppressed?: number;
|
|
71
|
+
},
|
|
72
|
+
): boolean | Promise<boolean>;
|
|
73
|
+
}
|
|
74
|
+
|
|
55
75
|
/** Injectables for fixtures; production callers pass nothing. */
|
|
56
76
|
export interface AdherenceWiringDeps {
|
|
57
77
|
now?: () => number;
|
|
58
78
|
/** Owner-channel factory; defaults to the published TG signal sender. */
|
|
59
|
-
sender?: () =>
|
|
79
|
+
sender?: () => AdherenceOwnerSender | null;
|
|
80
|
+
/**
|
|
81
|
+
* True when THIS session is the one that reads the owner's chat. Defaults to
|
|
82
|
+
* the Telegram poller's live state (the seat that owns the poller is the seat
|
|
83
|
+
* whose escalations the owner must hear). A second seat shares the bot token
|
|
84
|
+
* but not the conversation, and its escalations never become owner messages;
|
|
85
|
+
* a spawned seat registers no Telegram surface at all, so it publishes no
|
|
86
|
+
* sender either.
|
|
87
|
+
*/
|
|
88
|
+
ownerSession?: () => boolean;
|
|
89
|
+
/** How often the owner may receive a digest (default one hour). */
|
|
90
|
+
digestWindowMs?: number;
|
|
60
91
|
reportDir?: string;
|
|
61
92
|
k?: number;
|
|
62
93
|
}
|
|
63
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Owner-facing session predicate. The liveness probe is core's own registry
|
|
97
|
+
* (`telegram-poller-liveness.ts`): "running" or "unknown" means no other seat
|
|
98
|
+
* is holding the owner's inbound channel, "stopped" means this process loaded
|
|
99
|
+
* Telegram and its poller is not the one reading the chat.
|
|
100
|
+
*/
|
|
101
|
+
function isOwnerChannelSession(): boolean {
|
|
102
|
+
return readTelegramPollerLiveness() !== "stopped";
|
|
103
|
+
}
|
|
104
|
+
|
|
64
105
|
function adherenceState(ctx: ExtensionContext): AdherenceEscalationState {
|
|
65
106
|
return getExtensionState(ctx, ADHERENCE_STATE_KEY, createEscalationState);
|
|
66
107
|
}
|
|
@@ -75,30 +116,68 @@ function branchEntries(ctx: ExtensionContext): unknown[] {
|
|
|
75
116
|
}
|
|
76
117
|
}
|
|
77
118
|
|
|
78
|
-
/**
|
|
79
|
-
function
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
119
|
+
/** Escape the three characters Telegram's HTML parser reserves. */
|
|
120
|
+
function escapeHtml(text: string): string {
|
|
121
|
+
return text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function shorten(text: string, max: number): string {
|
|
125
|
+
const trimmed = text.trim();
|
|
126
|
+
return trimmed.length <= max ? trimmed : `${trimmed.slice(0, max - 3)}...`;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** The rule's matched text (the key's `source::pattern` tail), shortened. */
|
|
130
|
+
function ruleLabel(entry: PendingEscalation): string {
|
|
131
|
+
const tail = entry.ruleKey.includes("::")
|
|
132
|
+
? entry.ruleKey.split("::").slice(1).join("::")
|
|
133
|
+
: entry.ruleKey;
|
|
134
|
+
return shorten(`${entry.source}: ${tail}`, 64);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* The hourly digest: a bold subject naming the highest rung it reports, then
|
|
139
|
+
* labelled lines. Short enough to survive as plain text (the owner's
|
|
140
|
+
* tweet-length rule) and valid under `parse_mode: HTML` (the Telegram outbound
|
|
141
|
+
* gate validates it before any send).
|
|
142
|
+
*/
|
|
143
|
+
function renderDigest(
|
|
144
|
+
digest: { pending: PendingEscalation[]; suppressed: number },
|
|
83
145
|
ctx: ExtensionContext,
|
|
84
146
|
nowMs: number,
|
|
85
147
|
): string {
|
|
86
|
-
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
`[adherence] rule repeating · rung ${milestone}\n` +
|
|
90
|
-
`${rule.source}: "${signal}"\n` +
|
|
91
|
-
`\n` +
|
|
92
|
-
`deliveries ${rule.deliveries} · repeats ${rule.repeats} · suppressed ${rule.suppressions} · escalation ${state.total + 1}\n` +
|
|
93
|
-
`session ${sessionId} · ${formatLocalTime(new Date(nowMs))}`
|
|
148
|
+
// The subject is the highest rung in the digest; ties go to the latest.
|
|
149
|
+
const subject = digest.pending.reduce((best, entry) =>
|
|
150
|
+
entry.milestone >= best.milestone ? entry : best,
|
|
94
151
|
);
|
|
152
|
+
const lines = [
|
|
153
|
+
`<b>adherence · rung ${subject.milestone}</b>`,
|
|
154
|
+
`rule ${escapeHtml(ruleLabel(subject))}`,
|
|
155
|
+
...(digest.pending.length > 1 ? [`rules ${digest.pending.length}`] : []),
|
|
156
|
+
`deliveries ${subject.deliveries}`,
|
|
157
|
+
`repeats ${subject.repeats}`,
|
|
158
|
+
`suppressed ${digest.suppressed}`,
|
|
159
|
+
`session ${escapeHtml(shorten(getSessionIdSafe(ctx) || "unknown", 32))}`,
|
|
160
|
+
`at ${formatLocalTime(new Date(nowMs))}`,
|
|
161
|
+
// The next step is a pointer, not advice: the full signal text and the
|
|
162
|
+
// per-rule counts live in the session report.
|
|
163
|
+
`where ${escapeHtml(shorten(adherenceReportPath(), 72))}`,
|
|
164
|
+
];
|
|
165
|
+
return lines.join("\n");
|
|
95
166
|
}
|
|
96
167
|
|
|
97
168
|
/**
|
|
98
|
-
* Aggregate the current branch and
|
|
99
|
-
*
|
|
100
|
-
*
|
|
101
|
-
*
|
|
169
|
+
* Aggregate the current branch and fold every crossing into the owner's
|
|
170
|
+
* hourly digest.
|
|
171
|
+
*
|
|
172
|
+
* What did NOT change: the ladder. `planEscalation` is asked exactly as before,
|
|
173
|
+
* and a rung is still spent only when the owner channel accepted the message
|
|
174
|
+
* that reported it, so the milestones, thresholds and detection are untouched.
|
|
175
|
+
*
|
|
176
|
+
* What changed: a crossing is buffered (`recordEscalation`) and delivered as
|
|
177
|
+
* ONE digest per window (`planDigest`), which states how many crossings it
|
|
178
|
+
* folded instead of sending each one as its own message. Two seats sharing a
|
|
179
|
+
* bot token no longer both write into the owner's chat: only the session that
|
|
180
|
+
* owns the inbound channel escalates.
|
|
102
181
|
*/
|
|
103
182
|
export async function checkAdherence(
|
|
104
183
|
ctx: ExtensionContext,
|
|
@@ -106,38 +185,73 @@ export async function checkAdherence(
|
|
|
106
185
|
): Promise<{ fired: Array<{ rule: string; milestone: number }> }> {
|
|
107
186
|
const state = adherenceState(ctx);
|
|
108
187
|
const entries = branchEntries(ctx);
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
const rules = aggregateRuleAdherence(activeBranchEntries(entries));
|
|
188
|
+
const rules = entries.length > 0 ? aggregateRuleAdherence(activeBranchEntries(entries)) : [];
|
|
112
189
|
const sender = (deps.sender ?? getWatchdogTgSignalSender)();
|
|
113
190
|
const nowMs = (deps.now ?? Date.now)();
|
|
114
|
-
const
|
|
191
|
+
const sessionId = getSessionIdSafe(ctx);
|
|
115
192
|
|
|
116
193
|
for (const rule of rules) {
|
|
117
194
|
const plan = planEscalation(state, rule.rule, rule.repeats);
|
|
118
195
|
if (!plan.fire) continue;
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
continue;
|
|
137
|
-
}
|
|
138
|
-
markEscalated(state, rule.rule);
|
|
139
|
-
fired.push({ rule: rule.rule, milestone: plan.milestone });
|
|
196
|
+
recordEscalation(state, {
|
|
197
|
+
ruleKey: rule.rule,
|
|
198
|
+
rung: (state.rules[rule.rule]?.rung ?? 0) + 1,
|
|
199
|
+
milestone: plan.milestone,
|
|
200
|
+
source: rule.source,
|
|
201
|
+
deliveries: rule.deliveries,
|
|
202
|
+
repeats: rule.repeats,
|
|
203
|
+
sessionId,
|
|
204
|
+
at: nowMs,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const buffered = state.pending ?? [];
|
|
209
|
+
if (buffered.length === 0) return { fired: [] };
|
|
210
|
+
if (!sender) {
|
|
211
|
+
log.debug(`[adherence] ${buffered.length} escalation(s) buffered; no owner channel published`);
|
|
212
|
+
return { fired: [] };
|
|
140
213
|
}
|
|
214
|
+
if (!(deps.ownerSession ?? isOwnerChannelSession)()) {
|
|
215
|
+
log.debug(
|
|
216
|
+
`[adherence] ${buffered.length} escalation(s) buffered; this session does not own the owner channel`,
|
|
217
|
+
);
|
|
218
|
+
return { fired: [] };
|
|
219
|
+
}
|
|
220
|
+
const digest = planDigest(state, nowMs, {
|
|
221
|
+
...(deps.digestWindowMs !== undefined ? { windowMs: deps.digestWindowMs } : {}),
|
|
222
|
+
});
|
|
223
|
+
if (!digest.send) {
|
|
224
|
+
log.debug(
|
|
225
|
+
`[adherence] ${digest.pending.length} escalation(s) buffered; digest window closed (${digest.suppressed} suppressed so far)`,
|
|
226
|
+
);
|
|
227
|
+
return { fired: [] };
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
let ok = false;
|
|
231
|
+
try {
|
|
232
|
+
ok =
|
|
233
|
+
(await Promise.resolve(
|
|
234
|
+
sender(renderDigest(digest, ctx, nowMs), {
|
|
235
|
+
parseMode: "HTML",
|
|
236
|
+
source: "adherence-digest",
|
|
237
|
+
suppressed: digest.suppressed,
|
|
238
|
+
}),
|
|
239
|
+
)) === true;
|
|
240
|
+
} catch (e) {
|
|
241
|
+
log.debug(`[adherence] digest send threw: ${(e as Error).message}`);
|
|
242
|
+
}
|
|
243
|
+
if (!ok) {
|
|
244
|
+
log.debug("[adherence] digest not delivered; crossings stay buffered, rungs unspent");
|
|
245
|
+
return { fired: [] };
|
|
246
|
+
}
|
|
247
|
+
const fired = digest.pending.map((entry) => ({
|
|
248
|
+
rule: entry.ruleKey,
|
|
249
|
+
milestone: entry.milestone,
|
|
250
|
+
}));
|
|
251
|
+
commitDigest(state, nowMs);
|
|
252
|
+
log.debug(
|
|
253
|
+
`[adherence] digest delivered: ${fired.length} crossing(s), ${digest.suppressed} suppressed`,
|
|
254
|
+
);
|
|
141
255
|
return { fired };
|
|
142
256
|
}
|
|
143
257
|
|
package/src/lib/ai/llm-call.ts
CHANGED
|
@@ -28,6 +28,11 @@
|
|
|
28
28
|
* callLlm(model, apiKey, systemPrompt, userPrompt, options)
|
|
29
29
|
* Returns { text } on success or { error } on failure (discriminated union).
|
|
30
30
|
* const text = result.text;
|
|
31
|
+
*
|
|
32
|
+
* callVision(model, apiKey, systemPrompt, instruction, images, options)
|
|
33
|
+
* The same union with image parts attached to the user message. A model
|
|
34
|
+
* that declares no image input is refused before anything is sent, so a
|
|
35
|
+
* text-only model can never quietly answer about an image it never saw.
|
|
31
36
|
*/
|
|
32
37
|
|
|
33
38
|
import type {
|
|
@@ -320,6 +325,30 @@ function releaseModelSlot(modelId: string): void {
|
|
|
320
325
|
|
|
321
326
|
// ── Options ────────────────────────────────────────────────────────────────────
|
|
322
327
|
|
|
328
|
+
/**
|
|
329
|
+
* One image attached to an auxiliary request: base64 bytes and the MIME type
|
|
330
|
+
* they carry. `data` is the payload itself, without a `data:` prefix.
|
|
331
|
+
*/
|
|
332
|
+
export interface LlmImagePart {
|
|
333
|
+
/** Base64 payload. */
|
|
334
|
+
data: string;
|
|
335
|
+
/** Declared MIME type, e.g. "image/png". The transport passes it through. */
|
|
336
|
+
mimeType: string;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* What an auxiliary call returns: the text (with the stop reason and usage the
|
|
341
|
+
* provider reported) or one line naming why it failed. A vision call returns
|
|
342
|
+
* the same shape, so a caller never branches on which kind of call it made.
|
|
343
|
+
*/
|
|
344
|
+
export type LlmCallResult =
|
|
345
|
+
| {
|
|
346
|
+
text: string;
|
|
347
|
+
stopReason?: string;
|
|
348
|
+
usage?: { input?: number; output?: number; totalTokens?: number };
|
|
349
|
+
}
|
|
350
|
+
| { error: string };
|
|
351
|
+
|
|
323
352
|
export interface LlmCallOptions {
|
|
324
353
|
/** Temperature for generation. Default: 0.1 */
|
|
325
354
|
temperature?: number;
|
|
@@ -357,6 +386,16 @@ export interface LlmCallOptions {
|
|
|
357
386
|
sdkRetries?: number;
|
|
358
387
|
/** Optional AbortSignal for cancellation */
|
|
359
388
|
signal?: AbortSignal;
|
|
389
|
+
/**
|
|
390
|
+
* Image parts attached to the user message. `callVision` is the entry point
|
|
391
|
+
* that sets this: a text call leaves it out and its user content stays a
|
|
392
|
+
* plain string, so the request shape of every existing caller is unchanged.
|
|
393
|
+
*
|
|
394
|
+
* The model's declared input is checked here as well as in `callVision`,
|
|
395
|
+
* because the transport would otherwise drop the image silently or return a
|
|
396
|
+
* provider error that reads like a transient failure.
|
|
397
|
+
*/
|
|
398
|
+
images?: readonly LlmImagePart[];
|
|
360
399
|
}
|
|
361
400
|
|
|
362
401
|
// ── Public API ─────────────────────────────────────────────────────────────────
|
|
@@ -379,14 +418,7 @@ export async function callLlm(
|
|
|
379
418
|
systemPrompt: string,
|
|
380
419
|
userPrompt: string,
|
|
381
420
|
options: LlmCallOptions = {},
|
|
382
|
-
): Promise<
|
|
383
|
-
| {
|
|
384
|
-
text: string;
|
|
385
|
-
stopReason?: string;
|
|
386
|
-
usage?: { input?: number; output?: number; totalTokens?: number };
|
|
387
|
-
}
|
|
388
|
-
| { error: string }
|
|
389
|
-
> {
|
|
421
|
+
): Promise<LlmCallResult> {
|
|
390
422
|
// Circuit breaker: skip while a recent 429 cooldown is active ───────
|
|
391
423
|
const now = Date.now();
|
|
392
424
|
if (rateLimitCooldownUntil > 0 && now < rateLimitCooldownUntil) {
|
|
@@ -395,6 +427,13 @@ export async function callLlm(
|
|
|
395
427
|
return { error: "Rate-limit cooldown active" };
|
|
396
428
|
}
|
|
397
429
|
|
|
430
|
+
// Images: a request the model cannot read is refused before it is built,
|
|
431
|
+
// not sent and reported as a provider error (or, worse, answered from the
|
|
432
|
+
// text alone with no complaint about the missing picture).
|
|
433
|
+
const images = options.images ?? [];
|
|
434
|
+
const imageRefusal = refuseImages(model, images);
|
|
435
|
+
if (imageRefusal !== null) return { error: imageRefusal };
|
|
436
|
+
|
|
398
437
|
const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
|
|
399
438
|
|
|
400
439
|
const maxTokens =
|
|
@@ -424,7 +463,9 @@ export async function callLlm(
|
|
|
424
463
|
|
|
425
464
|
const context: Context = {
|
|
426
465
|
systemPrompt: systemSafe.text,
|
|
427
|
-
messages: [
|
|
466
|
+
messages: [
|
|
467
|
+
{ role: "user", content: userContent(userSafe.text, images), timestamp: Date.now() },
|
|
468
|
+
],
|
|
428
469
|
};
|
|
429
470
|
|
|
430
471
|
// ── pre-send budget ──
|
|
@@ -439,7 +480,8 @@ export async function callLlm(
|
|
|
439
480
|
measure: () =>
|
|
440
481
|
calibrateEstimate(
|
|
441
482
|
estimatePayloadTokens({ systemPrompt: context.systemPrompt, messages: context.messages }),
|
|
442
|
-
).tokens
|
|
483
|
+
).tokens +
|
|
484
|
+
images.length * IMAGE_TOKEN_ALLOWANCE,
|
|
443
485
|
outputTokens: maxTokens,
|
|
444
486
|
contextWindow: resolvedWindow.contextWindow,
|
|
445
487
|
// The only compaction available to an auxiliary call is the published
|
|
@@ -763,8 +805,115 @@ export async function callLlm(
|
|
|
763
805
|
}
|
|
764
806
|
}
|
|
765
807
|
|
|
808
|
+
// ── Vision ────────────────────────────────────────────────────────────────────
|
|
809
|
+
|
|
810
|
+
/**
|
|
811
|
+
* Make an auxiliary call that carries images: the same dispatch, retry ladder,
|
|
812
|
+
* circuit breaker, pre-send budget and key-ring feedback as `callLlm`, with the
|
|
813
|
+
* images attached to the user message.
|
|
814
|
+
*
|
|
815
|
+
* Why this is a wrapper and not a second dispatcher: every policy the text
|
|
816
|
+
* path carries (per-model concurrency, transient retry with the backoff
|
|
817
|
+
* ceiling, the 413 emergency offload, the listener and key-ring seams) applies
|
|
818
|
+
* to a vision call exactly as it applies to a text one. A copy would drift from
|
|
819
|
+
* that policy at the first fix made in one place, and a vision call that does
|
|
820
|
+
* not feed the key ring back is a route that keeps failing without rotating.
|
|
821
|
+
* The image-specific decisions - is this model image-capable, is this part a
|
|
822
|
+
* real image - are the only ones the wrapper owns.
|
|
823
|
+
*
|
|
824
|
+
* `instruction` is the user turn; `images` are attached after it, in order, so
|
|
825
|
+
* a caller can refer to "the first image" and mean what it sent.
|
|
826
|
+
*/
|
|
827
|
+
export async function callVision(
|
|
828
|
+
model: Model<Api>,
|
|
829
|
+
apiKey: string | undefined,
|
|
830
|
+
systemPrompt: string,
|
|
831
|
+
instruction: string,
|
|
832
|
+
images: readonly LlmImagePart[],
|
|
833
|
+
options: Omit<LlmCallOptions, "images"> = {},
|
|
834
|
+
): Promise<LlmCallResult> {
|
|
835
|
+
if (images.length === 0) {
|
|
836
|
+
return {
|
|
837
|
+
error:
|
|
838
|
+
"Image input refused: a vision call needs at least one image part. " +
|
|
839
|
+
"Use callLlm for a text-only request.",
|
|
840
|
+
};
|
|
841
|
+
}
|
|
842
|
+
return callLlm(model, apiKey, systemPrompt, instruction, { ...options, images });
|
|
843
|
+
}
|
|
844
|
+
|
|
766
845
|
// ── Helpers ────────────────────────────────────────────────────────────────────
|
|
767
846
|
|
|
847
|
+
/**
|
|
848
|
+
* Tokens the pre-send budget counts per attached image.
|
|
849
|
+
*
|
|
850
|
+
* The payload estimator reads text only: an image block contributes its
|
|
851
|
+
* `mimeType` and nothing of its `data` (see estimateMessageTokens in
|
|
852
|
+
* detection/context-usage.ts), so without this allowance an image-heavy
|
|
853
|
+
* request looks free and the guard waves through a payload the provider then
|
|
854
|
+
* rejects. An image is priced by its pixels, not its bytes, and a full-page
|
|
855
|
+
* capture is a few thousand tokens on every provider that accepts one: OpenAI
|
|
856
|
+
* at high detail tiles a 1440x3000 capture into 18 tiles (~3.2K tokens),
|
|
857
|
+
* Anthropic's pixel rule puts the same image near 1.6K after its own downscale.
|
|
858
|
+
* This constant is the conservative ceiling of that range, so the estimate errs
|
|
859
|
+
* toward refusing rather than toward a doomed send.
|
|
860
|
+
*/
|
|
861
|
+
const IMAGE_TOKEN_ALLOWANCE = 4_000;
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* The one place that decides whether a request carrying images may be built.
|
|
865
|
+
* Null means yes. A refusal names the model, what it declares, and the offending
|
|
866
|
+
* part, because the caller's next move (pick another model, re-encode the
|
|
867
|
+
* image) depends on which of those failed.
|
|
868
|
+
*/
|
|
869
|
+
function refuseImages(model: Model<Api>, images: readonly LlmImagePart[]): string | null {
|
|
870
|
+
if (images.length === 0) return null;
|
|
871
|
+
const declared = model.input ?? [];
|
|
872
|
+
if (!declared.includes("image")) {
|
|
873
|
+
return (
|
|
874
|
+
`Image input refused: ${model.provider ?? "unknown"}/${model.id} declares input ` +
|
|
875
|
+
`[${declared.join(", ") || "text"}] and does not accept images. A vision call needs an ` +
|
|
876
|
+
`image-capable model; resolveVisionModel names one for the current session.`
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
for (let index = 0; index < images.length; index++) {
|
|
880
|
+
const image = images[index];
|
|
881
|
+
const position = `Image part ${index + 1} of ${images.length}`;
|
|
882
|
+
if (typeof image.data !== "string" || image.data.length === 0) {
|
|
883
|
+
return `${position} carries no data. Expected base64 image bytes, not an empty string.`;
|
|
884
|
+
}
|
|
885
|
+
if (typeof image.mimeType !== "string" || !/^image\/[a-z0-9.+-]+$/i.test(image.mimeType)) {
|
|
886
|
+
return (
|
|
887
|
+
`${position} declares no image MIME type (got ${JSON.stringify(image.mimeType)}). ` +
|
|
888
|
+
`Expected a type such as "image/png".`
|
|
889
|
+
);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
return null;
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/**
|
|
896
|
+
* The user message content: a plain string for a text call, text-then-images for
|
|
897
|
+
* a vision call. The text block comes first so an instruction that says "the
|
|
898
|
+
* image below" reads in the order it was written.
|
|
899
|
+
*/
|
|
900
|
+
function userContent(
|
|
901
|
+
text: string,
|
|
902
|
+
images: readonly LlmImagePart[],
|
|
903
|
+
):
|
|
904
|
+
| string
|
|
905
|
+
| Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }> {
|
|
906
|
+
if (images.length === 0) return text;
|
|
907
|
+
return [
|
|
908
|
+
{ type: "text", text },
|
|
909
|
+
...images.map((image) => ({
|
|
910
|
+
type: "image" as const,
|
|
911
|
+
data: image.data,
|
|
912
|
+
mimeType: image.mimeType,
|
|
913
|
+
})),
|
|
914
|
+
];
|
|
915
|
+
}
|
|
916
|
+
|
|
768
917
|
/**
|
|
769
918
|
* Run the published emergency offloader (context-manager's
|
|
770
919
|
* context-emergency.ts) against the current session context. Lossy-safe:
|
|
@@ -47,6 +47,27 @@ export function normaliseRoleSetting(raw: unknown): string[] {
|
|
|
47
47
|
return patterns;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
/**
|
|
51
|
+
* A model named the way a report names it: provider then id, the pair a reader
|
|
52
|
+
* can paste into a model flag.
|
|
53
|
+
*/
|
|
54
|
+
function modelLabel(model: Model<Api>): string {
|
|
55
|
+
return `${model.provider ?? "unknown"}/${model.id}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Why a resolved model was not accepted for a role, in words, for a caller that
|
|
60
|
+
* has to explain an empty result. Null when the model is acceptable.
|
|
61
|
+
*/
|
|
62
|
+
function rejectionReason(model: Model<Api> | undefined, requireImage: boolean): string | null {
|
|
63
|
+
if (!model) return "no provider serves this pattern";
|
|
64
|
+
if (requireImage && !model.input?.includes("image")) {
|
|
65
|
+
const declared = model.input ?? [];
|
|
66
|
+
return `declares input [${declared.join(", ") || "text"}], no image input`;
|
|
67
|
+
}
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
|
|
50
71
|
/**
|
|
51
72
|
* Resolve a role setting to a single model.
|
|
52
73
|
*
|
|
@@ -60,24 +81,54 @@ export function normaliseRoleSetting(raw: unknown): string[] {
|
|
|
60
81
|
*
|
|
61
82
|
* @param requireImage - vision role only: entries must accept image input, so
|
|
62
83
|
* a list may hold a text-only entry without an image being sent to it.
|
|
84
|
+
* @param trace - optional sink for one line per entry, in the order tried, so a
|
|
85
|
+
* caller that gets nothing back can print what was tried and why each entry
|
|
86
|
+
* was passed over. Passed in rather than returned because the successful path
|
|
87
|
+
* must stay allocation-free for the classification callers.
|
|
63
88
|
*/
|
|
64
89
|
function resolveRoleModel(
|
|
65
90
|
ctx: ExtensionContext,
|
|
66
91
|
key: string,
|
|
67
92
|
requireImage = false,
|
|
93
|
+
trace?: string[],
|
|
68
94
|
): Model<Api> | undefined {
|
|
69
95
|
const raw: unknown = getCustomModelKey(key);
|
|
70
96
|
const patterns = normaliseRoleSetting(raw);
|
|
71
|
-
if (patterns.length === 0)
|
|
97
|
+
if (patterns.length === 0) {
|
|
98
|
+
trace?.push(`${key}: not configured`);
|
|
99
|
+
return undefined;
|
|
100
|
+
}
|
|
72
101
|
|
|
73
102
|
const acceptable = (model: Model<Api> | undefined): Model<Api> | undefined =>
|
|
74
103
|
model && (!requireImage || model.input?.includes("image")) ? model : undefined;
|
|
75
104
|
|
|
76
|
-
if (!Array.isArray(raw))
|
|
105
|
+
if (!Array.isArray(raw)) {
|
|
106
|
+
const resolved = resolveFromPattern(ctx, patterns[0]);
|
|
107
|
+
const model = acceptable(resolved);
|
|
108
|
+
if (!trace) return model;
|
|
109
|
+
const reason = rejectionReason(resolved, requireImage);
|
|
110
|
+
trace.push(
|
|
111
|
+
model && reason === null
|
|
112
|
+
? `${key} "${patterns[0]}": using ${modelLabel(model)}, accepts image input`
|
|
113
|
+
: `${key} "${patterns[0]}": ${reason}`,
|
|
114
|
+
);
|
|
115
|
+
return model;
|
|
116
|
+
}
|
|
77
117
|
|
|
78
118
|
for (const pattern of patterns) {
|
|
79
|
-
const
|
|
80
|
-
|
|
119
|
+
const resolved = resolveFromPattern(ctx, pattern);
|
|
120
|
+
const model = acceptable(resolved);
|
|
121
|
+
if (model && isModelRouteUsable(model)) {
|
|
122
|
+
trace?.push(
|
|
123
|
+
`${key} "${pattern}": using ${modelLabel(model)}, the provider reports the route usable`,
|
|
124
|
+
);
|
|
125
|
+
return model;
|
|
126
|
+
}
|
|
127
|
+
trace?.push(
|
|
128
|
+
model
|
|
129
|
+
? `${key} "${pattern}": resolved ${modelLabel(model)}, but the provider reports the route unusable`
|
|
130
|
+
: `${key} "${pattern}": ${rejectionReason(resolved, requireImage)}`,
|
|
131
|
+
);
|
|
81
132
|
}
|
|
82
133
|
return undefined;
|
|
83
134
|
}
|
|
@@ -95,7 +146,24 @@ export function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[] {
|
|
|
95
146
|
}
|
|
96
147
|
|
|
97
148
|
/**
|
|
98
|
-
*
|
|
149
|
+
* What a vision-model resolution found and what it passed over.
|
|
150
|
+
*
|
|
151
|
+
* `resolveVisionModel` answers "which model"; a caller that reports to a human
|
|
152
|
+
* also has to answer "and then what should I change", which needs the rungs
|
|
153
|
+
* that produced nothing. Both come from one walk, so the model a report names
|
|
154
|
+
* and the trace beside it cannot disagree.
|
|
155
|
+
*/
|
|
156
|
+
export interface VisionModelSearch {
|
|
157
|
+
/** The model a vision call should use, or undefined when the walk found none. */
|
|
158
|
+
model?: Model<Api>;
|
|
159
|
+
/** Which rung supplied `model`; null when none did. */
|
|
160
|
+
source: "current" | "utilityModel" | "available" | null;
|
|
161
|
+
/** One line per rung, in the order tried, saying what it found or why not. */
|
|
162
|
+
tried: string[];
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The model a vision call should use, with the walk that produced it.
|
|
99
167
|
*
|
|
100
168
|
* Priority:
|
|
101
169
|
* 1. The current model when it accepts images: the image stays in the call
|
|
@@ -106,18 +174,43 @@ export function resolveUtilityModels(ctx: ExtensionContext): Model<Api>[] {
|
|
|
106
174
|
*
|
|
107
175
|
* A text-only entry is never selected for an image call.
|
|
108
176
|
*/
|
|
109
|
-
export function
|
|
177
|
+
export function searchVisionModel(ctx: ExtensionContext): VisionModelSearch {
|
|
178
|
+
const tried: string[] = [];
|
|
179
|
+
|
|
110
180
|
// 1. Current model if vision-capable
|
|
111
|
-
|
|
181
|
+
const current = ctx.model;
|
|
182
|
+
if (current?.input?.includes("image")) {
|
|
183
|
+
tried.push(`current model ${modelLabel(current)}: accepts image input`);
|
|
184
|
+
return { model: current, source: "current", tried };
|
|
185
|
+
}
|
|
186
|
+
tried.push(
|
|
187
|
+
current
|
|
188
|
+
? `current model ${modelLabel(current)}: ${rejectionReason(current, true)}`
|
|
189
|
+
: "current model: none selected",
|
|
190
|
+
);
|
|
112
191
|
|
|
113
192
|
// 2. The merged role supplies an image-capable entry; a text-only entry in
|
|
114
193
|
// the list is skipped rather than sent an image.
|
|
115
|
-
const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true);
|
|
116
|
-
if (role) return role;
|
|
194
|
+
const role = resolveRoleModel(ctx, UTILITY_MODEL_KEY, true, tried);
|
|
195
|
+
if (role) return { model: role, source: "utilityModel", tried };
|
|
117
196
|
|
|
118
197
|
// 3. First available vision model
|
|
119
198
|
const available = ctx.modelRegistry.getAvailable();
|
|
120
|
-
|
|
199
|
+
const imageCapable = available.filter((m) => m?.input?.includes("image"));
|
|
200
|
+
tried.push(`available models: ${available.length}, ${imageCapable.length} accept image input`);
|
|
201
|
+
const fallback = imageCapable[0];
|
|
202
|
+
if (fallback) {
|
|
203
|
+
tried.push(`using ${modelLabel(fallback)}: first available model that accepts images`);
|
|
204
|
+
return { model: fallback, source: "available", tried };
|
|
205
|
+
}
|
|
206
|
+
return { source: null, tried };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* The model a vision call should use: the `model` of the walk above.
|
|
211
|
+
*/
|
|
212
|
+
export function resolveVisionModel(ctx: ExtensionContext): Model<Api> | undefined {
|
|
213
|
+
return searchVisionModel(ctx).model;
|
|
121
214
|
}
|
|
122
215
|
|
|
123
216
|
/**
|