pi-vision-handoff 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -1
- package/package.json +1 -1
- package/src/dataloader.ts +59 -8
- package/src/describer.ts +151 -20
- package/src/error-log.ts +131 -0
- package/vision-handoff.ts +36 -2
package/README.md
CHANGED
|
@@ -45,6 +45,7 @@ No `settings.json` touched. No per-provider glue. Pick a describer once and ever
|
|
|
45
45
|
- **⚡ Pre-warmed at paste-enter** — the moment you press enter, `before_agent_start` scans the prompt for pasted clipboard image temp-file paths (pi writes pasted images to `<tmpdir>/pi-clipboard-<uuid>.<ext>` and inserts the path as text), reads them, and kicks off the ONE batched vision call concurrent with the agent's first response — so by the time the agent reads the files, they're already cache hits.
|
|
46
46
|
- **🚀 Paste-time prewarm (opt-in)** — `prewarmPastedImages` (off by default) wraps the editor so pasted clipboard images are described the *instant* their path lands in the prompt — before you hit enter — not at submit. The vision call starts concurrent with you typing your question. Tradeoff: the description is generated without your typed question as context (the question usually isn't entered yet at paste time), and a paste-then-abandon wastes one vision call. TUI only; inactive if another extension replaces the editor. Toggle with `/vision-handoff prewarm on`.
|
|
47
47
|
- **🛡️ Graceful degradation** — no API key? Describer unreachable? Aborted? The image is replaced with a clean `[Image: description unavailable]` placeholder instead of crashing your turn.
|
|
48
|
+
- **📋 Error logging** — every describer failure and every `image description failed` warning is appended as a JSONL line to `~/.pi/agent/logs/pi-vision-handoff/errors.log`, so a warning like `image description failed — unknown error` is troubleshootable: the detailed reason (exception stack, `stopReason`, config snapshot) is captured at the describer's failure source even when the in-memory error string was already reset by the time the warning fired. Size-capped with a single `.1` backup.
|
|
48
49
|
- **📊 Usage reporting** — every real describer call reports model + tokens (and Neuralwatt energy/cost when the vision model is a Neuralwatt model), via `pi.appendEntry` + a `vision-handoff:usage` event for live consumers.
|
|
49
50
|
- **🔧 Tunable** — cap description length (`maxDescriptionLines`; unbounded by default, so `read`'s native `ctrl+o` collapse handles compactness) and cache size, in the config file.
|
|
50
51
|
|
|
@@ -100,6 +101,20 @@ Created automatically at `~/.pi/agent/extensions/pi-vision-handoff.json` on firs
|
|
|
100
101
|
|
|
101
102
|
> The config path uses pi's `getAgentDir()` — set `PI_CODING_AGENT_DIR` to relocate it.
|
|
102
103
|
|
|
104
|
+
### Troubleshooting
|
|
105
|
+
|
|
106
|
+
When handoff fails, pi shows a short warning like `pi-vision-handoff: image description failed — <reason>. Vision model: <model>`. The full detail is in the error log:
|
|
107
|
+
|
|
108
|
+
```
|
|
109
|
+
~/.pi/agent/logs/pi-vision-handoff/errors.log
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
(one JSON object per line; relocate via `PI_CODING_AGENT_DIR`). Each entry records the `phase` (`batch` / `single` = describer failure source, `warn` = the user-facing warning), the `reason`, the `visionModel`, the failing `imageHashes`, and — when applicable — the `stopReason`, `errorMessage`, `errorStack`, and a `config` snapshot (`maxTokens` / `thinking` / `thinkingLevel`).
|
|
113
|
+
|
|
114
|
+
**`reason: "unknown error"`** in a `warn` entry means the engine's shared last-error string was already reset (typically a concurrent batch cleared it) by the time the warning fired — so the real cause isn't in the warning. Correlate it with the matching `batch`/`single` entry: match on the `imageHashes` (and the timestamp) to recover the actual exception/`stopReason`. Deliberate user cancels (ESC) are not logged — they aren't errors.
|
|
115
|
+
|
|
116
|
+
The log is append-only and size-capped (10 MB, rotating to `errors.log.1`), so a runaway broken vision model can't fill the disk; truncate it any time.
|
|
117
|
+
|
|
103
118
|
## Installation
|
|
104
119
|
|
|
105
120
|
**With `pi install`** (recommended):
|
|
@@ -372,6 +387,7 @@ pnpm lint:dead # Dead code detection (knip)
|
|
|
372
387
|
│ ├── image.ts # Image hashing, MIME sniffing, clipboard-path reading
|
|
373
388
|
│ ├── prewarm-editor.ts # Opt-in paste-time prewarm CustomEditor wrapper (chains onChange)
|
|
374
389
|
│ ├── dispose.ts # `Disposable` guard factories for `using` (fetch interceptor, timer, abort wire)
|
|
390
|
+
│ ├── error-log.ts # Best-effort JSONL error log → ~/.pi/agent/logs/pi-vision-handoff/errors.log
|
|
375
391
|
│ ├── usage.ts # Describer usage + Neuralwatt energy capture, fetch interceptor
|
|
376
392
|
│ └── vision-model-selector.ts # Interactive picker TUI component
|
|
377
393
|
├── __tests__/unit/
|
|
@@ -379,8 +395,9 @@ pnpm lint:dead # Dead code detection (knip)
|
|
|
379
395
|
│ ├── usage.test.ts # Energy parsing, usage records, concurrency-safe fetch routing
|
|
380
396
|
│ ├── vision-handoff.test.ts # Config, refs, image-block extraction, insertion, truncation, round-trip
|
|
381
397
|
│ ├── dataloader.test.ts # Batch coalescing, memoization, failure eviction, Disposable reset
|
|
382
|
-
│ ├── describer.test.ts # stopReason handling (length → truncation marker, aborted/error)
|
|
398
|
+
│ ├── describer.test.ts # stopReason handling (length → truncation marker, aborted/error), error-log wiring
|
|
383
399
|
│ ├── image.test.ts # MIME sniffing, clipboard-path confinement, file reading, diffPrewarmPaths
|
|
400
|
+
│ ├── error-log.test.ts # JSONL error log: path resolution, append, size-capped rotation, never-throws
|
|
384
401
|
│ └── dispose.test.ts # `using` guards: fetch refcount, timeout, abort-wire propagation
|
|
385
402
|
├── package.json
|
|
386
403
|
├── tsconfig.json
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-vision-handoff",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Give text-only pi models vision — describe images with a vision model you pick via an interactive picker, then hand off the text description to non-vision models",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"author": "Tom X Nguyen",
|
package/src/dataloader.ts
CHANGED
|
@@ -91,19 +91,68 @@ export class DescriptionLoader implements Disposable {
|
|
|
91
91
|
private dispatchScheduled = false;
|
|
92
92
|
private turnModelRegistry: ModelRegistry | null = null;
|
|
93
93
|
private turnVisionModel: Model<Api> | null = null;
|
|
94
|
-
|
|
94
|
+
/** Turn-level abort controller. In-flight describer batches (`runBatch`)
|
|
95
|
+
* are wired to `turnAbortController.signal` so a user cancel (ESC) aborts
|
|
96
|
+
* them EVEN WHEN the batch was dispatched before the run's live abort
|
|
97
|
+
* signal existed — the `before_agent_start` / paste-time prewarm case,
|
|
98
|
+
* where `ctx.signal` is `undefined` (the agent run hasn't started yet).
|
|
99
|
+
* The run's live signal, once it arrives via `bindTurnContext`, is forwarded
|
|
100
|
+
* into `turnAbortController.abort()`. Because `runBatch` holds a reference
|
|
101
|
+
* to this controller's signal (not a snapshot of `ctx.signal`), a batch
|
|
102
|
+
* that started during prewarm becomes abortable the instant a later
|
|
103
|
+
* `bindTurnContext` (from `tool_result`/`context`) brings the live signal.
|
|
104
|
+
* Reset per turn via {@link resetTurnAbort} so a previous turn's cancel
|
|
105
|
+
* can't poison the next turn's prewarm. */
|
|
106
|
+
private turnAbortController = new AbortController();
|
|
107
|
+
/** The run signal currently forwarded into {@link turnAbortController}, so
|
|
108
|
+
* the abort listener is attached at most once per signal (the same signal
|
|
109
|
+
* is bound by every `tool_result`/`context` event in a turn). */
|
|
110
|
+
private wiredSignal: AbortSignal | undefined;
|
|
95
111
|
private pendingTurnPrompt = "";
|
|
96
112
|
|
|
97
113
|
constructor(private readonly deps: LoaderDeps) {}
|
|
98
114
|
|
|
99
|
-
/** Bind the turn context (model registry, resolved vision model, abort
|
|
100
|
-
* for the loader's next dispatch. Called from every handler that
|
|
101
|
-
* `loadDescription()`.
|
|
115
|
+
/** Bind the turn context (model registry, resolved vision model, abort
|
|
116
|
+
* signal) for the loader's next dispatch. Called from every handler that
|
|
117
|
+
* may trigger `loadDescription()`.
|
|
118
|
+
*
|
|
119
|
+
* The abort signal is forwarded into the loader's {@link turnAbortController}
|
|
120
|
+
* rather than stored directly. Storing `ctx.signal` directly would leave a
|
|
121
|
+
* prewarm-dispatched batch (started in `before_agent_start`, where
|
|
122
|
+
* `ctx.signal` is `undefined` because the run hasn't started) with no abort
|
|
123
|
+
* wire — ESC couldn't cancel it, so the `tool_result` handler would only
|
|
124
|
+
* discard the result AFTER the vision call ran to completion. Forwarding
|
|
125
|
+
* the live signal into a stable, loader-owned controller lets an in-flight
|
|
126
|
+
* prewarm batch be aborted the moment the live signal arrives. */
|
|
102
127
|
bindTurnContext(ctx: { modelRegistry: ModelRegistry; signal?: AbortSignal }): void {
|
|
103
128
|
this.turnModelRegistry = ctx.modelRegistry;
|
|
104
|
-
this.turnSignal = ctx.signal;
|
|
105
129
|
const resolved = this.deps.resolveVisionModel(ctx.modelRegistry, this.deps.getConfig().visionModel!);
|
|
106
130
|
if (resolved) this.turnVisionModel = resolved;
|
|
131
|
+
const signal = ctx.signal;
|
|
132
|
+
if (!signal) return;
|
|
133
|
+
if (signal.aborted) {
|
|
134
|
+
this.turnAbortController.abort();
|
|
135
|
+
this.wiredSignal = signal;
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
if (signal === this.wiredSignal) return; // already forwarded this signal
|
|
139
|
+
this.wiredSignal = signal;
|
|
140
|
+
signal.addEventListener("abort", () => this.turnAbortController.abort(), { once: true });
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Reset the turn-level abort controller for a fresh turn. Call at turn
|
|
144
|
+
* boundaries (`before_agent_start`, paste-time prewarm) so a previous turn's
|
|
145
|
+
* cancel doesn't leave {@link turnAbortController} aborted — which would
|
|
146
|
+
* make every subsequent dispatch short-circuit to UNAVAILABLE. A
|
|
147
|
+
* non-aborted controller is reused (avoids orphaning an in-flight paste
|
|
148
|
+
* prewarm holding its signal); only an aborted one is replaced. `wiredSignal`
|
|
149
|
+
* is always cleared so the next live signal re-wires. Safe at a real turn
|
|
150
|
+
* boundary, where the prior turn's batch has settled. */
|
|
151
|
+
resetTurnAbort(): void {
|
|
152
|
+
if (this.turnAbortController.signal.aborted) {
|
|
153
|
+
this.turnAbortController = new AbortController();
|
|
154
|
+
}
|
|
155
|
+
this.wiredSignal = undefined;
|
|
107
156
|
}
|
|
108
157
|
|
|
109
158
|
/** Capture this turn's user prompt so every image in the turn is described in
|
|
@@ -160,7 +209,9 @@ export class DescriptionLoader implements Disposable {
|
|
|
160
209
|
this.dispatchScheduled = false;
|
|
161
210
|
this.turnModelRegistry = null;
|
|
162
211
|
this.turnVisionModel = null;
|
|
163
|
-
|
|
212
|
+
// Fresh controller on session reset (no in-flight batch to orphan).
|
|
213
|
+
this.turnAbortController = new AbortController();
|
|
214
|
+
this.wiredSignal = undefined;
|
|
164
215
|
this.pendingTurnPrompt = "";
|
|
165
216
|
}
|
|
166
217
|
|
|
@@ -194,7 +245,7 @@ export class DescriptionLoader implements Disposable {
|
|
|
194
245
|
for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
|
|
195
246
|
return;
|
|
196
247
|
}
|
|
197
|
-
if (this.
|
|
248
|
+
if (this.turnAbortController.signal.aborted) {
|
|
198
249
|
for (const key of batch.keys) this.cache.delete(key);
|
|
199
250
|
for (const cb of batch.callbacks) cb.resolve(UNAVAILABLE);
|
|
200
251
|
return;
|
|
@@ -209,7 +260,7 @@ export class DescriptionLoader implements Disposable {
|
|
|
209
260
|
this.turnModelRegistry,
|
|
210
261
|
cfg,
|
|
211
262
|
this.deps,
|
|
212
|
-
this.
|
|
263
|
+
this.turnAbortController.signal,
|
|
213
264
|
);
|
|
214
265
|
// Build per-hash results, then fan them out to every callback. This reaches
|
|
215
266
|
// duplicate-hash callbacks that a key-indexed loop would have skipped (and
|
package/src/describer.ts
CHANGED
|
@@ -34,11 +34,13 @@ import {
|
|
|
34
34
|
DEFAULT_USER_PROMPT_PREFIX,
|
|
35
35
|
DEFAULT_VISION_PROMPT,
|
|
36
36
|
describeTimeoutMs,
|
|
37
|
+
formatModelRef,
|
|
37
38
|
markDescriptionTruncated,
|
|
38
39
|
parseBatchedDescriptions,
|
|
39
40
|
type ExtractedImage,
|
|
40
41
|
type VisionHandoffConfig,
|
|
41
42
|
} from "./index.js";
|
|
43
|
+
import { appendVisionError } from "./error-log.js";
|
|
42
44
|
import {
|
|
43
45
|
buildUsageRecord,
|
|
44
46
|
describeAls,
|
|
@@ -104,6 +106,17 @@ export function resolveReasoning(
|
|
|
104
106
|
return cfg.thinkingLevel;
|
|
105
107
|
}
|
|
106
108
|
|
|
109
|
+
/** Small config snapshot embedded in every error-log entry — the fields most
|
|
110
|
+
* relevant to troubleshooting a describer failure (a bad maxTokens clamp, a
|
|
111
|
+
* reasoning level the model rejects, etc.). */
|
|
112
|
+
function configSnapshot(cfg: VisionHandoffConfig): {
|
|
113
|
+
maxTokens?: number;
|
|
114
|
+
thinking: boolean;
|
|
115
|
+
thinkingLevel: string;
|
|
116
|
+
} {
|
|
117
|
+
return { maxTokens: cfg.maxTokens, thinking: cfg.thinking, thinkingLevel: cfg.thinkingLevel };
|
|
118
|
+
}
|
|
119
|
+
|
|
107
120
|
/** Dependencies the describer can't own itself (held by the engine). */
|
|
108
121
|
export interface DescriberDeps {
|
|
109
122
|
/** Report a usage+energy record for one real describer call (cache hits emit none). */
|
|
@@ -138,9 +151,18 @@ export async function runBatch(
|
|
|
138
151
|
const out: BatchResult = new Map();
|
|
139
152
|
const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
|
|
140
153
|
if (!auth.ok || !auth.apiKey) {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
154
|
+
const reason = !auth.ok
|
|
155
|
+
? auth.error
|
|
156
|
+
: `No API key for vision model "${formatModelRef(visionModel.provider, visionModel.id)}"`;
|
|
157
|
+
deps.setLastError(reason);
|
|
158
|
+
appendVisionError({
|
|
159
|
+
phase: "batch",
|
|
160
|
+
reason,
|
|
161
|
+
visionModel: cfg.visionModel,
|
|
162
|
+
imageHashes: misses.map((m) => m.hash),
|
|
163
|
+
imageCount: misses.length,
|
|
164
|
+
config: configSnapshot(cfg),
|
|
165
|
+
});
|
|
144
166
|
return out;
|
|
145
167
|
}
|
|
146
168
|
|
|
@@ -178,7 +200,28 @@ export async function runBatch(
|
|
|
178
200
|
const record = buildUsageRecord(response, capture, visionModel, hashes[0], hashes.length > 1 ? hashes : undefined);
|
|
179
201
|
if (record) deps.reportUsage(record);
|
|
180
202
|
if (response.stopReason === "aborted" || response.stopReason === "error") {
|
|
181
|
-
|
|
203
|
+
const reason = setStopReasonError(
|
|
204
|
+
deps,
|
|
205
|
+
response.stopReason,
|
|
206
|
+
response.errorMessage,
|
|
207
|
+
abortWire,
|
|
208
|
+
timedOut,
|
|
209
|
+
timeoutMs,
|
|
210
|
+
);
|
|
211
|
+
if (reason) {
|
|
212
|
+
appendVisionError({
|
|
213
|
+
phase: "batch",
|
|
214
|
+
reason,
|
|
215
|
+
visionModel: cfg.visionModel,
|
|
216
|
+
imageHashes: hashes,
|
|
217
|
+
imageCount: misses.length,
|
|
218
|
+
stopReason: response.stopReason,
|
|
219
|
+
timedOut,
|
|
220
|
+
timeoutMs,
|
|
221
|
+
errorMessage: response.errorMessage,
|
|
222
|
+
config: configSnapshot(cfg),
|
|
223
|
+
});
|
|
224
|
+
}
|
|
182
225
|
return out;
|
|
183
226
|
}
|
|
184
227
|
const text = response.content
|
|
@@ -188,6 +231,14 @@ export async function runBatch(
|
|
|
188
231
|
.trim();
|
|
189
232
|
if (!text) {
|
|
190
233
|
deps.setLastError("vision model returned an empty description");
|
|
234
|
+
appendVisionError({
|
|
235
|
+
phase: "batch",
|
|
236
|
+
reason: "vision model returned an empty description",
|
|
237
|
+
visionModel: cfg.visionModel,
|
|
238
|
+
imageHashes: hashes,
|
|
239
|
+
imageCount: misses.length,
|
|
240
|
+
config: configSnapshot(cfg),
|
|
241
|
+
});
|
|
191
242
|
return out;
|
|
192
243
|
}
|
|
193
244
|
const parsed = parseBatchedDescriptions(text, misses.length);
|
|
@@ -227,9 +278,28 @@ export async function runBatch(
|
|
|
227
278
|
}
|
|
228
279
|
return out;
|
|
229
280
|
} catch (err) {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
281
|
+
const userAborted = abortWire.userAborted();
|
|
282
|
+
const reason = timedOut
|
|
283
|
+
? `describer timed out after ${timeoutMs / 1000}s`
|
|
284
|
+
: err instanceof Error
|
|
285
|
+
? err.message
|
|
286
|
+
: String(err);
|
|
287
|
+
deps.setLastError(reason);
|
|
288
|
+
// A deliberate user cancel isn't a troubleshooting-worthy error — skip
|
|
289
|
+
// logging it (mirroring the no-warn-on-user-abort contract).
|
|
290
|
+
if (!userAborted) {
|
|
291
|
+
appendVisionError({
|
|
292
|
+
phase: "batch",
|
|
293
|
+
reason,
|
|
294
|
+
visionModel: cfg.visionModel,
|
|
295
|
+
imageHashes: misses.map((m) => m.hash),
|
|
296
|
+
imageCount: misses.length,
|
|
297
|
+
timedOut,
|
|
298
|
+
timeoutMs,
|
|
299
|
+
errorStack: err instanceof Error ? err.stack : undefined,
|
|
300
|
+
config: configSnapshot(cfg),
|
|
301
|
+
});
|
|
302
|
+
}
|
|
233
303
|
return out;
|
|
234
304
|
} finally {
|
|
235
305
|
// The `using` guards above already released the fetch interceptor, timer,
|
|
@@ -254,9 +324,18 @@ export async function describeSingle(
|
|
|
254
324
|
): Promise<string | null> {
|
|
255
325
|
const auth = await modelRegistry.getApiKeyAndHeaders(visionModel);
|
|
256
326
|
if (!auth.ok || !auth.apiKey) {
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
327
|
+
const reason = !auth.ok
|
|
328
|
+
? auth.error
|
|
329
|
+
: `No API key for vision model "${formatModelRef(visionModel.provider, visionModel.id)}"`;
|
|
330
|
+
deps.setLastError(reason);
|
|
331
|
+
appendVisionError({
|
|
332
|
+
phase: "single",
|
|
333
|
+
reason,
|
|
334
|
+
visionModel: cfg.visionModel,
|
|
335
|
+
imageHashes: [imageHash(img.mimeType, img.data)],
|
|
336
|
+
imageCount: 1,
|
|
337
|
+
config: configSnapshot(cfg),
|
|
338
|
+
});
|
|
260
339
|
return null;
|
|
261
340
|
}
|
|
262
341
|
const prefix = cfg.userPromptPrefix ?? DEFAULT_USER_PROMPT_PREFIX;
|
|
@@ -292,7 +371,28 @@ export async function describeSingle(
|
|
|
292
371
|
const record = buildUsageRecord(response, capture, visionModel, hash);
|
|
293
372
|
if (record) deps.reportUsage(record);
|
|
294
373
|
if (response.stopReason === "aborted" || response.stopReason === "error") {
|
|
295
|
-
|
|
374
|
+
const reason = setStopReasonError(
|
|
375
|
+
deps,
|
|
376
|
+
response.stopReason,
|
|
377
|
+
response.errorMessage,
|
|
378
|
+
abortWire,
|
|
379
|
+
timedOut,
|
|
380
|
+
timeoutMs,
|
|
381
|
+
);
|
|
382
|
+
if (reason) {
|
|
383
|
+
appendVisionError({
|
|
384
|
+
phase: "single",
|
|
385
|
+
reason,
|
|
386
|
+
visionModel: cfg.visionModel,
|
|
387
|
+
imageHashes: [hash],
|
|
388
|
+
imageCount: 1,
|
|
389
|
+
stopReason: response.stopReason,
|
|
390
|
+
timedOut,
|
|
391
|
+
timeoutMs,
|
|
392
|
+
errorMessage: response.errorMessage,
|
|
393
|
+
config: configSnapshot(cfg),
|
|
394
|
+
});
|
|
395
|
+
}
|
|
296
396
|
return null;
|
|
297
397
|
}
|
|
298
398
|
const text = response.content
|
|
@@ -302,6 +402,14 @@ export async function describeSingle(
|
|
|
302
402
|
.trim();
|
|
303
403
|
if (!text) {
|
|
304
404
|
deps.setLastError("vision model returned an empty description");
|
|
405
|
+
appendVisionError({
|
|
406
|
+
phase: "single",
|
|
407
|
+
reason: "vision model returned an empty description",
|
|
408
|
+
visionModel: cfg.visionModel,
|
|
409
|
+
imageHashes: [hash],
|
|
410
|
+
imageCount: 1,
|
|
411
|
+
config: configSnapshot(cfg),
|
|
412
|
+
});
|
|
305
413
|
return null;
|
|
306
414
|
}
|
|
307
415
|
// stopReason "length" = the model hit a token limit (configured maxTokens or
|
|
@@ -309,9 +417,28 @@ export async function describeSingle(
|
|
|
309
417
|
// useful, but it must not pass as complete — mark it so the agent/user know.
|
|
310
418
|
return response.stopReason === "length" ? markDescriptionTruncated(text) : text;
|
|
311
419
|
} catch (err) {
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
420
|
+
const userAborted = abortWire.userAborted();
|
|
421
|
+
const reason = timedOut
|
|
422
|
+
? `describer timed out after ${timeoutMs / 1000}s`
|
|
423
|
+
: err instanceof Error
|
|
424
|
+
? err.message
|
|
425
|
+
: String(err);
|
|
426
|
+
deps.setLastError(reason);
|
|
427
|
+
// A deliberate user cancel isn't a troubleshooting-worthy error — skip
|
|
428
|
+
// logging it (mirroring the no-warn-on-user-abort contract).
|
|
429
|
+
if (!userAborted) {
|
|
430
|
+
appendVisionError({
|
|
431
|
+
phase: "single",
|
|
432
|
+
reason,
|
|
433
|
+
visionModel: cfg.visionModel,
|
|
434
|
+
imageHashes: [imageHash(img.mimeType, img.data)],
|
|
435
|
+
imageCount: 1,
|
|
436
|
+
timedOut,
|
|
437
|
+
timeoutMs,
|
|
438
|
+
errorStack: err instanceof Error ? err.stack : undefined,
|
|
439
|
+
config: configSnapshot(cfg),
|
|
440
|
+
});
|
|
441
|
+
}
|
|
315
442
|
return null;
|
|
316
443
|
} finally {
|
|
317
444
|
describeCtx.energyReader?.catch(() => {});
|
|
@@ -332,7 +459,8 @@ async function readCapture(describeCtx: DescribeContext): Promise<VisionHandoffE
|
|
|
332
459
|
|
|
333
460
|
/** Translate a non-OK stopReason into a user-facing failure message, unless the
|
|
334
461
|
* abort came from the user cancelling the turn (then stay silent — no warning
|
|
335
|
-
* for a deliberate cancel).
|
|
462
|
+
* for a deliberate cancel). Returns the message that was set (so the caller
|
|
463
|
+
* can log it), or null when the abort was a user cancel / nothing was set. */
|
|
336
464
|
function setStopReasonError(
|
|
337
465
|
deps: DescriberDeps,
|
|
338
466
|
stopReason: string,
|
|
@@ -340,11 +468,14 @@ function setStopReasonError(
|
|
|
340
468
|
abortWire: AbortWire,
|
|
341
469
|
timedOut: boolean,
|
|
342
470
|
timeoutMs: number,
|
|
343
|
-
):
|
|
344
|
-
if (abortWire.userAborted()) return; // user cancelled the turn — no warning
|
|
471
|
+
): string | null {
|
|
472
|
+
if (abortWire.userAborted()) return null; // user cancelled the turn — no warning
|
|
345
473
|
if (timedOut) {
|
|
346
|
-
|
|
347
|
-
|
|
474
|
+
const reason = `describer timed out after ${timeoutMs / 1000}s`;
|
|
475
|
+
deps.setLastError(reason);
|
|
476
|
+
return reason;
|
|
348
477
|
}
|
|
349
|
-
|
|
478
|
+
const reason = `vision model returned stopReason "${stopReason}"${errorMessage ? ": " + errorMessage : ""}`;
|
|
479
|
+
deps.setLastError(reason);
|
|
480
|
+
return reason;
|
|
350
481
|
}
|
package/src/error-log.ts
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Best-effort structured error logging for pi-vision-handoff.
|
|
3
|
+
*
|
|
4
|
+
* Every describer failure (auth error, network error, abort, empty response,
|
|
5
|
+
* stopReason error/aborted, timeout) and every user-facing "image description
|
|
6
|
+
* failed" warning is appended as one JSONL line to
|
|
7
|
+
* ~/.pi/agent/logs/pi-vision-handoff/errors.log
|
|
8
|
+
* (resolvable via $PI_CODING_AGENT_DIR, like every other pi path).
|
|
9
|
+
*
|
|
10
|
+
* This is the troubleshooting surface for the "image description failed —
|
|
11
|
+
* unknown error" warning. That warning fires with reason "unknown error" when
|
|
12
|
+
* the engine's shared last-error string was already null — typically because a
|
|
13
|
+
* concurrent batch reset it between the failure and the warn — so the detailed
|
|
14
|
+
* reason lives only here, captured at the describer's failure source where the
|
|
15
|
+
* real exception/stopReason is still in hand. A `warn` entry with reason
|
|
16
|
+
* "unknown error" carries the failing image hashes; correlate them (and the
|
|
17
|
+
* timestamp) with the matching `batch`/`single` entry to recover the real
|
|
18
|
+
* cause.
|
|
19
|
+
*
|
|
20
|
+
* Best-effort: logging never throws and never breaks a describer turn. The
|
|
21
|
+
* directory is created lazily; a single size-based rotation (errors.log →
|
|
22
|
+
* errors.log.1) bounds growth to ~2× the cap.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { getAgentDir } from "@earendil-works/pi-coding-agent";
|
|
26
|
+
import { appendFileSync, existsSync, mkdirSync, renameSync, statSync } from "node:fs";
|
|
27
|
+
import { join } from "node:path";
|
|
28
|
+
|
|
29
|
+
/** Subdirectory under the pi agent dir holding this extension's logs. */
|
|
30
|
+
const LOG_SUBDIR = "logs/pi-vision-handoff";
|
|
31
|
+
/** Active log file name. */
|
|
32
|
+
const LOG_FILENAME = "errors.log";
|
|
33
|
+
|
|
34
|
+
/** A single error log is capped at this many bytes before it rotates to the
|
|
35
|
+
* single `.1` backup, bounding total on-disk log to ~2× this. Generous enough
|
|
36
|
+
* to retain a long troubleshooting history; small enough that a runaway
|
|
37
|
+
* broken vision model (failures aren't cached, so they re-fire per turn)
|
|
38
|
+
* can't fill the disk. */
|
|
39
|
+
export const MAX_LOG_BYTES = 10 * 1024 * 1024;
|
|
40
|
+
|
|
41
|
+
/** One structured error-log record (one JSONL line). */
|
|
42
|
+
export interface VisionErrorLogEntry {
|
|
43
|
+
/** ISO 8601 timestamp (filled by {@link appendVisionError}). */
|
|
44
|
+
timestamp: string;
|
|
45
|
+
/** Where the entry originated: "batch"/"single" = describer failure source
|
|
46
|
+
* (rich detail), "warn" = the user-facing warning (may carry reason
|
|
47
|
+
* "unknown error" when the engine's shared error was already reset). */
|
|
48
|
+
phase: "batch" | "single" | "warn";
|
|
49
|
+
/** Human-readable failure reason (what `setLastError` received, or the warn
|
|
50
|
+
* reason). "unknown error" in a `warn` entry means no describer error was
|
|
51
|
+
* captured — correlate via {@link imageHashes} + {@link timestamp} with the
|
|
52
|
+
* matching batch/single entry. */
|
|
53
|
+
reason: string;
|
|
54
|
+
/** Configured vision model ref ("provider/id"), or null if unset. */
|
|
55
|
+
visionModel: string | null;
|
|
56
|
+
/** Image hashes the failure covered (batch/single) or the warning named. */
|
|
57
|
+
imageHashes: string[];
|
|
58
|
+
/** Number of images involved. */
|
|
59
|
+
imageCount: number;
|
|
60
|
+
/** Describer stopReason, when the failure came from a completed response. */
|
|
61
|
+
stopReason?: string;
|
|
62
|
+
/** True when the describer's timeout fired. */
|
|
63
|
+
timedOut?: boolean;
|
|
64
|
+
/** The timeout budget in ms, when timedOut applies. */
|
|
65
|
+
timeoutMs?: number;
|
|
66
|
+
/** Provider error message from the response (response.errorMessage). */
|
|
67
|
+
errorMessage?: string;
|
|
68
|
+
/** Stack trace of a thrown error, when available. */
|
|
69
|
+
errorStack?: string;
|
|
70
|
+
/** Small config snapshot relevant to troubleshooting. */
|
|
71
|
+
config?: { maxTokens?: number; thinking: boolean; thinkingLevel: string };
|
|
72
|
+
/** The model being handed off to ("provider/id"), at the warn point. */
|
|
73
|
+
activeModel?: string;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Directory holding this extension's logs:
|
|
77
|
+
* ~/.pi/agent/logs/pi-vision-handoff (overridable via $PI_CODING_AGENT_DIR). */
|
|
78
|
+
export function getErrorLogDir(): string {
|
|
79
|
+
return join(getAgentDir(), LOG_SUBDIR);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Path to the active error log file. */
|
|
83
|
+
export function getErrorLogPath(): string {
|
|
84
|
+
return join(getErrorLogDir(), LOG_FILENAME);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Serialize an entry to one JSONL line (JSON + terminating newline). Pure and
|
|
88
|
+
* unit-testable independent of the disk. */
|
|
89
|
+
export function formatErrorLogLine(entry: VisionErrorLogEntry): string {
|
|
90
|
+
return JSON.stringify(entry) + "\n";
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Rotate the log when it has grown to at least `maxSize`: the current file
|
|
94
|
+
* becomes the `.1` backup (overwriting any prior backup) and the active path
|
|
95
|
+
* is cleared for a fresh log. Bounds total on-disk log to ~2× `maxSize`.
|
|
96
|
+
* No-op when the file doesn't exist or is under the cap. Never throws. */
|
|
97
|
+
export function rotateLogIfNeeded(path: string = getErrorLogPath(), maxSize: number = MAX_LOG_BYTES): void {
|
|
98
|
+
let size: number;
|
|
99
|
+
try {
|
|
100
|
+
size = statSync(path).size;
|
|
101
|
+
} catch {
|
|
102
|
+
return; // missing file — nothing to rotate
|
|
103
|
+
}
|
|
104
|
+
if (size < maxSize) return;
|
|
105
|
+
try {
|
|
106
|
+
renameSync(path, path + ".1");
|
|
107
|
+
} catch {
|
|
108
|
+
// rename failed (permissions, etc.) — leave the file; it'll keep growing
|
|
109
|
+
// until the condition clears. Logging must not throw.
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Append one structured error entry to the log. Fills `timestamp`, ensures the
|
|
115
|
+
* log directory exists, rotates on size, and writes a JSONL line. Best-effort:
|
|
116
|
+
* swallows every error so a logging failure can NEVER break a describer turn
|
|
117
|
+
* (the describer calls this from its failure paths, where throwing would mask
|
|
118
|
+
* the real error and abort the batch).
|
|
119
|
+
*/
|
|
120
|
+
export function appendVisionError(input: Omit<VisionErrorLogEntry, "timestamp">): void {
|
|
121
|
+
try {
|
|
122
|
+
const dir = getErrorLogDir();
|
|
123
|
+
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
124
|
+
const path = getErrorLogPath();
|
|
125
|
+
rotateLogIfNeeded(path);
|
|
126
|
+
const entry: VisionErrorLogEntry = { timestamp: new Date().toISOString(), ...input };
|
|
127
|
+
appendFileSync(path, formatErrorLogLine(entry), "utf8");
|
|
128
|
+
} catch {
|
|
129
|
+
// never break the describer on a logging failure
|
|
130
|
+
}
|
|
131
|
+
}
|
package/vision-handoff.ts
CHANGED
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
} from "./src/usage.js";
|
|
52
52
|
import { DescriptionLoader, UNAVAILABLE, type LoaderDeps } from "./src/dataloader.js";
|
|
53
53
|
import { imageHash, findClipboardImagePaths, readImageBuffer, resolvePrewarmImage } from "./src/image.js";
|
|
54
|
+
import { appendVisionError } from "./src/error-log.js";
|
|
54
55
|
import { resizeImage } from "@earendil-works/pi-coding-agent";
|
|
55
56
|
import { VisionModelSelectorComponent, type VisionModelSelectorResult } from "./src/vision-model-selector.js";
|
|
56
57
|
import { PrewarmEditor } from "./src/prewarm-editor.js";
|
|
@@ -144,6 +145,13 @@ function prewarmClipboardPath(path: string, modelRegistry: ModelRegistry): void
|
|
|
144
145
|
resolvePrewarmImage(read.buf, read.mimeType, resizeImage)
|
|
145
146
|
.then((img) => {
|
|
146
147
|
if (!img) return;
|
|
148
|
+
// Paste happens while the agent is idle (no run → no live signal).
|
|
149
|
+
// Reset the turn-abort controller so a previous turn's ESC doesn't leave
|
|
150
|
+
// it aborted and short-circuit this prewarm; the next submit's
|
|
151
|
+
// `before_agent_start` resets again. The prewarm batch uses the loader's
|
|
152
|
+
// controller and becomes abortable once a run starts and binds a live
|
|
153
|
+
// signal.
|
|
154
|
+
loader.resetTurnAbort();
|
|
147
155
|
loader.setPendingTurnPrompt("");
|
|
148
156
|
loader.bindTurnContext({ modelRegistry });
|
|
149
157
|
loader.loadDescription(img).catch(() => {});
|
|
@@ -187,14 +195,17 @@ function notifyUnresolvedVisionModel(ctx: ExtensionContext, ref: string): void {
|
|
|
187
195
|
}
|
|
188
196
|
}
|
|
189
197
|
|
|
190
|
-
/** Notify once per failing image per session (dedup via warnedHashes)
|
|
198
|
+
/** Notify once per failing image per session (dedup via warnedHashes), and log
|
|
199
|
+
* EVERY failure (even in headless/SDK mode with no UI) to the error log so the
|
|
200
|
+
* user can troubleshoot. The log entry's `phase: "warn"` carries the failing
|
|
201
|
+
* image hashes and the surfaced reason — when the reason is "unknown error",
|
|
202
|
+
* the real cause lives in the matching `batch`/`single` entry for those hashes. */
|
|
191
203
|
function warnFailedImages(
|
|
192
204
|
ctx: ExtensionContext,
|
|
193
205
|
imgs: ExtractedImage[],
|
|
194
206
|
descs: string[],
|
|
195
207
|
reason: string,
|
|
196
208
|
): void {
|
|
197
|
-
if (!ctx.hasUI) return;
|
|
198
209
|
const newlyFailed: string[] = [];
|
|
199
210
|
for (let i = 0; i < imgs.length; i++) {
|
|
200
211
|
if (descs[i] === UNAVAILABLE) {
|
|
@@ -204,6 +215,17 @@ function warnFailedImages(
|
|
|
204
215
|
}
|
|
205
216
|
if (newlyFailed.length === 0) return;
|
|
206
217
|
for (const h of newlyFailed) warnedHashes.add(h);
|
|
218
|
+
// Always log: troubleshooting must work in headless/SDK mode too, where the
|
|
219
|
+
// notify below never fires.
|
|
220
|
+
appendVisionError({
|
|
221
|
+
phase: "warn",
|
|
222
|
+
reason,
|
|
223
|
+
visionModel: config.visionModel,
|
|
224
|
+
imageHashes: newlyFailed,
|
|
225
|
+
imageCount: newlyFailed.length,
|
|
226
|
+
activeModel: ctx.model ? formatModelRef(ctx.model.provider, ctx.model.id) : undefined,
|
|
227
|
+
});
|
|
228
|
+
if (!ctx.hasUI) return;
|
|
207
229
|
ctx.ui.notify(
|
|
208
230
|
`pi-vision-handoff: image description failed — ${reason}. Vision model: ${config.visionModel}`,
|
|
209
231
|
"warning",
|
|
@@ -248,6 +270,18 @@ export default function (pi: ExtensionAPI) {
|
|
|
248
270
|
if (!isConfigured(config)) return;
|
|
249
271
|
if (!isHandoffTarget(ctx.model, config)) return;
|
|
250
272
|
|
|
273
|
+
// Fresh turn → fresh turn-abort controller. `before_agent_start` fires
|
|
274
|
+
// BEFORE the agent run starts, so `ctx.signal` is undefined here (the run's
|
|
275
|
+
// abort signal doesn't exist yet — it's created in `agent.prompt()` →
|
|
276
|
+
// `runWithLifecycle`, which runs AFTER this event). The prewarm below
|
|
277
|
+
// dispatches describer batches now, and they must be abortable once the
|
|
278
|
+
// run's live signal arrives — so reset the loader's turn-abort controller
|
|
279
|
+
// here, and let the later `tool_result`/`context` binds forward the live
|
|
280
|
+
// signal into it. Without this reset, a previous turn's ESC would leave
|
|
281
|
+
// the controller aborted and every dispatch would short-circuit to
|
|
282
|
+
// UNAVAILABLE.
|
|
283
|
+
loader.resetTurnAbort();
|
|
284
|
+
|
|
251
285
|
// Capture this turn's user prompt so every image in the turn — attached or
|
|
252
286
|
// read via the read tool — is described in the same request context.
|
|
253
287
|
loader.setPendingTurnPrompt(event.prompt || "");
|