pi-pocket 0.25.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/AGENTS.md +2 -0
  2. package/README.md +5 -3
  3. package/docs/architecture.md +21 -0
  4. package/docs/design/design-mode.md +108 -0
  5. package/docs/design/notes-a.md +106 -0
  6. package/docs/design/notes-b.md +123 -0
  7. package/docs/design/notes-c.md +92 -0
  8. package/docs/design/notes-d.md +78 -0
  9. package/docs/design/release-notes-design.md +46 -0
  10. package/docs/extensions/browser-modules.md +2 -0
  11. package/docs/extensions/policies/api-stability.md +2 -0
  12. package/docs/extensions/reference/api-versions.md +1 -0
  13. package/docs/extensions/reference/host.md +1 -1
  14. package/docs/extensions/reference/limits.md +4 -0
  15. package/docs/extensions/reference/names.md +2 -2
  16. package/docs/extensions/reference/pocket.md +38 -0
  17. package/docs/features.md +19 -5
  18. package/docs/map.md +75 -54
  19. package/docs/release-notes.md +1 -1
  20. package/docs/self-editing.md +1 -1
  21. package/docs/systemd.md +1 -1
  22. package/npm-shrinkwrap.json +2 -2
  23. package/package.json +1 -1
  24. package/src/launcher/main.ts +20 -9
  25. package/src/server/app.ts +113 -4
  26. package/src/server/background.ts +3 -3
  27. package/src/server/cache-keeper.ts +605 -0
  28. package/src/server/compaction.ts +68 -15
  29. package/src/server/design/bundle.ts +65 -0
  30. package/src/server/design/export.ts +118 -0
  31. package/src/server/design/files.ts +49 -0
  32. package/src/server/design/guide.ts +56 -0
  33. package/src/server/design/index.ts +232 -0
  34. package/src/server/design/pptx.ts +304 -0
  35. package/src/server/design/render.ts +593 -0
  36. package/src/server/design/system.ts +248 -0
  37. package/src/server/docs.ts +55 -0
  38. package/src/server/extensions/artifacts.ts +202 -18
  39. package/src/server/extensions/design/canvas.js +304 -0
  40. package/src/server/extensions/design/comments.js +15 -0
  41. package/src/server/extensions/design/data.js +263 -0
  42. package/src/server/extensions/design/devices.js +70 -0
  43. package/src/server/extensions/design/docks.js +647 -0
  44. package/src/server/extensions/design/edits.js +37 -0
  45. package/src/server/extensions/design/index.js +326 -0
  46. package/src/server/extensions/design/messages.js +92 -0
  47. package/src/server/extensions/design/parts.js +160 -0
  48. package/src/server/extensions/design/state.js +146 -0
  49. package/src/server/extensions/design/style.css +1332 -0
  50. package/src/server/extensions/design/tweaks.js +161 -0
  51. package/src/server/extensions/design/views.js +375 -0
  52. package/src/server/extensions/design/workspace.js +723 -0
  53. package/src/server/extensions/design.ts +620 -0
  54. package/src/server/extensions/plan.ts +11 -9
  55. package/src/server/extensions/prompt.ts +11 -3
  56. package/src/server/extensions/subagents.ts +7 -1
  57. package/src/server/extensions/updates/index.js +16 -6
  58. package/src/server/extensions/updates.ts +2 -2
  59. package/src/server/http/artifact-assets.ts +216 -0
  60. package/src/server/http/artifact-inspect.ts +1281 -0
  61. package/src/server/http/artifacts.ts +89 -14
  62. package/src/server/http/conversation-routes.ts +46 -1
  63. package/src/server/http.ts +12 -4
  64. package/src/server/main.ts +4 -0
  65. package/src/server/npm-install.ts +82 -0
  66. package/src/server/npm-update.ts +100 -0
  67. package/src/server/reload.ts +5 -0
  68. package/src/server/room.ts +2 -0
  69. package/src/server/spend.ts +73 -9
  70. package/src/server/update-state.ts +257 -8
  71. package/src/server/updates.ts +268 -33
  72. package/src/server/zip.ts +163 -0
  73. package/types/pocket.d.ts +32 -0
  74. package/web/activity-fold.js +28 -0
  75. package/web/artifact-bridge.js +506 -82
  76. package/web/extension-ui.js +21 -1
  77. package/web/pip.js +46 -9
  78. package/web/sessions.js +1 -1
  79. package/web/sheets/appearance.js +10 -0
  80. package/web/sheets/viewers.js +26 -1
  81. package/web/style.css +11 -0
  82. package/web/taps.js +26 -0
  83. package/web/theme.js +6 -0
  84. package/web/transcript.js +19 -10
  85. package/src/server/http/public.ts +0 -30
@@ -0,0 +1,605 @@
1
+ /**
2
+ * The cache keeper: while a run waits on a tool call, it re-sends the last model request with a one-token output cap
3
+ * just before its prompt cache entry expires, when that costs less than the expected rebill. Pi's own cache warmer
4
+ * (`cache-warmer.ts` in pi-coding-agent, its "streaming" mode) does this for Pi's CLI; Pi Durable has none.
5
+ *
6
+ * It warms only between the requests of one run: a request that ended in a tool call will be followed by another when
7
+ * the tool ends, so the chance of a real request is high. It never warms an idle session: in the owner's own history
8
+ * a warm during idle time costs about what it saves. It is on by default; `PI_POCKET_CACHE_KEEPER=off` turns it off.
9
+ *
10
+ * At each refresh point: warm when `hazard * missCost > warmCost`.
11
+ * - `warmCost` a cache read of the whole prompt plus one output token.
12
+ * - `missCost` what the next real request pays more if the entry is gone: a cache write instead of a read.
13
+ * - `hazard` the chance that the run's next request comes in the next interval (`TOOL_HAZARD`).
14
+ * A new real request, the run ending or being stopped (`runEnded`), an error, shutdown, or the age cap (`MAX_AGE_MS`,
15
+ * 30 minutes) stops it. An entry whose first refresh would fall after the cap (the 1h tier) is never started: a
16
+ * 30-minute tool wait does not outlive a 1h cache.
17
+ * The cap is the best of 10 to 60 minutes on the owner's own tool-call gaps, and it bounds the loss when a tool waits
18
+ * on a person (an approval).
19
+ *
20
+ * `install` wraps `models.streamSimple` in place, as `timing.ts` and `limit-gate.ts` do. The warm request goes to the
21
+ * wrapped function, so it is not recorded as a real request. Deleting this module and its install line removes it.
22
+ */
23
+ import { createHash } from "node:crypto";
24
+ import { noWait } from "./limit-gate.ts";
25
+ import type {
26
+ Api,
27
+ AssistantMessage,
28
+ AssistantMessageEventStream,
29
+ Model,
30
+ SimpleStreamOptions,
31
+ TranscriptContext,
32
+ } from "@earendil-works/pi-ai";
33
+
34
+ type StreamSimple = (
35
+ model: Model<Api>,
36
+ context: TranscriptContext,
37
+ options?: SimpleStreamOptions,
38
+ ) => AssistantMessageEventStream;
39
+
40
+ const FINAL_PAYLOAD = Symbol("cache-keeper-final-payload");
41
+
42
+ type ObservedOptions = SimpleStreamOptions & { [FINAL_PAYLOAD]?: (payload: unknown) => void };
43
+
44
+ /** Hash the JSON sent to the provider. Only provider output-cap fields may differ on a warm. */
45
+ function payloadHash(payload: unknown): string {
46
+ const caps = new Set(
47
+ [
48
+ ["max_tokens"],
49
+ ["max_completion_tokens"],
50
+ ["max_output_tokens"],
51
+ ["maxTokens"],
52
+ ["options", "maxTokens"],
53
+ ["config", "maxOutputTokens"],
54
+ ["inferenceConfig", "maxTokens"],
55
+ ["generationConfig", "maxOutputTokens"],
56
+ ].map((path) => JSON.stringify(path)),
57
+ );
58
+
59
+ const canonical = (value: unknown, path: string[] = []): unknown => {
60
+ if (Array.isArray(value)) {
61
+ return value.map((item, index) => canonical(item, [...path, String(index)]));
62
+ }
63
+
64
+ if (value !== null && typeof value === "object") {
65
+ return Object.fromEntries(
66
+ Object.entries(value)
67
+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
68
+ .flatMap(([key, item]) => {
69
+ const next = [...path, key];
70
+
71
+ return caps.has(JSON.stringify(next)) ? [] : [[key, canonical(item, next)]];
72
+ }),
73
+ );
74
+ }
75
+
76
+ return value;
77
+ };
78
+
79
+ // Match wire JSON semantics (undefined fields, toJSON), without retaining another prompt copy.
80
+ return createHash("sha256")
81
+ .update(JSON.stringify(canonical(JSON.parse(JSON.stringify(payload)))))
82
+ .digest("hex");
83
+ }
84
+
85
+ /** Refresh at 90% of the lifetime, and never closer than 10 seconds to its end. */
86
+ export function refreshDelayMs(ttlMs: number): number | undefined {
87
+ return ttlMs <= 10_000
88
+ ? undefined
89
+ : Math.max(1, Math.floor(Math.min(ttlMs * 0.9, ttlMs - 10_000)));
90
+ }
91
+
92
+ /** Warming never goes on past this long after the real request that started it. */
93
+ export const MAX_AGE_MS = 30 * 60_000;
94
+ /** Chance of a real request in the next interval after a tool call: the run goes on when the tool ends. */
95
+ export const TOOL_HAZARD = 0.9;
96
+
97
+ export type Rates = { input: number; output: number; cacheRead: number; cacheWrite: number };
98
+ export type Decision = {
99
+ action: "warm" | "stop";
100
+ warmCost: number;
101
+ missCost: number;
102
+ hazard: number;
103
+ expectedSavings: number;
104
+ };
105
+
106
+ /** Dollars. `rates` are per million tokens, as `Model.cost` is. */
107
+ export function decide(input: {
108
+ promptTokens: number;
109
+ rates: Rates;
110
+ /** What a cache write costs per million tokens for the tier in use (the 1h tier costs more than the 5m tier). */
111
+ writeRate: number;
112
+ hazard: number;
113
+ }): Decision {
114
+ const { promptTokens, rates, writeRate, hazard } = input;
115
+ const warmCost = (promptTokens * rates.cacheRead + rates.output) / 1e6;
116
+ const missCost = (promptTokens * (writeRate - rates.cacheRead)) / 1e6;
117
+ const expectedSavings = hazard * missCost - warmCost;
118
+
119
+ return {
120
+ action: expectedSavings > 0 ? "warm" : "stop",
121
+ warmCost,
122
+ missCost,
123
+ hazard,
124
+ expectedSavings,
125
+ };
126
+ }
127
+
128
+ /** Whether a one-token replay leaves the cache entry alone: budget-based thinking would change `budget_tokens`. */
129
+ export function isReplayable(model: Model<Api>, options: SimpleStreamOptions | undefined): boolean {
130
+ if (!options?.reasoning || model.api !== "anthropic-messages") {
131
+ return true;
132
+ }
133
+
134
+ return (
135
+ (model.compat as { forceAdaptiveThinking?: boolean } | undefined)?.forceAdaptiveThinking ===
136
+ true
137
+ );
138
+ }
139
+
140
+ function ttlMsOf(model: Model<Api>, options: SimpleStreamOptions | undefined): number | undefined {
141
+ const long =
142
+ options?.cacheRetention === "long" ||
143
+ (options?.cacheRetention === undefined &&
144
+ (options?.env?.PI_CACHE_RETENTION ?? process.env.PI_CACHE_RETENTION) === "long");
145
+
146
+ if (options?.cacheRetention === "none") {
147
+ return undefined;
148
+ }
149
+
150
+ const seconds = model.promptCache?.[long ? "long" : "short"];
151
+
152
+ return seconds === undefined ? undefined : seconds * 1000;
153
+ }
154
+
155
+ type Entry = {
156
+ model: Model<Api>;
157
+ context: TranscriptContext;
158
+ options: SimpleStreamOptions;
159
+ promptTokens: number;
160
+ ttlMs: number;
161
+ payloadHash: string;
162
+ /** The conversation this session belongs to, once known. */
163
+ conversationId?: number;
164
+ warms: number;
165
+ spent: number;
166
+ /** When the real request ended; the age cap counts from here, not from a refresh. */
167
+ endedAt: number;
168
+ /** When the next refresh should go out; a refresh later than the slack after it would pay a full rewrite. */
169
+ dueAt: number;
170
+ timer?: ReturnType<typeof setTimeout>;
171
+ abort?: AbortController;
172
+ deadline?: ReturnType<typeof setTimeout>;
173
+ account?: WarmAccount;
174
+ };
175
+
176
+ type WarmAccount = {
177
+ /** Check the captured payer's current spend limits immediately before each warm. */
178
+ check: () => boolean;
179
+ /** Count even failed, aborted, or superseded responses when the provider reports usage. */
180
+ recordUsage: (message: AssistantMessage) => Promise<void>;
181
+ };
182
+
183
+ export type KeeperStatus = { warming: number; warms: number; spent: number };
184
+ export type KeeperDeps = {
185
+ now?: () => number;
186
+ setTimer?: (fn: () => void, ms: number) => ReturnType<typeof setTimeout>;
187
+ clearTimer?: (timer: ReturnType<typeof setTimeout>) => void;
188
+ log?: (line: string) => void;
189
+ /** The conversation that sends requests under a provider session id, so `runEnded` can find its warm. */
190
+ conversationOf?: (sessionId: string) => Promise<number | undefined>;
191
+ /** Whether a conversation is in a run now. */
192
+ isRunning?: (conversationId: number) => boolean;
193
+ /** Capture who pays for this run, with its spend check and usage recorder. */
194
+ accountFor?: (conversationId: number) => WarmAccount | undefined;
195
+ /** True turns the keeper off. Read at each decision. */
196
+ disabled?: () => boolean;
197
+ };
198
+
199
+ export class CacheKeeper {
200
+ readonly #deps: Required<Omit<KeeperDeps, "conversationOf" | "isRunning" | "accountFor">> &
201
+ Pick<KeeperDeps, "conversationOf" | "isRunning" | "accountFor">;
202
+ readonly #entries = new Map<string, Entry>();
203
+ readonly #requests = new Map<string, { conversationId?: number }>();
204
+ readonly #pending = new Set<Promise<void>>();
205
+ #underlying: StreamSimple | undefined;
206
+ #closed = false;
207
+ #warms = 0;
208
+ #spent = 0;
209
+
210
+ constructor(deps: KeeperDeps = {}) {
211
+ this.#deps = {
212
+ now: Date.now,
213
+ setTimer: (fn, ms) => {
214
+ const timer = setTimeout(fn, ms);
215
+
216
+ timer.unref();
217
+
218
+ return timer;
219
+ },
220
+ clearTimer: clearTimeout,
221
+ log: () => {},
222
+ disabled: () => process.env.PI_POCKET_CACHE_KEEPER === "off",
223
+ ...deps,
224
+ };
225
+ }
226
+
227
+ status(): KeeperStatus {
228
+ return { warming: this.#entries.size, warms: this.#warms, spent: this.#spent };
229
+ }
230
+
231
+ /** Shutdown: no warm goes out after this. */
232
+ async close(): Promise<void> {
233
+ this.#closed = true;
234
+ this.#requests.clear();
235
+
236
+ for (const key of [...this.#entries.keys()]) {
237
+ this.#drop(key);
238
+ }
239
+
240
+ // Keep the usage listener and storage open until aborted responses have been counted.
241
+ await Promise.allSettled([...this.#pending]);
242
+ }
243
+
244
+ /** Install before request shaping, so this callback sees its final output at the adapter boundary. */
245
+ installPayloadObserver(models: { streamSimple: StreamSimple }): void {
246
+ const underlying = models.streamSimple.bind(models);
247
+
248
+ models.streamSimple = (model, context, options) => {
249
+ const observe = (options as ObservedOptions | undefined)?.[FINAL_PAYLOAD];
250
+
251
+ if (observe === undefined) {
252
+ return underlying(model, context, options);
253
+ }
254
+
255
+ return underlying(model, context, {
256
+ ...options,
257
+ onPayload: async (payload, payloadModel) => {
258
+ const shaped = await options?.onPayload?.(payload, payloadModel);
259
+
260
+ observe(shaped === undefined ? payload : shaped);
261
+
262
+ return shaped;
263
+ },
264
+ });
265
+ };
266
+ }
267
+
268
+ install(models: { streamSimple: StreamSimple }): void {
269
+ const underlying = models.streamSimple.bind(models);
270
+
271
+ this.#underlying = underlying;
272
+ models.streamSimple = (model, context, options) =>
273
+ this.#stream(underlying, model, context, options);
274
+ }
275
+
276
+ #stream(
277
+ underlying: StreamSimple,
278
+ model: Model<Api>,
279
+ context: TranscriptContext,
280
+ options: SimpleStreamOptions | undefined,
281
+ ): AssistantMessageEventStream {
282
+ const key = options?.sessionId;
283
+ const ttlMs = ttlMsOf(model, options);
284
+
285
+ // Uncached calls leave an existing entry alone and need no payload bookkeeping.
286
+ if (key === undefined || ttlMs === undefined) {
287
+ return underlying(model, context, options);
288
+ }
289
+
290
+ const delay = refreshDelayMs(ttlMs);
291
+ const eligible =
292
+ !this.#closed &&
293
+ !this.#deps.disabled() &&
294
+ isReplayable(model, options) &&
295
+ delay !== undefined &&
296
+ delay < MAX_AGE_MS;
297
+ let fingerprint: string | undefined;
298
+ const observed: ObservedOptions = {
299
+ ...options,
300
+ [FINAL_PAYLOAD]: (payload) => {
301
+ try {
302
+ fingerprint = payloadHash(payload);
303
+ } catch {
304
+ // A payload we cannot compare can serve the real request, but must never be warmed.
305
+ fingerprint = undefined;
306
+ }
307
+ },
308
+ };
309
+ const stream = underlying(model, context, eligible ? observed : options);
310
+
311
+ // A real request that writes the cache replaces the entry, so stop warming the old one.
312
+ this.#drop(key);
313
+ const request: { conversationId?: number } = {};
314
+
315
+ this.#requests.set(key, request);
316
+ void this.#deps.conversationOf?.(key).then(
317
+ (id) => {
318
+ if (id !== undefined) {
319
+ request.conversationId = id;
320
+ }
321
+ },
322
+ () => {},
323
+ );
324
+ void stream.result().then(
325
+ (message) => {
326
+ if (this.#requests.get(key) !== request) {
327
+ return;
328
+ }
329
+
330
+ this.#requests.delete(key);
331
+ this.#onSettled(key, model, context, options, message, fingerprint);
332
+ },
333
+ () => {
334
+ if (this.#requests.get(key) === request) {
335
+ this.#requests.delete(key);
336
+ }
337
+ },
338
+ );
339
+
340
+ return stream;
341
+ }
342
+
343
+ #onSettled(
344
+ key: string,
345
+ model: Model<Api>,
346
+ context: TranscriptContext,
347
+ options: SimpleStreamOptions | undefined,
348
+ message: AssistantMessage,
349
+ fingerprint: string | undefined,
350
+ ): void {
351
+ // Only a request that ended in a tool call is followed by another one soon.
352
+ if (
353
+ message.stopReason !== "toolUse" ||
354
+ options === undefined ||
355
+ fingerprint === undefined
356
+ ) {
357
+ return;
358
+ }
359
+
360
+ if (this.#closed || this.#deps.disabled() || !isReplayable(model, options)) {
361
+ return;
362
+ }
363
+
364
+ const ttlMs = ttlMsOf(model, options);
365
+ const usage = message.usage;
366
+ const promptTokens = usage.input + usage.cacheRead + usage.cacheWrite;
367
+ const delay = ttlMs === undefined ? undefined : refreshDelayMs(ttlMs);
368
+
369
+ // The first refresh must fall inside the cap: a longer cache outlives any wait the cap allows.
370
+ if (
371
+ ttlMs === undefined ||
372
+ delay === undefined ||
373
+ delay >= MAX_AGE_MS ||
374
+ promptTokens === 0
375
+ ) {
376
+ return;
377
+ }
378
+
379
+ const now = this.#deps.now();
380
+ const entry: Entry = {
381
+ model,
382
+ context,
383
+ options,
384
+ promptTokens,
385
+ ttlMs,
386
+ payloadHash: fingerprint,
387
+ endedAt: now,
388
+ dueAt: now + delay,
389
+ warms: 0,
390
+ spent: 0,
391
+ };
392
+
393
+ this.#entries.set(key, entry);
394
+ this.#schedule(key, entry, delay);
395
+
396
+ // Which conversation it is, so that the run's end can stop it. It still warms when this is not known.
397
+ void this.#deps.conversationOf?.(key).then(
398
+ (id) => {
399
+ if (id === undefined || this.#entries.get(key) !== entry) {
400
+ return;
401
+ }
402
+
403
+ entry.conversationId = id;
404
+ entry.account = this.#deps.accountFor?.(id);
405
+
406
+ // The run ended before the lookup came back: its end was missed, so stop now.
407
+ if (this.#deps.isRunning?.(id) === false) {
408
+ this.#drop(key);
409
+ }
410
+ },
411
+ () => {},
412
+ );
413
+ }
414
+
415
+ /** A run ended or was stopped: no request follows its last tool call, so its warm stops. */
416
+ runEnded(conversationId: number): void {
417
+ // An unresolved lookup cannot prove which run owns the request. Discard it rather than charge a later run.
418
+ for (const [key, request] of this.#requests) {
419
+ if (request.conversationId === undefined || request.conversationId === conversationId) {
420
+ this.#requests.delete(key);
421
+ }
422
+ }
423
+
424
+ for (const [key, entry] of [...this.#entries]) {
425
+ if (entry.conversationId === undefined || entry.conversationId === conversationId) {
426
+ this.#drop(key);
427
+ }
428
+ }
429
+ }
430
+
431
+ #schedule(key: string, entry: Entry, delay: number): void {
432
+ entry.dueAt = this.#deps.now() + delay;
433
+ entry.timer = this.#deps.setTimer(
434
+ () => {
435
+ const pending = this.#tick(key, entry);
436
+
437
+ this.#pending.add(pending);
438
+ void pending.finally(() => this.#pending.delete(pending));
439
+ },
440
+ Math.min(delay, Math.max(0, entry.endedAt + MAX_AGE_MS - this.#deps.now())),
441
+ );
442
+ }
443
+
444
+ #decision(entry: Entry): Decision {
445
+ // Only the 5m tier is ever warmed (see `#onSettled`), which writes at the model's `cacheWrite` price.
446
+ const writeRate = entry.model.cost.cacheWrite || entry.model.cost.input;
447
+
448
+ return decide({
449
+ promptTokens: entry.promptTokens,
450
+ rates: entry.model.cost,
451
+ writeRate,
452
+ hazard: TOOL_HAZARD,
453
+ });
454
+ }
455
+
456
+ async #tick(key: string, entry: Entry): Promise<void> {
457
+ if (this.#entries.get(key) !== entry) {
458
+ return;
459
+ }
460
+
461
+ const now = this.#deps.now();
462
+ const delay = refreshDelayMs(entry.ttlMs) as number;
463
+ // A timer that fired late (the machine slept) would send its refresh after the entry expired.
464
+ const slack = (entry.ttlMs - delay) / 2;
465
+
466
+ if (
467
+ this.#deps.disabled() ||
468
+ now - entry.endedAt >= MAX_AGE_MS ||
469
+ now > entry.dueAt + slack
470
+ ) {
471
+ return this.#drop(key);
472
+ }
473
+
474
+ if (
475
+ this.#decision(entry).action === "stop" ||
476
+ (this.#deps.accountFor !== undefined && entry.account === undefined) ||
477
+ entry.account?.check() === false
478
+ ) {
479
+ return this.#drop(key);
480
+ }
481
+
482
+ const {
483
+ signal: _signal,
484
+ telemetry: _telemetry,
485
+ ...rest
486
+ } = entry.options as SimpleStreamOptions & {
487
+ telemetry?: unknown;
488
+ };
489
+ const abort = new AbortController();
490
+
491
+ entry.abort = abort;
492
+ entry.deadline = this.#deps.setTimer(
493
+ () => {
494
+ if (this.#entries.get(key) === entry) {
495
+ this.#drop(key);
496
+ }
497
+ },
498
+ entry.endedAt + MAX_AGE_MS - now,
499
+ );
500
+
501
+ let rejected = true;
502
+ const options: ObservedOptions = {
503
+ ...rest,
504
+ maxTokens: 1,
505
+ maxRetries: 0,
506
+ signal: abort.signal,
507
+ [FINAL_PAYLOAD]: (payload) => {
508
+ rejected = true;
509
+ abort.signal.throwIfAborted();
510
+
511
+ let matches = false;
512
+
513
+ try {
514
+ matches = payloadHash(payload) === entry.payloadHash;
515
+ } catch {
516
+ // Fail closed when the shaped payload no longer has a comparable JSON representation.
517
+ }
518
+
519
+ if (!matches) {
520
+ if (this.#entries.get(key) === entry) {
521
+ this.#drop(key);
522
+ }
523
+
524
+ // This callback runs after all shapers, outside their error handlers, before transmission.
525
+ throw new Error("The warm payload differs from the cached request.");
526
+ }
527
+
528
+ rejected = false;
529
+ },
530
+ };
531
+
532
+ try {
533
+ // `noWait`: a warm request that meets a limit fails; it must not queue behind the limit gate.
534
+ const stream = (this.#underlying as StreamSimple)(
535
+ entry.model,
536
+ entry.context,
537
+ noWait(options),
538
+ );
539
+ const message = await stream.result();
540
+
541
+ if (rejected) {
542
+ if (this.#entries.get(key) === entry) {
543
+ this.#drop(key);
544
+ }
545
+
546
+ return;
547
+ }
548
+
549
+ // A request can cost money even when a new request or Stop superseded it.
550
+ await entry.account?.recordUsage(message);
551
+ this.#warms++;
552
+ this.#spent += message.usage.cost.total;
553
+ entry.warms++;
554
+ entry.spent += message.usage.cost.total;
555
+
556
+ if (this.#entries.get(key) !== entry) {
557
+ return;
558
+ }
559
+
560
+ if (message.stopReason === "error" || message.stopReason === "aborted") {
561
+ return this.#drop(key);
562
+ }
563
+
564
+ this.#schedule(key, entry, delay);
565
+ } catch (error) {
566
+ this.#deps.log(
567
+ `Cache keeper stopped for a session: ${error instanceof Error ? error.message : String(error)}`,
568
+ );
569
+
570
+ if (this.#entries.get(key) === entry) {
571
+ this.#drop(key);
572
+ }
573
+ } finally {
574
+ if (entry.deadline !== undefined) {
575
+ this.#deps.clearTimer(entry.deadline);
576
+ }
577
+ }
578
+ }
579
+
580
+ #drop(key: string): void {
581
+ const entry = this.#entries.get(key);
582
+
583
+ if (entry === undefined) {
584
+ return;
585
+ }
586
+
587
+ if (entry.timer !== undefined) {
588
+ this.#deps.clearTimer(entry.timer);
589
+ }
590
+
591
+ if (entry.deadline !== undefined) {
592
+ this.#deps.clearTimer(entry.deadline);
593
+ }
594
+
595
+ // A diagnostic only: production requests also count in the conversation's usage and spend.
596
+ if (entry.warms > 0) {
597
+ this.#deps.log(
598
+ `Cache keeper: ${String(entry.warms)} warm requests for one tool wait, $${entry.spent.toFixed(4)}.`,
599
+ );
600
+ }
601
+
602
+ entry.abort?.abort();
603
+ this.#entries.delete(key);
604
+ }
605
+ }