pi-diagnostics 0.0.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +24 -2
- package/package.json +53 -8
- package/src/core.ts +408 -0
- package/src/expiry-core.ts +734 -0
- package/src/expiry.ts +267 -0
- package/src/index.ts +333 -0
- package/src/native.ts +181 -0
- package/index.js +0 -2
|
@@ -0,0 +1,734 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure logic for the cache-expiry reminder. No runtime imports so it can be
|
|
3
|
+
* unit-tested with plain `node --test`.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export const REMINDER_TYPE = "cache-expiry-reminder";
|
|
7
|
+
|
|
8
|
+
export type Retention = "none" | "short" | "long";
|
|
9
|
+
export type CacheWarmingMode = "off" | "streaming" | "idle";
|
|
10
|
+
|
|
11
|
+
/** Mirrors pi's cache-warmer constants (dist/core/cache-warmer.js). */
|
|
12
|
+
export const PI_IDLE_WARMING_MAX_AGE_MS = 30 * 60_000;
|
|
13
|
+
export const PI_MIN_EXPECTED_SAVINGS = 0.05;
|
|
14
|
+
/** Time allowed for an in-flight pi warm request to land in the session. */
|
|
15
|
+
export const DEFAULT_WARM_GRACE_MS = 20_000;
|
|
16
|
+
|
|
17
|
+
export interface ModelLike {
|
|
18
|
+
provider: string;
|
|
19
|
+
id: string;
|
|
20
|
+
name?: string;
|
|
21
|
+
api?: string;
|
|
22
|
+
baseUrl?: string;
|
|
23
|
+
reasoning?: boolean;
|
|
24
|
+
promptCache?: { short?: number; long?: number };
|
|
25
|
+
compat?: { forceAdaptiveThinking?: boolean } & Record<string, unknown>;
|
|
26
|
+
cost?: { input: number; output: number; cacheRead: number; cacheWrite: number };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface UsageLike {
|
|
30
|
+
input: number;
|
|
31
|
+
output: number;
|
|
32
|
+
cacheRead: number;
|
|
33
|
+
cacheWrite: number;
|
|
34
|
+
cacheWrite1h?: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Minimal session entry shape used by the branch scan. */
|
|
38
|
+
export interface EntryLike {
|
|
39
|
+
id?: string;
|
|
40
|
+
parentId?: string | null;
|
|
41
|
+
type: string;
|
|
42
|
+
timestamp: string;
|
|
43
|
+
customType?: string;
|
|
44
|
+
data?: unknown;
|
|
45
|
+
targetId?: string;
|
|
46
|
+
kind?: string;
|
|
47
|
+
provider?: string;
|
|
48
|
+
model?: string;
|
|
49
|
+
message?: {
|
|
50
|
+
role: string;
|
|
51
|
+
provider?: string;
|
|
52
|
+
model?: string;
|
|
53
|
+
stopReason?: string;
|
|
54
|
+
usage?: UsageLike;
|
|
55
|
+
timestamp?: number;
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
// Payload inspection
|
|
61
|
+
// ---------------------------------------------------------------------------
|
|
62
|
+
|
|
63
|
+
export interface PayloadCacheHints {
|
|
64
|
+
retention?: Retention;
|
|
65
|
+
ttlMs?: number;
|
|
66
|
+
source?: string;
|
|
67
|
+
/** Payload carried a session-derived cache key (`prompt_cache_key`/`promptCacheKey`). */
|
|
68
|
+
sessionKeyed?: boolean;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Parse "5m", "1h", "24h", "300s", "in_memory". */
|
|
72
|
+
export function parseDurationMs(value: unknown): number | undefined {
|
|
73
|
+
if (typeof value === "number" && Number.isFinite(value) && value > 0) return value * 1000;
|
|
74
|
+
if (typeof value !== "string") return undefined;
|
|
75
|
+
const match = /^(\d+(?:\.\d+)?)\s*(s|m|h|d)$/i.exec(value.trim());
|
|
76
|
+
if (!match) return undefined;
|
|
77
|
+
const n = Number(match[1]);
|
|
78
|
+
const unit = match[2]!.toLowerCase();
|
|
79
|
+
const mult = unit === "s" ? 1000 : unit === "m" ? 60_000 : unit === "h" ? 3_600_000 : 86_400_000;
|
|
80
|
+
return n * mult;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
84
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Look at the first/last few items only; cache markers live at the edges. */
|
|
88
|
+
function edgeItems(arr: unknown, n = 4): unknown[] {
|
|
89
|
+
if (!Array.isArray(arr)) return [];
|
|
90
|
+
if (arr.length <= n * 2) return arr;
|
|
91
|
+
return [...arr.slice(0, n), ...arr.slice(-n)];
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function findCacheMarkers(payload: Record<string, unknown>): Record<string, unknown>[] {
|
|
95
|
+
const out: Record<string, unknown>[] = [];
|
|
96
|
+
const visit = (item: unknown, depth: number) => {
|
|
97
|
+
if (!isRecord(item) || depth > 3) return;
|
|
98
|
+
for (const key of ["cache_control", "cachePoint", "cacheControl"]) {
|
|
99
|
+
const marker = item[key];
|
|
100
|
+
if (isRecord(marker)) out.push(marker);
|
|
101
|
+
}
|
|
102
|
+
for (const key of ["content", "system"]) {
|
|
103
|
+
const inner = item[key];
|
|
104
|
+
if (Array.isArray(inner)) for (const child of edgeItems(inner)) visit(child, depth + 1);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
visit(payload, 0);
|
|
108
|
+
for (const key of ["system", "tools", "messages", "input"]) {
|
|
109
|
+
for (const item of edgeItems(payload[key])) visit(item, 1);
|
|
110
|
+
}
|
|
111
|
+
return out;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Extract explicit cache-lifetime signals from the final provider payload
|
|
116
|
+
* (after other extensions' `before_provider_request` transforms).
|
|
117
|
+
*/
|
|
118
|
+
export function inspectPayload(payload: unknown): PayloadCacheHints {
|
|
119
|
+
if (!isRecord(payload)) return {};
|
|
120
|
+
const hints = inspectRetention(payload);
|
|
121
|
+
const key = payload.prompt_cache_key ?? payload.promptCacheKey;
|
|
122
|
+
return typeof key === "string" && key.length > 0 ? { ...hints, sessionKeyed: true } : hints;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function inspectRetention(payload: Record<string, unknown>): PayloadCacheHints {
|
|
126
|
+
// OpenAI Responses explicit prompt-cache mode.
|
|
127
|
+
const options = payload.prompt_cache_options;
|
|
128
|
+
if (isRecord(options)) {
|
|
129
|
+
const ttlMs = parseDurationMs(options.ttl);
|
|
130
|
+
if (ttlMs) return { retention: "long", ttlMs, source: `prompt_cache_options.ttl=${String(options.ttl)}` };
|
|
131
|
+
if (options.mode === "explicit") return { retention: "none", source: "prompt_cache_options.mode=explicit" };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// OpenAI extended retention.
|
|
135
|
+
const retention = payload.prompt_cache_retention;
|
|
136
|
+
if (typeof retention === "string") {
|
|
137
|
+
const ttlMs = parseDurationMs(retention);
|
|
138
|
+
if (ttlMs) return { retention: "long", ttlMs, source: `prompt_cache_retention=${retention}` };
|
|
139
|
+
if (retention === "in_memory") return { retention: "short", source: "prompt_cache_retention=in_memory" };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Anthropic cache_control / Bedrock cachePoint.
|
|
143
|
+
const markers = findCacheMarkers(payload);
|
|
144
|
+
if (markers.length > 0) {
|
|
145
|
+
let best: number | undefined;
|
|
146
|
+
for (const marker of markers) {
|
|
147
|
+
const ttl = parseDurationMs(marker.ttl);
|
|
148
|
+
if (ttl !== undefined && (best === undefined || ttl < best)) best = ttl;
|
|
149
|
+
}
|
|
150
|
+
// Anthropic evicts by the shortest breakpoint; unspecified TTL = 5m.
|
|
151
|
+
const hasDefault = markers.some((m) => m.ttl === undefined);
|
|
152
|
+
if (hasDefault) return { retention: "short", source: "cache_control (default ttl)" };
|
|
153
|
+
if (best !== undefined) {
|
|
154
|
+
return { retention: best > 300_000 ? "long" : "short", ttlMs: best, source: `cache_control.ttl` };
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return {};
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// ---------------------------------------------------------------------------
|
|
161
|
+
// TTL resolution
|
|
162
|
+
// ---------------------------------------------------------------------------
|
|
163
|
+
|
|
164
|
+
export type ModelFamily = "anthropic" | "openai" | "google" | "other";
|
|
165
|
+
|
|
166
|
+
export function modelFamily(model: ModelLike): ModelFamily {
|
|
167
|
+
const hay = `${model.provider}/${model.id}/${model.name ?? ""}`.toLowerCase();
|
|
168
|
+
if (/claude|anthropic/.test(hay)) return "anthropic";
|
|
169
|
+
if (/gemini|google|vertex/.test(hay)) return "google";
|
|
170
|
+
if (/(^|[/\s-])(gpt|o\d|chatgpt|codex)|openai/.test(hay)) return "openai";
|
|
171
|
+
return "other";
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Fallback lifetimes in seconds when model does not declare `promptCache`.
|
|
176
|
+
* These are "likely expired" thresholds, not provider guarantees. Values for
|
|
177
|
+
* OpenAI are calibrated from observed cache-hit rates in local Pi sessions:
|
|
178
|
+
* Responses hits drop sharply after ~30m idle; Codex hits become minority after
|
|
179
|
+
* ~40m. Long retention remains provider's documented 24h tier.
|
|
180
|
+
*/
|
|
181
|
+
export const FAMILY_TTL_SECONDS: Record<ModelFamily, { short: number; long: number; label: string }> = {
|
|
182
|
+
anthropic: { short: 300, long: 3600, label: "Anthropic default" },
|
|
183
|
+
openai: { short: 1800, long: 86_400, label: "OpenAI automatic cache ~30m" },
|
|
184
|
+
google: { short: 300, long: 3600, label: "Gemini implicit cache" },
|
|
185
|
+
other: { short: 300, long: 3600, label: "generic default" },
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* OpenAI protocols show different idle-expiry knees in our session data. Only
|
|
190
|
+
* OpenAI-family models use these; routed Claude/Gemini keep family defaults.
|
|
191
|
+
*/
|
|
192
|
+
const OPENAI_API_TTL_SECONDS: Record<string, { short: number; long: number; label: string }> = {
|
|
193
|
+
"openai-responses": { short: 1800, long: 86_400, label: "OpenAI Responses observed ~30m" },
|
|
194
|
+
"openai-codex-responses": { short: 2400, long: 86_400, label: "OpenAI Codex observed ~40m" },
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
function fallbackTtl(model: ModelLike, retention: Retention): { seconds: number; label: string } {
|
|
198
|
+
const fam = modelFamily(model);
|
|
199
|
+
const api = fam === "openai" && model.api ? OPENAI_API_TTL_SECONDS[model.api] : undefined;
|
|
200
|
+
const family = FAMILY_TTL_SECONDS[fam];
|
|
201
|
+
const source = api ?? family;
|
|
202
|
+
return { seconds: source[retention === "long" ? "long" : "short"], label: source.label };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export interface TtlResolution {
|
|
206
|
+
ttlMs: number;
|
|
207
|
+
retention: Retention;
|
|
208
|
+
source: string;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export interface ResolveTtlInput {
|
|
212
|
+
model: ModelLike;
|
|
213
|
+
hints?: PayloadCacheHints;
|
|
214
|
+
lastUsage?: UsageLike;
|
|
215
|
+
envRetention?: string;
|
|
216
|
+
overrides?: Record<string, number>;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function globToRegExp(glob: string): RegExp {
|
|
220
|
+
const escaped = glob.replace(/[.+^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*").replace(/\?/g, ".");
|
|
221
|
+
return new RegExp(`^${escaped}$`, "i");
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Returns undefined when prompt caching is disabled for the request. */
|
|
225
|
+
export function resolveTtl(input: ResolveTtlInput): TtlResolution | undefined {
|
|
226
|
+
const { model, hints, lastUsage, overrides } = input;
|
|
227
|
+
const key = `${model.provider}/${model.id}`;
|
|
228
|
+
for (const [pattern, seconds] of Object.entries(overrides ?? {})) {
|
|
229
|
+
if (typeof seconds === "number" && seconds > 0 && globToRegExp(pattern).test(key)) {
|
|
230
|
+
return { ttlMs: seconds * 1000, retention: "short", source: `config override ${pattern}` };
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
if (hints?.retention === "none") return undefined;
|
|
235
|
+
if (hints?.ttlMs) return { ttlMs: hints.ttlMs, retention: hints.retention ?? "short", source: hints.source ?? "request" };
|
|
236
|
+
|
|
237
|
+
let retention: Retention = "short";
|
|
238
|
+
let retentionSource = "";
|
|
239
|
+
if (hints?.retention) {
|
|
240
|
+
retention = hints.retention;
|
|
241
|
+
retentionSource = hints.source ?? "request";
|
|
242
|
+
} else if ((lastUsage?.cacheWrite1h ?? 0) > 0) {
|
|
243
|
+
retention = "long";
|
|
244
|
+
retentionSource = "1h cache write observed";
|
|
245
|
+
} else if (input.envRetention === "long") {
|
|
246
|
+
retention = "long";
|
|
247
|
+
retentionSource = "PI_CACHE_RETENTION=long";
|
|
248
|
+
}
|
|
249
|
+
const suffix = retentionSource ? `, ${retentionSource}` : "";
|
|
250
|
+
|
|
251
|
+
const declared = model.promptCache?.[retention === "long" ? "long" : "short"];
|
|
252
|
+
if (declared !== undefined && declared > 0) {
|
|
253
|
+
return { ttlMs: declared * 1000, retention, source: `model promptCache.${retention}${suffix}` };
|
|
254
|
+
}
|
|
255
|
+
// Do not manufacture expiry reminders for providers with no cache billing or
|
|
256
|
+
// observed cache tokens. Dynamic routers often omit promptCache metadata but
|
|
257
|
+
// still expose cacheRead pricing, which is enough for the family fallback.
|
|
258
|
+
const cacheKnown = (model.cost?.cacheRead ?? 0) > 0 ||
|
|
259
|
+
(model.cost?.cacheWrite ?? 0) > 0 ||
|
|
260
|
+
(lastUsage?.cacheRead ?? 0) > 0 ||
|
|
261
|
+
(lastUsage?.cacheWrite ?? 0) > 0;
|
|
262
|
+
if (!cacheKnown) return undefined;
|
|
263
|
+
const fallback = fallbackTtl(model, retention);
|
|
264
|
+
return { ttlMs: fallback.seconds * 1000, retention, source: `heuristic: ${fallback.label}${suffix}` };
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// ---------------------------------------------------------------------------
|
|
268
|
+
// Pi cache warming prediction
|
|
269
|
+
// ---------------------------------------------------------------------------
|
|
270
|
+
|
|
271
|
+
export interface WarmingEligibility {
|
|
272
|
+
mode: CacheWarmingMode;
|
|
273
|
+
/** Pi would arm a warming run for requests with this model/settings. */
|
|
274
|
+
eligible: boolean;
|
|
275
|
+
/** Pi keeps warming after the agent settles. */
|
|
276
|
+
idle: boolean;
|
|
277
|
+
reason?: string;
|
|
278
|
+
ttlMs?: number;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
export function normalizeWarmingMode(value: unknown): CacheWarmingMode {
|
|
282
|
+
return value === "off" || value === "idle" || value === "streaming" ? value : "streaming";
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Replicates pi's `CacheWarmer.start` gating. Economics are observed separately. */
|
|
286
|
+
export function predictPiWarming(
|
|
287
|
+
mode: CacheWarmingMode,
|
|
288
|
+
model: ModelLike,
|
|
289
|
+
thinkingLevel: string | undefined,
|
|
290
|
+
envRetention: string | undefined,
|
|
291
|
+
): WarmingEligibility {
|
|
292
|
+
if (mode === "off") return { mode, eligible: false, idle: false, reason: "cacheWarming=off" };
|
|
293
|
+
const reasoning = !!model.reasoning && thinkingLevel !== undefined && thinkingLevel !== "off";
|
|
294
|
+
if (reasoning && model.api === "anthropic-messages" && model.compat?.forceAdaptiveThinking !== true) {
|
|
295
|
+
return { mode, eligible: false, idle: false, reason: "budget-thinking request not replayable" };
|
|
296
|
+
}
|
|
297
|
+
const retention = envRetention === "long" ? "long" : "short";
|
|
298
|
+
const seconds = model.promptCache?.[retention];
|
|
299
|
+
if (seconds === undefined) {
|
|
300
|
+
return { mode, eligible: false, idle: false, reason: `model has no promptCache.${retention}` };
|
|
301
|
+
}
|
|
302
|
+
if (seconds * 1000 <= 10_000) return { mode, eligible: false, idle: false, reason: "cache lifetime too short" };
|
|
303
|
+
return { mode, eligible: true, idle: mode === "idle", ttlMs: seconds * 1000 };
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// ---------------------------------------------------------------------------
|
|
307
|
+
// Branch scan
|
|
308
|
+
// ---------------------------------------------------------------------------
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* How a provider keeps a cached prefix alive.
|
|
312
|
+
*
|
|
313
|
+
* - `automatic`: prefix/block caches (OpenAI, Gemini implicit, DeepSeek, …).
|
|
314
|
+
* Any request that resends a prefix reuses and refreshes every block of it,
|
|
315
|
+
* so a request anywhere below the fork point keeps the shared prefix warm.
|
|
316
|
+
* - `breakpoint`: Anthropic `cache_control`. Each request writes one entry at
|
|
317
|
+
* its own end and refreshes only the entry it read. Deeper requests on a
|
|
318
|
+
* side branch read *their* newer breakpoints, so only the first request past
|
|
319
|
+
* the fork point re-reads (and refreshes) the entry this branch will hit.
|
|
320
|
+
*/
|
|
321
|
+
export type PrefixCacheMode = "automatic" | "breakpoint";
|
|
322
|
+
|
|
323
|
+
export function prefixCacheMode(model: ModelLike, hints?: PayloadCacheHints): PrefixCacheMode {
|
|
324
|
+
// Protocol wins over model name. Claude sent through OpenAI Responses uses
|
|
325
|
+
// Pi's session-derived key, not Anthropic breakpoint semantics.
|
|
326
|
+
return cacheKeyScope(model, hints) === "session"
|
|
327
|
+
? "automatic"
|
|
328
|
+
: modelFamily(model) === "anthropic" ? "breakpoint" : "automatic";
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* What namespaces a provider cache entry. Decided by wire protocol, not model
|
|
333
|
+
* family: Claude behind an OpenAI Responses router is session-keyed.
|
|
334
|
+
*
|
|
335
|
+
* - `prefix`: cache keyed only by request prefix (Anthropic Messages, Bedrock,
|
|
336
|
+
* Gemini). A `/fork` resends the same prefix, so it inherits the parent's
|
|
337
|
+
* cache and parent activity keeps it warm.
|
|
338
|
+
* - `session`: Pi sends `prompt_cache_key = sessionId` (OpenAI Responses,
|
|
339
|
+
* Codex, Azure, Mistral, OpenAI completions on api.openai.com). `/fork`
|
|
340
|
+
* gets a new session id, so the fork starts in an empty namespace.
|
|
341
|
+
*/
|
|
342
|
+
export type CacheKeyScope = "prefix" | "session";
|
|
343
|
+
|
|
344
|
+
const SESSION_KEYED_APIS = new Set([
|
|
345
|
+
"openai-responses",
|
|
346
|
+
"azure-openai-responses",
|
|
347
|
+
"openai-codex-responses",
|
|
348
|
+
"mistral-conversations",
|
|
349
|
+
]);
|
|
350
|
+
|
|
351
|
+
export function cacheKeyScope(model: ModelLike, hints?: PayloadCacheHints): CacheKeyScope {
|
|
352
|
+
if (hints?.sessionKeyed) return "session";
|
|
353
|
+
if (model.api && SESSION_KEYED_APIS.has(model.api)) return "session";
|
|
354
|
+
if (model.api === "openai-completions" && model.baseUrl?.includes("api.openai.com")) return "session";
|
|
355
|
+
return "prefix";
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
export interface BranchScan {
|
|
359
|
+
lastRequest?: { id?: string; at: number; provider?: string; model?: string; usage?: UsageLike };
|
|
360
|
+
/** Pi cache-warm replays of the prefix this branch will resend. */
|
|
361
|
+
warmAts: number[];
|
|
362
|
+
/**
|
|
363
|
+
* Newest request on another branch that refreshed this branch's cached
|
|
364
|
+
* prefix. Provider caches are keyed by prompt content, not Pi branch.
|
|
365
|
+
* Only requests below this branch's last request qualify: anything that
|
|
366
|
+
* forked earlier shares a strictly shorter prefix, so most of the next
|
|
367
|
+
* request would miss anyway.
|
|
368
|
+
*/
|
|
369
|
+
relatedRequestAt?: number;
|
|
370
|
+
/** `lastTouchAt` recorded by the newest reminder on the branch. */
|
|
371
|
+
remindedForTouchAt?: number;
|
|
372
|
+
/**
|
|
373
|
+
* Start of the current cache namespace (session-keyed caches only). Set when
|
|
374
|
+
* the last request predates it, i.e. was copied in by `/fork` and populated
|
|
375
|
+
* the parent session's namespace, not ours.
|
|
376
|
+
*/
|
|
377
|
+
inheritedFrom?: number;
|
|
378
|
+
/** Compaction/summary, system prompt change, or context edit invalidated the last request's prefix. */
|
|
379
|
+
contextReset: boolean;
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function entryTime(entry: EntryLike): number {
|
|
383
|
+
const t = Date.parse(entry.timestamp);
|
|
384
|
+
return Number.isFinite(t) ? t : 0;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function requestTime(entry: EntryLike): number {
|
|
388
|
+
const ts = entry.message?.timestamp;
|
|
389
|
+
return typeof ts === "number" ? ts : entryTime(entry);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** Assistant entries that correspond to a provider request that processed its prompt. */
|
|
393
|
+
function isRequest(entry: EntryLike): boolean {
|
|
394
|
+
if (entry.type !== "message" || entry.message?.role !== "assistant") return false;
|
|
395
|
+
const m = entry.message;
|
|
396
|
+
const u = m.usage;
|
|
397
|
+
const promptTokens = u ? u.input + u.cacheRead + u.cacheWrite : 0;
|
|
398
|
+
return !(m.stopReason === "error" && promptTokens === 0);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** Entries after which later requests no longer resend the same prefix. */
|
|
402
|
+
function breaksPrefix(entry: EntryLike, sharedIds: ReadonlySet<string>): boolean {
|
|
403
|
+
if (entry.type === "compaction" || entry.type === "branch_summary") return true;
|
|
404
|
+
// Prompt/tool loadout update. Providers without mid-conversation system
|
|
405
|
+
// messages get a whole-transcript checkpoint here, so be conservative.
|
|
406
|
+
if (entry.type === "message" && entry.message?.role === "system") return true;
|
|
407
|
+
// Editing an entry inside the shared prefix rewrites it for later requests.
|
|
408
|
+
if (entry.type === "context_edit" && entry.targetId && sharedIds.has(entry.targetId)) return true;
|
|
409
|
+
return false;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
export interface ScanOptions {
|
|
413
|
+
/** Whole session tree; enables counting activity on other branches. */
|
|
414
|
+
allEntries?: readonly EntryLike[];
|
|
415
|
+
mode?: PrefixCacheMode;
|
|
416
|
+
/**
|
|
417
|
+
* Session-keyed caches: when the current session id came into existence.
|
|
418
|
+
* Requests and warms before it (copied by `/fork`) filled another namespace.
|
|
419
|
+
*/
|
|
420
|
+
namespaceStartAt?: number;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
export function scanBranch(entries: readonly EntryLike[], options: ScanOptions = {}): BranchScan {
|
|
424
|
+
const mode = options.mode ?? "automatic";
|
|
425
|
+
const warm: { at: number; provider?: string }[] = [];
|
|
426
|
+
const editTargets: string[] = [];
|
|
427
|
+
let remindedForTouchAt: number | undefined;
|
|
428
|
+
let lastRequest: BranchScan["lastRequest"];
|
|
429
|
+
let lastRequestIndex = -1;
|
|
430
|
+
let contextReset = false;
|
|
431
|
+
|
|
432
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
433
|
+
const entry = entries[i]!;
|
|
434
|
+
if (entry.type === "custom" && entry.customType === REMINDER_TYPE) {
|
|
435
|
+
if (remindedForTouchAt === undefined) {
|
|
436
|
+
const at = (entry.data as { lastTouchAt?: unknown } | undefined)?.lastTouchAt;
|
|
437
|
+
if (typeof at === "number") remindedForTouchAt = at;
|
|
438
|
+
}
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
if (entry.type === "usage" && entry.kind === "cache_warm") {
|
|
442
|
+
// Warm replays the branch's last request, i.e. exactly our prefix.
|
|
443
|
+
warm.push({ at: entryTime(entry), provider: entry.provider });
|
|
444
|
+
continue;
|
|
445
|
+
}
|
|
446
|
+
if (entry.type === "context_edit" && entry.targetId) {
|
|
447
|
+
editTargets.push(entry.targetId);
|
|
448
|
+
continue;
|
|
449
|
+
}
|
|
450
|
+
if (breaksPrefix(entry, new Set())) {
|
|
451
|
+
contextReset = true;
|
|
452
|
+
break;
|
|
453
|
+
}
|
|
454
|
+
if (isRequest(entry)) {
|
|
455
|
+
const m = entry.message!;
|
|
456
|
+
lastRequest = { id: entry.id, at: requestTime(entry), provider: m.provider, model: m.model, usage: m.usage };
|
|
457
|
+
lastRequestIndex = i;
|
|
458
|
+
break;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// Ids of everything the last request sent (plus its own output).
|
|
463
|
+
const sharedIds = new Set<string>();
|
|
464
|
+
if (lastRequestIndex >= 0) {
|
|
465
|
+
for (let i = 0; i <= lastRequestIndex; i++) {
|
|
466
|
+
const id = entries[i]!.id;
|
|
467
|
+
if (id) sharedIds.add(id);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
// An edit made after the last request to something it had sent changes
|
|
471
|
+
// the prefix the next request resends.
|
|
472
|
+
if (editTargets.some((id) => sharedIds.has(id))) contextReset = true;
|
|
473
|
+
|
|
474
|
+
let relatedRequestAt: number | undefined;
|
|
475
|
+
if (lastRequest?.id && options.allEntries && !contextReset) {
|
|
476
|
+
const onBranch = new Set(entries.map((e) => e.id).filter((id): id is string => !!id));
|
|
477
|
+
const children = new Map<string, EntryLike[]>();
|
|
478
|
+
for (const e of options.allEntries) {
|
|
479
|
+
if (!e.parentId) continue;
|
|
480
|
+
const list = children.get(e.parentId);
|
|
481
|
+
if (list) list.push(e);
|
|
482
|
+
else children.set(e.parentId, [e]);
|
|
483
|
+
}
|
|
484
|
+
const sameModel = (e: EntryLike) =>
|
|
485
|
+
e.message?.provider === lastRequest!.provider && e.message?.model === lastRequest!.model;
|
|
486
|
+
|
|
487
|
+
// Walk only the subtree below the last request: every path there
|
|
488
|
+
// resends the full prefix up to it. `passedRequest` marks paths where a
|
|
489
|
+
// same-model request already moved Anthropic's read point deeper.
|
|
490
|
+
const stack: { entry: EntryLike; passedRequest: boolean }[] =
|
|
491
|
+
(children.get(lastRequest.id) ?? []).map((entry) => ({ entry, passedRequest: false }));
|
|
492
|
+
while (stack.length) {
|
|
493
|
+
const { entry: e, passedRequest } = stack.pop()!;
|
|
494
|
+
if (breaksPrefix(e, sharedIds)) continue;
|
|
495
|
+
const off = !(e.id && onBranch.has(e.id));
|
|
496
|
+
const counts = mode === "automatic" || !passedRequest;
|
|
497
|
+
let next = passedRequest;
|
|
498
|
+
if (e.type === "usage" && e.kind === "cache_warm") {
|
|
499
|
+
// Replays the newest request above it on this path.
|
|
500
|
+
if (off && counts) warm.push({ at: entryTime(e), provider: e.provider });
|
|
501
|
+
} else if (isRequest(e) && sameModel(e)) {
|
|
502
|
+
if (off && counts) {
|
|
503
|
+
const at = requestTime(e);
|
|
504
|
+
if (relatedRequestAt === undefined || at > relatedRequestAt) relatedRequestAt = at;
|
|
505
|
+
}
|
|
506
|
+
next = true;
|
|
507
|
+
}
|
|
508
|
+
if (mode === "breakpoint" && next) {
|
|
509
|
+
// Nothing deeper can refresh our entry; a warm below would replay
|
|
510
|
+
// this deeper request, and later requests read newer breakpoints.
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
for (const child of children.get(e.id ?? "") ?? []) stack.push({ entry: child, passedRequest: next });
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
const nsStart = options.namespaceStartAt;
|
|
518
|
+
if (nsStart !== undefined && relatedRequestAt !== undefined && relatedRequestAt < nsStart) relatedRequestAt = undefined;
|
|
519
|
+
const warmAts = warm
|
|
520
|
+
.filter((w) => !lastRequest?.provider || !w.provider || w.provider === lastRequest.provider)
|
|
521
|
+
.filter((w) => !lastRequest || w.at >= lastRequest.at)
|
|
522
|
+
.filter((w) => nsStart === undefined || w.at >= nsStart)
|
|
523
|
+
.map((w) => w.at)
|
|
524
|
+
.sort((a, b) => a - b);
|
|
525
|
+
const inheritedFrom = nsStart !== undefined && lastRequest && lastRequest.at < nsStart ? nsStart : undefined;
|
|
526
|
+
return { lastRequest, warmAts, relatedRequestAt, remindedForTouchAt, contextReset, inheritedFrom };
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
// ---------------------------------------------------------------------------
|
|
530
|
+
// Expiry evaluation
|
|
531
|
+
// ---------------------------------------------------------------------------
|
|
532
|
+
|
|
533
|
+
export interface WarmingDecisionObservation {
|
|
534
|
+
at: number;
|
|
535
|
+
action: "warm" | "stop";
|
|
536
|
+
expectedSavings: number;
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
export interface EvaluateInput {
|
|
540
|
+
scan: BranchScan;
|
|
541
|
+
model: ModelLike;
|
|
542
|
+
ttl: TtlResolution;
|
|
543
|
+
warming: WarmingEligibility;
|
|
544
|
+
lastDecision?: WarmingDecisionObservation;
|
|
545
|
+
graceMs?: number;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
export interface Evaluation {
|
|
549
|
+
lastRequestAt: number;
|
|
550
|
+
lastTouchAt: number;
|
|
551
|
+
expiresAt: number;
|
|
552
|
+
/** When to (re)check; includes grace for an in-flight pi warm. */
|
|
553
|
+
checkAt: number;
|
|
554
|
+
warmCount: number;
|
|
555
|
+
lastWarmAt?: number;
|
|
556
|
+
warmingSummary: string;
|
|
557
|
+
/** Prompt size of the last request, as counted by that model's tokenizer. */
|
|
558
|
+
promptTokens?: number;
|
|
559
|
+
/** Session-keyed fork that has not sent anything under its own key yet. */
|
|
560
|
+
coldFork?: boolean;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
export function formatClock(ms: number): string {
|
|
564
|
+
const d = new Date(ms);
|
|
565
|
+
return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}`;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
export function formatDuration(ms: number): string {
|
|
569
|
+
const s = Math.round(ms / 1000);
|
|
570
|
+
if (s < 60) return `${s}s`;
|
|
571
|
+
if (s < 3600) return s % 60 === 0 ? `${s / 60}m` : `${Math.floor(s / 60)}m${s % 60}s`;
|
|
572
|
+
if (s < 86_400) return s % 3600 === 0 ? `${s / 3600}h` : `${Math.floor(s / 3600)}h${Math.round((s % 3600) / 60)}m`;
|
|
573
|
+
return `${Math.round(s / 86_400)}d`;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
export function formatTokens(n: number): string {
|
|
577
|
+
if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
|
|
578
|
+
if (n >= 1000) return `${Math.round(n / 1000)}k`;
|
|
579
|
+
return String(n);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/** Plain-English reason for Pi warming being unavailable. */
|
|
583
|
+
function warmingUnavailableReason(reason: string | undefined): string {
|
|
584
|
+
if (!reason) return "not available";
|
|
585
|
+
if (reason === "cacheWarming=off") return "off in settings";
|
|
586
|
+
if (reason.startsWith("model has no promptCache")) return "not available for this model";
|
|
587
|
+
if (reason.includes("not replayable")) return "not available with budget thinking";
|
|
588
|
+
if (reason.includes("too short")) return "not available (cache lifetime too short)";
|
|
589
|
+
return reason;
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/** One short phrase describing what Pi cache warming did for this cache. */
|
|
593
|
+
function summarizeWarming(
|
|
594
|
+
w: WarmingEligibility,
|
|
595
|
+
warmCount: number,
|
|
596
|
+
lastWarmAt: number | undefined,
|
|
597
|
+
lastRequestAt: number,
|
|
598
|
+
lastDecision: WarmingDecisionObservation | undefined,
|
|
599
|
+
): string {
|
|
600
|
+
const refreshed = warmCount > 0 && lastWarmAt !== undefined
|
|
601
|
+
? `refreshed ${warmCount}×, last ${formatClock(lastWarmAt)}`
|
|
602
|
+
: undefined;
|
|
603
|
+
if (!w.eligible) {
|
|
604
|
+
const why = warmingUnavailableReason(w.reason);
|
|
605
|
+
return refreshed ? `${refreshed}, then stopped (${why})` : capitalize(why);
|
|
606
|
+
}
|
|
607
|
+
if (!w.idle) {
|
|
608
|
+
return refreshed
|
|
609
|
+
? `${capitalize(refreshed)}; stops when agent finishes (streaming mode)`
|
|
610
|
+
: "Stops when agent finishes (streaming mode)";
|
|
611
|
+
}
|
|
612
|
+
let why = "";
|
|
613
|
+
if (lastDecision && lastDecision.action === "stop" && lastDecision.at >= lastRequestAt) {
|
|
614
|
+
why = "not worth the cost";
|
|
615
|
+
} else if (lastWarmAt !== undefined && w.ttlMs && lastWarmAt + w.ttlMs >= lastRequestAt + PI_IDLE_WARMING_MAX_AGE_MS) {
|
|
616
|
+
why = "30-min idle limit";
|
|
617
|
+
}
|
|
618
|
+
if (refreshed) return `${capitalize(refreshed)}, then stopped${why ? ` (${why})` : ""}`;
|
|
619
|
+
return why ? `Skipped (${why})` : "Did not refresh";
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
function capitalize(text: string): string {
|
|
623
|
+
return text ? text[0]!.toUpperCase() + text.slice(1) : text;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
/**
|
|
627
|
+
* Warming text is stored in each entry. Rewrite wording saved by earlier
|
|
628
|
+
* versions so old reminders read the same as new ones.
|
|
629
|
+
*/
|
|
630
|
+
export function displayWarming(stored: string): string {
|
|
631
|
+
const inactive = /^pi warming inactive \((.*)\)$/.exec(stored);
|
|
632
|
+
if (inactive) return capitalize(warmingUnavailableReason(inactive[1]));
|
|
633
|
+
if (stored.startsWith("pi idle warming did not refresh")) return "Did not refresh";
|
|
634
|
+
if (stored.startsWith("pi warming in streaming mode")) return "Stops when agent finishes (streaming mode)";
|
|
635
|
+
return capitalize(stored.replace(/^pi (idle )?warming /, ""));
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
export function evaluate(input: EvaluateInput): Evaluation | undefined {
|
|
639
|
+
const { scan, model, ttl, warming } = input;
|
|
640
|
+
if (!scan.lastRequest || scan.contextReset || scan.inheritedFrom !== undefined) return undefined;
|
|
641
|
+
const lastRequestAt = scan.lastRequest.at;
|
|
642
|
+
const lastWarmAt = scan.warmAts.length ? scan.warmAts[scan.warmAts.length - 1] : undefined;
|
|
643
|
+
const lastTouchAt = Math.max(lastRequestAt, lastWarmAt ?? 0, scan.relatedRequestAt ?? 0);
|
|
644
|
+
const expiresAt = lastTouchAt + ttl.ttlMs;
|
|
645
|
+
|
|
646
|
+
// A decision of `stop` means Pi's economics gate declined warming. Do not
|
|
647
|
+
// wait for an in-flight refresh that Pi will not send.
|
|
648
|
+
let checkAt = expiresAt;
|
|
649
|
+
const warmingCouldStillRefresh = warming.eligible &&
|
|
650
|
+
(!input.lastDecision || input.lastDecision.at < lastRequestAt || input.lastDecision.action === "warm");
|
|
651
|
+
if (warmingCouldStillRefresh && warming.idle && lastTouchAt < lastRequestAt + PI_IDLE_WARMING_MAX_AGE_MS) {
|
|
652
|
+
checkAt += input.graceMs ?? DEFAULT_WARM_GRACE_MS;
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
const u = scan.lastRequest.usage;
|
|
656
|
+
const promptTokens = u ? u.input + u.cacheRead + u.cacheWrite : undefined;
|
|
657
|
+
return {
|
|
658
|
+
lastRequestAt,
|
|
659
|
+
lastTouchAt,
|
|
660
|
+
expiresAt,
|
|
661
|
+
checkAt,
|
|
662
|
+
warmCount: scan.warmAts.length,
|
|
663
|
+
lastWarmAt,
|
|
664
|
+
warmingSummary: summarizeWarming(warming, scan.warmAts.length, lastWarmAt, lastRequestAt, input.lastDecision),
|
|
665
|
+
promptTokens,
|
|
666
|
+
};
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
// ---------------------------------------------------------------------------
|
|
670
|
+
// Reminder payload
|
|
671
|
+
// ---------------------------------------------------------------------------
|
|
672
|
+
|
|
673
|
+
export interface ReminderData {
|
|
674
|
+
version: 1;
|
|
675
|
+
model: string;
|
|
676
|
+
lastTouchAt: number;
|
|
677
|
+
expiredAt: number;
|
|
678
|
+
ttlMs: number;
|
|
679
|
+
ttlSource: string;
|
|
680
|
+
warming: string;
|
|
681
|
+
promptTokens?: number;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
export function buildReminder(model: ModelLike, ttl: TtlResolution, ev: Evaluation): ReminderData {
|
|
685
|
+
return {
|
|
686
|
+
version: 1,
|
|
687
|
+
model: `${model.provider}/${model.id}`,
|
|
688
|
+
lastTouchAt: ev.lastTouchAt,
|
|
689
|
+
expiredAt: ev.expiresAt,
|
|
690
|
+
ttlMs: ttl.ttlMs,
|
|
691
|
+
ttlSource: ttl.source,
|
|
692
|
+
warming: ev.warmingSummary,
|
|
693
|
+
promptTokens: ev.promptTokens,
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
/** Short human label for where the TTL came from. */
|
|
698
|
+
export function ttlLabel(source: string): string {
|
|
699
|
+
if (source.startsWith("heuristic:")) return "estimated";
|
|
700
|
+
if (source.startsWith("model promptCache")) return "from model";
|
|
701
|
+
if (source.startsWith("config override")) return "configured";
|
|
702
|
+
if (source.includes("1h cache write")) return "1h cache seen";
|
|
703
|
+
if (source.includes("PI_CACHE_RETENTION")) return "long retention";
|
|
704
|
+
return "from request";
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
function splitModel(model: string): { id: string; provider?: string } {
|
|
708
|
+
const slash = model.indexOf("/");
|
|
709
|
+
return slash < 0 ? { id: model } : { provider: model.slice(0, slash), id: model.slice(slash + 1) };
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
export interface ReminderView {
|
|
713
|
+
/** Headline: what happened and what to do. */
|
|
714
|
+
title: string;
|
|
715
|
+
/** Details shown when expanded, as aligned label/value rows. */
|
|
716
|
+
rows: [label: string, value: string][];
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
export function reminderView(data: ReminderData): ReminderView {
|
|
720
|
+
const { id, provider } = splitModel(data.model);
|
|
721
|
+
// No cost estimate: the cache is already gone, so the re-cache cost is
|
|
722
|
+
// unavoidable, and it would be stale after a model switch (different
|
|
723
|
+
// pricing and tokenizer). Context size still informs /compact or /new.
|
|
724
|
+
const rows: [string, string][] = [
|
|
725
|
+
["Model", provider ? `${id} (${provider})` : id],
|
|
726
|
+
["Cache", `${formatClock(data.lastTouchAt)} → ~${formatClock(data.expiredAt)} · ${formatDuration(data.ttlMs)} TTL (${ttlLabel(data.ttlSource)})`],
|
|
727
|
+
];
|
|
728
|
+
if (data.promptTokens) rows.push(["Context", `${formatTokens(data.promptTokens)} tokens`]);
|
|
729
|
+
rows.push(["Warming", displayWarming(data.warming)]);
|
|
730
|
+
return {
|
|
731
|
+
title: "Prompt cache may have expired. Now is a cheaper time to /compact, switch model, or change tool/skill loadout.",
|
|
732
|
+
rows,
|
|
733
|
+
};
|
|
734
|
+
}
|