pi-cache-optimizer 2.8.16 → 2.8.18

package/src/request-payload.ts

@@ -0,0 +1,197 @@
+import { join } from "node:path";
+import { type PiModel, type UnknownRecord, asRecord, isNonEmptyString, lower } from "./common.ts";
+import { type CacheCompat, getEffectiveCompatSources, resolveEffectiveCompatFromConfig } from "./compat-config.ts";
+import { type PersistedCacheOptimizerConfig, type PersistedCacheOptimizerConfigV3, persistedCacheOptimizerConfig, runtimeOptimizerEnabled } from "./config.ts";
+import { getAssistantRecord, isOfficialOpenAIBaseUrl, isOpenAICompatibleProxyApi, modelKey, readEffectiveCompatConfig } from "./model-identity.ts";
+
+export const OPENAI_PROMPT_CACHE_KEY_MAX_LENGTH = 64;
+
+export function clampPromptCacheKey(key: string | undefined): string | undefined {
+  const normalized = key?.trim();
+  if (!normalized) return undefined;
+
+  const chars = Array.from(normalized);
+  if (chars.length <= OPENAI_PROMPT_CACHE_KEY_MAX_LENGTH) return normalized;
+  return chars.slice(0, OPENAI_PROMPT_CACHE_KEY_MAX_LENGTH).join("");
+}
+
+export function getEffectiveCompatValueSource(
+  model: PiModel,
+  config: unknown,
+  key: keyof CacheCompat,
+): "provider" | "model" | "runtime" | "modelOverride" | undefined {
+  let source: "provider" | "model" | "runtime" | "modelOverride" | undefined;
+  for (const candidate of getEffectiveCompatSources(model, config)) {
+    if (Object.prototype.hasOwnProperty.call(candidate.compat, key)) source = candidate.source;
+  }
+  return source;
+}
+
+export function hasProviderHeader(headers: Record<string, unknown>, name: string): boolean {
+  const normalized = name.toLowerCase();
+  return Object.keys(headers).some((headerName) => headerName.toLowerCase() === normalized);
+}
+
+export function setProviderHeaderIfMissing(
+  headers: Record<string, string | null | undefined>,
+  name: string,
+  value: string,
+): boolean {
+  if (hasProviderHeader(headers, name)) return false;
+  headers[name] = value;
+  return true;
+}
+
+export function addEffectiveSessionAffinityHeaders(
+  headers: Record<string, string | null | undefined>,
+  model: PiModel | undefined,
+  sessionId: string | undefined,
+  effectiveCompat?: CacheCompat,
+  optimizerEnabled: boolean = runtimeOptimizerEnabled,
+  effectiveSource?: "provider" | "model" | "runtime" | "modelOverride",
+): boolean {
+  if (!optimizerEnabled || !model || !isNonEmptyString(sessionId)) return false;
+  if (!isOpenAICompatibleProxyApi(model.api) || !isNonEmptyString(model.baseUrl) || isOfficialOpenAIBaseUrl(model)) return false;
+  const config = effectiveCompat === undefined || effectiveSource === undefined
+    ? readEffectiveCompatConfig()
+    : undefined;
+  const compat = effectiveCompat ?? resolveEffectiveCompatFromConfig(model, config);
+  const source = effectiveSource
+    ?? getEffectiveCompatValueSource(model, config, "sendSessionAffinityHeaders")
+    ?? (Object.prototype.hasOwnProperty.call(asRecord(model.compat) ?? {}, "sendSessionAffinityHeaders") ? "runtime" : undefined);
+  if (compat.sendSessionAffinityHeaders !== true) return false;
+
+  // Pi already handles an effective runtime-model true. The bridge is only for
+  // effective values that live in models.json layers and were lost when an
+  // extension provider rebuilt/replaced the runtime model object.
+  if (source === "runtime") return false;
+
+  const format = compat.sessionAffinityFormat ?? (
+    lower(model.provider).includes("openrouter") || lower(model.baseUrl).includes("openrouter.ai")
+      ? "openrouter"
+      : "openai"
+  );
+  const value = sessionId.trim();
+  let changed = false;
+  if (format === "openrouter") {
+    return setProviderHeaderIfMissing(headers, "x-session-id", value);
+  }
+  if (format === "openai") {
+    changed = setProviderHeaderIfMissing(headers, "session_id", value) || changed;
+  }
+  changed = setProviderHeaderIfMissing(headers, "x-client-request-id", value) || changed;
+  changed = setProviderHeaderIfMissing(headers, "x-session-affinity", value) || changed;
+  return changed;
+}
+
+export function shouldInjectOpenAIPromptCacheKeyForModel(model: PiModel | undefined): boolean {
+  // Pi 1.0+ owns prompt_cache_key for Responses/Codex transports. Keep this
+  // extension's fallback limited to openai-completions proxies, where Pi's
+  // provider extensions may not supply the session key.
+  return isOpenAICompatibleProxyApi(model?.api);
+}
+
+export function isPromptCacheKeyOmittedForModel(model: PiModel | undefined, config: PersistedCacheOptimizerConfig | PersistedCacheOptimizerConfigV3 = persistedCacheOptimizerConfig): boolean {
+  if (!model || !isOpenAICompatibleProxyApi(model.api)) return false;
+  return "promptCacheKey" in config && config.promptCacheKey?.omit?.includes(modelKey(model)) === true;
+}
+
+export function omitOpenAIPromptCacheKeys(payload: unknown): unknown | undefined {
+  const record = asRecord(payload);
+  if (!record || (!Object.prototype.hasOwnProperty.call(record, "prompt_cache_key") && !Object.prototype.hasOwnProperty.call(record, "promptCacheKey"))) return undefined;
+  const copy = { ...record };
+  delete copy.prompt_cache_key;
+  delete copy.promptCacheKey;
+  return copy;
+}
+
+export function collectAnthropicCacheControlsInWireOrder(payload: unknown): UnknownRecord[] {
+  const record = asRecord(payload);
+  if (!record) return [];
+
+  const controls: UnknownRecord[] = [];
+  const collectFromBlock = (value: unknown): void => {
+    const cacheControl = asRecord(asRecord(value)?.cache_control);
+    if (cacheControl?.type === "ephemeral") controls.push(cacheControl);
+  };
+
+  // Anthropic processes cache breakpoints in tools → system → messages order.
+  if (Array.isArray(record.tools)) {
+    for (const tool of record.tools) collectFromBlock(tool);
+  }
+  if (Array.isArray(record.system)) {
+    for (const block of record.system) collectFromBlock(block);
+  }
+  if (Array.isArray(record.messages)) {
+    for (const message of record.messages) {
+      const content = asRecord(message)?.content;
+      if (!Array.isArray(content)) continue;
+      for (const block of content) collectFromBlock(block);
+    }
+  }
+
+  return controls;
+}
+
+/**
+ * Anthropic requires every 1h cache breakpoint to precede every 5m breakpoint.
+ * An ephemeral cache_control without ttl uses Anthropic's default 5m retention.
+ * If the final serialized payload contains a short → long transition, downgrade
+ * all 1h breakpoints to the default 5m so the request remains valid.
+ */
+export function downgradeAnthropicLongCacheControls(payload: unknown): boolean {
+  let changed = false;
+  for (const control of collectAnthropicCacheControlsInWireOrder(payload)) {
+    if (control.ttl === "1h") {
+      delete control.ttl;
+      changed = true;
+    }
+  }
+  return changed;
+}
+
+export function hasAnthropicCacheTtlOrderError(message: unknown): boolean {
+  const record = getAssistantRecord(message);
+  if (record?.stopReason !== "error" || typeof record.errorMessage !== "string") return false;
+
+  const error = lower(record.errorMessage);
+  return error.includes("cache_control") &&
+    error.includes("ttl='1h'") &&
+    error.includes("ttl='5m'") &&
+    error.includes("must not come after");
+}
+
+export function normalizeAnthropicCacheControlTtlOrder(payload: unknown): boolean {
+  const controls = collectAnthropicCacheControlsInWireOrder(payload);
+  let seenShort = false;
+  let hasInvalidLongAfterShort = false;
+
+  for (const control of controls) {
+    const ttl = control.ttl;
+    if (ttl === undefined || ttl === "5m") {
+      seenShort = true;
+    } else if (ttl === "1h" && seenShort) {
+      hasInvalidLongAfterShort = true;
+      break;
+    }
+  }
+  if (!hasInvalidLongAfterShort) return false;
+
+  return downgradeAnthropicLongCacheControls(payload);
+}
+
+export function addOpenAIPromptCacheKey(payload: unknown, cacheKey: string | undefined): unknown | undefined {
+  const record = asRecord(payload);
+  const normalizedCacheKey = clampPromptCacheKey(cacheKey);
+  if (!record || !normalizedCacheKey) return undefined;
+
+  if (hasEffectivePromptCacheKey(record)) {
+    return undefined;
+  }
+
+  return { ...record, prompt_cache_key: normalizedCacheKey };
+}
+
+export function hasEffectivePromptCacheKey(record: UnknownRecord): boolean {
+  return isNonEmptyString(record.prompt_cache_key) || isNonEmptyString(record.promptCacheKey);
+}

package/src/request-state.ts

@@ -0,0 +1,102 @@
+import { type PiModel, isNonEmptyString } from "./common.ts";
+import { type CacheCompat } from "./compat-config.ts";
+import { getCompat } from "./model-identity.ts";
+
+export const REQUEST_SNAPSHOT_COMPAT_KEYS: Array<keyof CacheCompat> = [
+  "supportsStore",
+  "supportsDeveloperRole",
+  "supportsReasoningEffort",
+  "supportsUsageInStreaming",
+  "supportsStrictMode",
+  "maxTokensField",
+  "sendSessionAffinityHeaders",
+  "sessionAffinityFormat",
+  "supportsLongCacheRetention",
+  "thinkingFormat",
+  "requiresReasoningContentOnAssistantMessages",
+  "cacheControlFormat",
+  "forceAdaptiveThinking",
+  "allowEmptySignature",
+];
+
+export function snapshotCompatForDiagnostics(model: PiModel): CacheCompat {
+  const source = getCompat(model);
+  const snapshot: CacheCompat = {};
+  const mutableSnapshot = snapshot as Record<string, unknown>;
+  for (const key of REQUEST_SNAPSHOT_COMPAT_KEYS) {
+    const value = source[key];
+    if (value !== undefined) mutableSnapshot[key] = value;
+  }
+  return snapshot;
+}
+
+export function snapshotBaseUrlForDiagnostics(value: unknown): string {
+  if (!isNonEmptyString(value)) return "";
+  try {
+    const url = new URL(value);
+    if ((url.protocol !== "http:" && url.protocol !== "https:") || !url.hostname) return "";
+    // Request correlation and doctor only need endpoint identity. Strip userinfo and
+    // query/fragment material so a provider URL cannot carry credentials into
+    // the process-local snapshot.
+    url.username = "";
+    url.password = "";
+    url.search = "";
+    url.hash = "";
+    return url.toString();
+  } catch {
+    // Never echo malformed/opaque input: a best-effort string fallback can
+    // retain credentials when the URL could not be parsed.
+    return "";
+  }
+}
+
+export function snapshotProviderRequestModel(model: PiModel | undefined): PiModel | undefined {
+  if (!model) return undefined;
+
+  // The provider response hooks do not receive the model that initiated the
+  // request (Pi's runner only forwards status/headers). Keep a process-local,
+  // credential-blind metadata snapshot so a model switch or route change
+  // between request and response cannot retarget a protocol observation. Do
+  // not retain the caller's complete model object or any request data.
+  return {
+    provider: model.provider,
+    id: model.id,
+    name: model.name,
+    api: model.api,
+    baseUrl: snapshotBaseUrlForDiagnostics(model.baseUrl),
+    compat: snapshotCompatForDiagnostics(model),
+    reasoning: model.reasoning ?? false,
+    input: model.input ?? ["text"],
+    cost: model.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+    contextWindow: model.contextWindow ?? 0,
+    maxTokens: model.maxTokens ?? 0,
+  } as PiModel;
+}
+
+// Some provider requests never produce a finalized assistant message: Pi's
+// prompt-cache warming (0.86+) replays a request with maxTokens: 1 outside the
+// agent loop, so message_end never consumes its lifecycle record. Cap the
+// process-local list and drop the oldest completed record first.
+export const MAX_PROVIDER_REQUEST_STATES = 32;
+
+export function pruneProviderRequestStates<T extends { responseReceived: boolean }>(
+  states: T[],
+  max = MAX_PROVIDER_REQUEST_STATES,
+): void {
+  while (states.length > max) {
+    const completed = states.findIndex((state) => state.responseReceived);
+    states.splice(completed >= 0 ? completed : 0, 1);
+  }
+}
+
+export type ProviderRequestState = {
+  model: PiModel;
+  /** Model selection visible to Pi when this request was dispatched. */
+  selectionModelKey: string;
+  /** Whether that selection routed to a physical model for this request. */
+  virtualSelection: boolean;
+  responseReceived: boolean;
+  correlationAmbiguous: boolean;
+  // A native virtual request whose dispatched id matched several providers.
+  identityAmbiguous?: boolean;
+};

package/src/retention.ts

@@ -0,0 +1,82 @@
+import { type MutableEnv } from "./common.ts";
+
+export type CacheRetentionEnvSnapshot = {
+  wasSet: boolean;
+  value?: string;
+};
+
+export const PI_CACHE_RETENTION_ENV = "PI_CACHE_RETENTION";
+
+export const LONG_CACHE_RETENTION_VALUE = "long";
+
+export const PI_CACHE_RETENTION_BASELINE_SYMBOL = Symbol.for("pi.cache.optimizer.retention-baseline.v1");
+
+export type CacheRetentionBaselineV1 = {
+  version: 1;
+  snapshot: CacheRetentionEnvSnapshot;
+};
+
+export function captureCacheRetentionEnv(env: MutableEnv = process.env): CacheRetentionEnvSnapshot {
+  return {
+    wasSet: Object.prototype.hasOwnProperty.call(env, PI_CACHE_RETENTION_ENV),
+    value: env[PI_CACHE_RETENTION_ENV],
+  };
+}
+
+export function requestLongCacheRetention(env: MutableEnv = process.env): void {
+  if (!env[PI_CACHE_RETENTION_ENV] || env[PI_CACHE_RETENTION_ENV] !== LONG_CACHE_RETENTION_VALUE) {
+    env[PI_CACHE_RETENTION_ENV] = LONG_CACHE_RETENTION_VALUE;
+  }
+}
+
+export function restoreCacheRetentionEnv(snapshot: CacheRetentionEnvSnapshot, env: MutableEnv = process.env): void {
+  if (snapshot.wasSet) {
+    env[PI_CACHE_RETENTION_ENV] = snapshot.value;
+  } else {
+    delete env[PI_CACHE_RETENTION_ENV];
+  }
+}
+
+export function isCacheRetentionBaselineV1(value: unknown): value is CacheRetentionBaselineV1 {
+  if (typeof value !== "object" || value === null) return false;
+  const record = value as { version?: unknown; snapshot?: unknown };
+  if (record.version !== 1 || typeof record.snapshot !== "object" || record.snapshot === null) return false;
+  const snapshot = record.snapshot as { wasSet?: unknown; value?: unknown };
+  return typeof snapshot.wasSet === "boolean" &&
+    (snapshot.value === undefined || typeof snapshot.value === "string");
+}
+
+export function getOrCaptureCacheRetentionBaseline(
+  env: MutableEnv = process.env,
+  globals: Record<symbol, unknown> = globalThis as Record<symbol, unknown>,
+): CacheRetentionEnvSnapshot {
+  const existing = globals[PI_CACHE_RETENTION_BASELINE_SYMBOL];
+  if (isCacheRetentionBaselineV1(existing)) return { ...existing.snapshot };
+
+  const snapshot = captureCacheRetentionEnv(env);
+  globals[PI_CACHE_RETENTION_BASELINE_SYMBOL] = { version: 1, snapshot: { ...snapshot } };
+  return snapshot;
+}
+
+export const STARTUP_CACHE_RETENTION_ENV = getOrCaptureCacheRetentionBaseline();
+
+/**
+ * Pi Cache Optimizer (formerly pi-deepseek-cache-optimizer)
+ *
+ * What it does:
+ * 1. Reorders Pi's system prompt so stable content is sent before dynamic context.
+ * 2. Sets PI_CACHE_RETENTION=long at extension load time.
+ * 3. Warns once for provider/model cache compat gaps where the signal is conservative.
+ * 4. Shows lightweight persisted provider-specific cache stats in Pi's footer.
+ * 5. Offers disabled-by-default deterministic built-in tool ordering when
+ *    explicitly opted in.
+ *
+ * Provider prompt/KV caches are provider-side and best-effort. This extension improves
+ * the odds of cache hits; it cannot guarantee hits, especially through proxies.
+ */
+
+// ============================================================
+// Automatically request long prompt-cache retention when Pi supports it.
+// /cache-optimizer disable restores the startup value for this Pi process.
+// ============================================================
+requestLongCacheRetention();