pi-cache-optimizer 2.8.27 → 2.9.0

package/README.md

@@ -137,6 +137,8 @@ The fixture verifier reports numeric tool-order changes and confirms cache-marke
 
 Current versions store stats under `pi-cache-optimizer-stats.d/shards/` in Pi's agent directory. Each loaded extension instance owns one active UUID-named daily shard and writes it through temp-file + atomic rename; at local midnight it closes the old shard and starts a new UUID without erasing session history. This prevents parent/child/parallel Pi processes from overwriting one another. Upgrading from the old v6 single-file format deletes the old local stats files and starts footer counters from zero; upstream provider caches are not affected.
 
+Since 2.9.0, maintenance (at most every six hours, under the shared maintenance lock, in the background) folds closed past-day shards, and active ones whose process has been gone for over 24 hours, into a single snapshot at `pi-cache-optimizer-stats.d/archive/archive.json`, then deletes the originals. The shard directory therefore only holds roughly today's shards, so startup and refresh cost no longer grows with the 60-day retention window. Archived counters keep their reset epochs, so `reset` hides them immediately, and they expire on the same 60-day window. The archive records which shards it absorbed, so a crash mid-compaction never double counts. Older versions ignore the archive: if you downgrade or run an older version alongside, its `session` footer shows only history that has not been archived yet; current-day `total` and `process` counters are unaffected.
+
 Response stats stay with the dispatched catalog/request-local model even if you switch to another model on the same provider while waiting. Legacy response aliases use the request snapshot, not the new selection; the footer continues showing the selected direct model. Old corrupt JSON/schema shards expire by file modification time after two months; young corrupt files, valid current-day shards, live old instances, and symlink targets remain protected.
 
 The footer defaults to `session`, which reflects the current Pi conversation rather than another parallel Pi terminal using the same provider/model. Use either the command or environment variable to select the scope:

package/README.zh-CN.md

@@ -137,6 +137,8 @@ bun .trellis/tasks/09-03-context-epoch-tool-ordering/verify.ts
 
 当前版本把统计保存在 Pi agent 目录的 `pi-cache-optimizer-stats.d/shards/` 下。每个已加载的 extension instance 独占一个当天的活跃 UUID shard，并通过临时文件 + 原子 rename 写入；本地日期切换时关闭旧 shard、创建新的 UUID，保留 session 历史，因此父/子/并行 Pi 进程不会互相覆盖。旧 v6 单文件统计在升级时直接删除，本地 footer 计数从零开始；不会影响上游 provider 的实际缓存。
 
+从 2.9.0 起，维护任务会把已关闭的往日 shard，以及进程已退出超过 24 小时的活跃 shard，合并进 `pi-cache-optimizer-stats.d/archive/archive.json` 这一份快照，再删除原文件。维护最多每 6 小时一次，在共享维护锁下后台运行。这样 shard 目录里基本只剩当天的 shard，启动和刷新的开销不再随 60 天保留期增长。归档计数保留 reset epoch，`reset` 立即生效，并按同样的 60 天窗口过期。归档会记录已吸收的 shard，所以合并中途崩溃也不会重复计数。旧版本不会读取归档：如果降级或与旧版本同时运行，旧版的 `session` footer 只显示尚未归档的历史；当天的 `total` 和 `process` 计数不受影响。
+
 即使等待响应时切换到同一 provider 的另一个模型，响应统计仍归属于实际请求的 catalog/request-local 模型。旧 transport 的响应别名只根据请求快照处理，不使用新选择的模型；footer 继续显示当前选择的 direct 模型。损坏 JSON/schema 的旧 shard 按文件修改时间在两个月后清理；较新的损坏文件、有效的当天 shard、仍存活的旧 instance 和 symlink 目标均受保护。
 
 Footer 默认使用 `session`，避免另一个并行 Pi 终端使用相同 provider/model 时污染当前窗口。可以通过命令或环境变量切换显示范围：

package/index.ts

@@ -13,7 +13,8 @@ import { atomicCreateTextFileNoReplace, atomicReplaceTextFilePreservingMode, ato
 import { getAssistantMessageModelTokenValues, getModelIdNameTokenValues, isAdaptiveGenerationModel, isKimiCodingAdaptiveModel } from "./src/model-detect.ts";
 import { findLastExactModelDefinition, mergeCacheCompat, resolveEffectiveCompatFromConfig } from "./src/compat-config.ts";
 import { analyzeModelsJsonForMissingEntry, applyModelsJsonFixTransaction, chooseFixPlacement, composeFixInsertion, composeMissingEntryInsertion, composeModelOverrideInsertion, composeModelsJsonReceiptRollback, composeProviderAffinityInsertion, createModelsJsonFixReceipt, createRollbackBackupPath, decideFixPlacement, formatCompatKeysForInsertion, hasExplicitLongRetentionOptInFromConfig, hasReceiptReasoningProtocolChange, isActionableModelsJsonFixReceipt, locateModelInJsonc, markModelsJsonFixReceiptRolledBack, parseModelsJsonFixReceipt, prepareModelsJsonRollback, readModelsJsonFixReceipt, readModelsJsonFixReceiptSnapshot, receiptBackupPath, resolveExplicitCompatValue, selfCheckFix, selfCheckMissingEntryInsertion, validateModelsJsonRollback, writeModelsJsonFixReceipt } from "./src/models-json-fix.ts";
-import { type CacheProviderId, type CacheStats, LEGACY_STATE_FILE_PATH, type PersistedRoutedModelRef, type PersistedStatsShardV7, SHARD_FILES_DIR, SHARD_GLOBAL_EPOCH_PATH, SHARD_STATE_DIR, STATE_FILE_PATH, type ShardAggregate, type UsageSnapshot, addUsageToCacheStats, advanceGlobalStatsEpoch, advanceModelStatsEpoch, aggregateStatsShardsV7, cleanupStatsShardsV7, cloneCacheStats, mapWithConcurrency, currentLocalDay, emptyAllCacheStats, emptyCacheStats, initialEpoch, loadStatsShardAggregateV7, maybeCleanupStatsShardsV7, modelEpochPath, parseCacheStats, parsePersistedRoutedModelRef, parsePersistedStatsShardV7, readGlobalStatsEpoch, readModelStatsEpoch, readValidStatsShardsV7, removeLegacyStatsFiles, writeStatsShardV7 } from "./src/stats-store.ts";
+import { SHARD_ARCHIVE_PATH, compactStatsShardsV7, maybeRunStatsMaintenance, readStatsArchive } from "./src/stats-archive.ts";
+import { type CacheProviderId, type CacheStats, LEGACY_STATE_FILE_PATH, acquireStatsMaintenanceLock, holdsStatsMaintenanceLock, releaseStatsMaintenanceLock, type PersistedRoutedModelRef, type PersistedStatsShardV7, SHARD_FILES_DIR, SHARD_GLOBAL_EPOCH_PATH, SHARD_RETENTION_MS, SHARD_STATE_DIR, STATE_FILE_PATH, type ShardAggregate, type UsageSnapshot, addUsageToCacheStats, advanceGlobalStatsEpoch, advanceModelStatsEpoch, aggregateStatsShardsV7, cleanupStatsShardsV7, cloneCacheStats, mapWithConcurrency, currentLocalDay, emptyAllCacheStats, emptyCacheStats, initialEpoch, loadStatsShardAggregateV7, modelEpochPath, parseCacheStats, parsePersistedRoutedModelRef, parsePersistedStatsShardV7, readGlobalStatsEpoch, readModelStatsEpoch, readValidStatsShardsV7, removeLegacyStatsFiles, writeStatsShardV7 } from "./src/stats-store.ts";
 import { PI_VIRTUAL_MODEL_API, getCompat, isAnthropicMessagesApi, isKimiCodingEmptySignatureModel, isKnownThirdPartyOpenAIEndpoint, isMistralConversationsApi, isNativeVirtualModel, isOfficialOpenAIBaseUrl, isOpenAICompatibleApi, isOpenAICompatibleProxyApi, isOpenAIFamilyAssistantMessage, isOpenAIFamilyModel, isOpenAIFamilyToken, isPiBuiltInLlamaCppModel, isResponsesPromptRewriteBypassApi, isRoutedFallbackModel, isValidModelsConfigForEffectiveCompat, modelKey, readEffectiveCompatConfig } from "./src/model-identity.ts";
 import { CONFIG_FILE_PATH, VIRTUAL_REWRITE_ENV, FOOTER_MODE_ENV, TOOL_ORDER_ENV, featureEnabled, footerStatsMode, isEnabledEnv, isToolOrderEnabled, parseFooterStatsMode, parsePersistedCacheOptimizerConfig, readPersistedCacheOptimizerConfig, resolveFooterStatsMode, runtimeOptimizerEnabled, setPersistedCacheOptimizerConfig, shouldInjectOpenAIPromptCacheKey, writePersistedCacheOptimizerConfig, writePersistedFeature, writePersistedFooterMode } from "./src/config.ts";
 import { decideZeroPriceWarming, readCacheWarmingMode } from "./src/cache-warming.ts";
@@ -323,6 +324,14 @@ export const __internals_for_tests = {
   aggregateStatsShardsV7,
   loadStatsShardAggregateV7,
   cleanupStatsShardsV7,
+  compactStatsShardsV7,
+  maybeRunStatsMaintenance,
+  readStatsArchive,
+  acquireStatsMaintenanceLock,
+  holdsStatsMaintenanceLock,
+  releaseStatsMaintenanceLock,
+  SHARD_ARCHIVE_PATH,
+  SHARD_RETENTION_MS,
   mapWithConcurrency,
   removeLegacyStatsFiles,
   readGlobalStatsEpoch,
@@ -599,10 +608,21 @@ export default function (pi: ExtensionAPI) {
   }
 
   async function refreshShardAggregate(): Promise<ShardAggregate> {
-    const persistedShards = await readValidStatsShardsV7();
+    // Order matters: compaction publishes the archive before deleting the
+    // shards it absorbed, so reading the directory first and the archive
+    // second always sees each absorbed shard in at least one of them. Reading
+    // them concurrently could pair an old archive with a post-delete listing
+    // and drop those counters (and the last routed model) for this refresh.
+    const rawShards = await readValidStatsShardsV7();
+    const archive = await readStatsArchive();
+    // Shards folded into the archive are counted there. Ignore their raw file
+    // (a crash may leave it behind) and this process's in-memory retired copy.
+    for (const id of archive.absorbedIds) retiredShards.delete(id);
+    const persistedShards = rawShards.filter((shard) => !archive.absorbedIds.has(shard.instanceId));
     for (const shard of persistedShards) retiredShards.delete(shard.instanceId);
     const shards = persistedShards.filter((shard) => shard.instanceId !== instanceId);
     shards.push(...retiredShards.values());
+    shards.push(...archive.shards);
     if (currentSessionHashSet) shards.push(buildCurrentStatsShard());
     // Keep lifetime session footer data separate from daily totals/commands.
     // null explicitly means all retained days; undefined invokes the default day.
@@ -885,7 +905,7 @@ export default function (pi: ExtensionAPI) {
     // no second directory pass is needed. Cleanup also warms the parsed-shard
     // cache, so the scan below only re-stats files.
     try {
-      await maybeCleanupStatsShardsV7();
+      await maybeRunStatsMaintenance();
     } catch (error) {
       console.warn(`${LOG_PREFIX}: stats shard maintenance failed`, error);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-cache-optimizer",
-  "version": "2.8.27",
+  "version": "2.9.0",
   "description": "Improve Pi prompt/KV cache hit rates with stable prompts, OpenAI-compatible cache keys, proxy compat warnings, and footer cache stats.",
   "keywords": [
     "pi-package",

package/src/stats-archive.ts

@@ -0,0 +1,560 @@
+/**
+ * Archive snapshot for closed past-day stats shards (issue #28, phase 2).
+ *
+ * Every Pi process writes one shard per local day. Shards are kept for
+ * SHARD_RETENTION_MS so session-scoped footer totals survive long
+ * conversations, which left ~1,500 files to stat and parse on every refresh.
+ *
+ * Past-day shards that are closed (or abandoned by a dead process) never
+ * change again, so maintenance folds them into one archive file and deletes
+ * the originals. The directory then only holds today's shards.
+ *
+ * Correctness rules:
+ * - Buckets keep the global and model epochs they were written under. Readers
+ *   filter them with the same epoch checks as raw shards, so a reset hides
+ *   archived counters at once, before the next compaction drops them.
+ * - The archive records every instanceId it absorbed. Readers ignore raw
+ *   shards (and in-memory retired shards) with an absorbed id, so a crash
+ *   between publishing the archive and deleting the originals, or two
+ *   processes compacting at once, never double counts.
+ * - Archived data only feeds the all-days (session-scoped) aggregate. Its
+ *   buckets keep their original day, so the current-day aggregate filters
+ *   them out exactly as it filters the raw past-day shards they replace.
+ *   Per-instance counters in the all-days aggregate count archive buckets,
+ *   not processes; only `bySession` and `lastRoutedModelBySession` are read
+ *   from it.
+ */
+import { randomUUID } from "node:crypto";
+import { lstat, mkdir, readFile, readdir, unlink, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { LOG_PREFIX, asRecord, getErrorCode, isNonEmptyString, isProcessAlive } from "./common.ts";
+import {
+  type CacheStats,
+  type PersistedRoutedModelRef,
+  type PersistedStatsShardV7,
+  SHARD_CLEANUP_MARKER_PATH,
+  SHARD_FILES_DIR,
+  SHARD_IO_CONCURRENCY,
+  SHARD_MAINTENANCE_DIR,
+  SHARD_RETENTION_MS,
+  SHARD_STATE_DIR,
+  SHARD_TEMP_RETENTION_MS,
+  acquireStatsMaintenanceLock,
+  cleanupStatsShardsV7,
+  cloneCacheStats,
+  currentLocalDay,
+  isMaintenanceMarkerDue,
+  mapWithConcurrency,
+  mergeStatsAcrossDays,
+  parseCacheStats,
+  parsePersistedRoutedModelRef,
+  holdsStatsMaintenanceLock,
+  readGlobalStatsEpochStrict,
+  readModelStatsEpochStrict,
+  readValidStatsShardsV7,
+  releaseStatsMaintenanceLock,
+  renameWithRetry,
+  writeMaintenanceMarker,
+} from "./stats-store.ts";
+
+export const SHARD_ARCHIVE_DIR = join(SHARD_STATE_DIR, "archive");
+export const SHARD_ARCHIVE_PATH = join(SHARD_ARCHIVE_DIR, "archive.json");
+export const SHARD_COMPACTION_MARKER_PATH = join(SHARD_MAINTENANCE_DIR, "last-compaction");
+
+/**
+ * An active shard is only treated as abandoned once its process is gone and
+ * it has not been touched for this long, so a recycled Windows PID or a
+ * transient process-check failure cannot fold a live shard away.
+ */
+export const ARCHIVE_ABANDONED_ACTIVE_MS = 24 * 60 * 60 * 1000;
+
+export type StatsArchiveBucketV1 = {
+  globalEpoch: string;
+  sessionHash: string;
+  modelKey: string;
+  modelEpoch: string;
+  /** Day of the shards folded into this bucket. */
+  day: string;
+  provider: string;
+  modelId: string;
+  modelName?: string;
+  api?: string;
+  /** Latest lifecycle.updatedAt among the folded shards. */
+  updatedAt: number;
+  stats: CacheStats;
+};
+
+export type StatsArchiveRoutedV1 = {
+  globalEpoch: string;
+  sessionHash: string;
+  day: string;
+  updatedAt: number;
+  model: PersistedRoutedModelRef;
+};
+
+export type StatsArchiveV1 = {
+  version: 1;
+  kind: "pi-cache-optimizer-archive";
+  updatedAt: number;
+  /** instanceId -> time it was folded in. */
+  absorbed: Record<string, number>;
+  buckets: StatsArchiveBucketV1[];
+  routed: StatsArchiveRoutedV1[];
+};
+
+const DAY_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
+
+export function emptyStatsArchive(now = Date.now()): StatsArchiveV1 {
+  return { version: 1, kind: "pi-cache-optimizer-archive", updatedAt: now, absorbed: {}, buckets: [], routed: [] };
+}
+
+function isFiniteNumber(value: unknown): value is number {
+  return typeof value === "number" && Number.isFinite(value);
+}
+
+/** Strict parse: returns undefined when the document is not a v1 archive. Invalid entries are dropped. */
+export function parseStatsArchive(value: unknown): StatsArchiveV1 | undefined {
+  const record = asRecord(value);
+  if (!record || record.version !== 1 || record.kind !== "pi-cache-optimizer-archive") return undefined;
+  const rawAbsorbed = asRecord(record.absorbed);
+  if (!rawAbsorbed || !Array.isArray(record.buckets) || !Array.isArray(record.routed)) return undefined;
+
+  const absorbed: Record<string, number> = {};
+  for (const [id, at] of Object.entries(rawAbsorbed)) {
+    if (isNonEmptyString(id) && isFiniteNumber(at)) absorbed[id] = at;
+  }
+
+  const buckets: StatsArchiveBucketV1[] = [];
+  for (const raw of record.buckets) {
+    const entry = asRecord(raw);
+    const stats = parseCacheStats(entry?.stats);
+    if (
+      !entry || !stats ||
+      !isNonEmptyString(entry.globalEpoch) || !isNonEmptyString(entry.sessionHash) ||
+      !isNonEmptyString(entry.modelKey) || !isNonEmptyString(entry.modelEpoch) ||
+      !isNonEmptyString(entry.provider) || !isNonEmptyString(entry.modelId) ||
+      typeof entry.day !== "string" || !DAY_PATTERN.test(entry.day) ||
+      !isFiniteNumber(entry.updatedAt) ||
+      entry.modelKey !== `${entry.provider}/${entry.modelId}`
+    ) continue;
+    buckets.push({
+      globalEpoch: entry.globalEpoch,
+      sessionHash: entry.sessionHash,
+      modelKey: entry.modelKey,
+      modelEpoch: entry.modelEpoch,
+      day: entry.day,
+      provider: entry.provider,
+      modelId: entry.modelId,
+      ...(isNonEmptyString(entry.modelName) ? { modelName: entry.modelName } : {}),
+      ...(isNonEmptyString(entry.api) ? { api: entry.api } : {}),
+      updatedAt: entry.updatedAt,
+      stats,
+    });
+  }
+
+  const routed: StatsArchiveRoutedV1[] = [];
+  for (const raw of record.routed) {
+    const entry = asRecord(raw);
+    const model = parsePersistedRoutedModelRef(entry?.model);
+    if (
+      !entry || !model ||
+      !isNonEmptyString(entry.globalEpoch) || !isNonEmptyString(entry.sessionHash) ||
+      typeof entry.day !== "string" || !DAY_PATTERN.test(entry.day) ||
+      !isFiniteNumber(entry.updatedAt)
+    ) continue;
+    routed.push({ globalEpoch: entry.globalEpoch, sessionHash: entry.sessionHash, day: entry.day, updatedAt: entry.updatedAt, model });
+  }
+
+  return {
+    version: 1,
+    kind: "pi-cache-optimizer-archive",
+    updatedAt: isFiniteNumber(record.updatedAt) ? record.updatedAt : 0,
+    absorbed,
+    buckets,
+    routed,
+  };
+}
+
+export type LoadedStatsArchive = {
+  /** False when no archive file exists; true for a parsed archive. */
+  present: boolean;
+  /** True when a file exists but cannot be parsed; compaction then refuses to overwrite it. */
+  corrupt: boolean;
+  archive: StatsArchiveV1;
+  absorbedIds: ReadonlySet<string>;
+  /** Archive buckets/routed entries as synthetic shards for aggregateStatsShardsV7. Do not mutate. */
+  shards: readonly PersistedStatsShardV7[];
+};
+
+type CachedArchive = { ino: number | bigint; size: number; mtimeMs: number; loaded: LoadedStatsArchive };
+const archiveCache = new Map<string, CachedArchive>();
+
+export function clearStatsArchiveCache(): void {
+  archiveCache.clear();
+}
+
+/** Convert archive entries into synthetic closed shards so the existing aggregation applies unchanged. */
+export function archiveToSyntheticShards(archive: StatsArchiveV1): PersistedStatsShardV7[] {
+  const shards: PersistedStatsShardV7[] = [];
+  const base = (id: string, sessionHash: string, day: string, globalEpoch: string, updatedAt: number): PersistedStatsShardV7 => ({
+    version: 7,
+    kind: "pi-cache-optimizer-shard",
+    instanceId: id,
+    sessionHash,
+    process: { pid: 0, ppid: 0, instanceStartedAt: 0 },
+    lifecycle: { state: "closed", createdAt: updatedAt, updatedAt, closedAt: updatedAt },
+    day,
+    globalEpoch,
+    models: {},
+  });
+  archive.buckets.forEach((bucket, index) => {
+    const shard = base(`archive-bucket-${index}`, bucket.sessionHash, bucket.day, bucket.globalEpoch, bucket.updatedAt);
+    shard.models[bucket.modelKey] = {
+      modelEpoch: bucket.modelEpoch,
+      provider: bucket.provider,
+      modelId: bucket.modelId,
+      ...(bucket.modelName ? { modelName: bucket.modelName } : {}),
+      ...(bucket.api ? { api: bucket.api } : {}),
+      stats: cloneCacheStats(bucket.stats),
+    };
+    shards.push(shard);
+  });
+  archive.routed.forEach((entry, index) => {
+    const shard = base(`archive-routed-${index}`, entry.sessionHash, entry.day, entry.globalEpoch, entry.updatedAt);
+    shard.lastRoutedModel = { ...entry.model };
+    shards.push(shard);
+  });
+  return shards;
+}
+
+function buildLoaded(archive: StatsArchiveV1, present: boolean, corrupt: boolean): LoadedStatsArchive {
+  return {
+    present,
+    corrupt,
+    archive,
+    absorbedIds: new Set(Object.keys(archive.absorbed)),
+    shards: archiveToSyntheticShards(archive),
+  };
+}
+
+/**
+ * Read the archive, re-parsing only when inode/size/mtime changed. A missing
+ * file is an empty archive; an invalid one is reported as corrupt and treated
+ * as empty, so raw shards stay the source of truth. A transient I/O failure
+ * (Windows EPERM/EBUSY while another process or a scanner holds the file) is
+ * never cached: the last good parse is returned when there is one, otherwise
+ * an uncached corrupt result, so the next refresh retries.
+ */
+export async function readStatsArchive(path: string = SHARD_ARCHIVE_PATH): Promise<LoadedStatsArchive> {
+  const transientFailure = (): LoadedStatsArchive => {
+    const previous = archiveCache.get(path);
+    return previous && !previous.loaded.corrupt ? previous.loaded : buildLoaded(emptyStatsArchive(0), true, true);
+  };
+  let info: import("node:fs").Stats;
+  try {
+    info = await lstat(path);
+  } catch (error) {
+    if (getErrorCode(error) === "ENOENT") {
+      archiveCache.delete(path);
+      return buildLoaded(emptyStatsArchive(0), false, false);
+    }
+    return transientFailure();
+  }
+  if (!info.isFile() || info.isSymbolicLink()) {
+    archiveCache.delete(path);
+    return buildLoaded(emptyStatsArchive(0), true, true);
+  }
+  const cached = archiveCache.get(path);
+  if (cached && cached.ino === info.ino && cached.size === info.size && cached.mtimeMs === info.mtimeMs) return cached.loaded;
+  let text: string;
+  try {
+    text = await readFile(path, "utf8");
+  } catch (error) {
+    // A file renamed away between lstat and read is simply gone.
+    if (getErrorCode(error) === "ENOENT") {
+      archiveCache.delete(path);
+      return buildLoaded(emptyStatsArchive(0), false, false);
+    }
+    return transientFailure();
+  }
+  let loaded: LoadedStatsArchive;
+  try {
+    const parsed = parseStatsArchive(JSON.parse(text));
+    loaded = parsed ? buildLoaded(parsed, true, false) : buildLoaded(emptyStatsArchive(0), true, true);
+  } catch {
+    loaded = buildLoaded(emptyStatsArchive(0), true, true);
+  }
+  archiveCache.set(path, { ino: info.ino, size: info.size, mtimeMs: info.mtimeMs, loaded });
+  return loaded;
+}
+
+type ArchiveFileIdentity = { ino: number | bigint; size: number; mtimeMs: number } | undefined;
+
+async function archiveFileIdentity(path: string): Promise<ArchiveFileIdentity> {
+  try {
+    const info = await lstat(path);
+    return { ino: info.ino, size: info.size, mtimeMs: info.mtimeMs };
+  } catch (error) {
+    if (getErrorCode(error) === "ENOENT") return undefined;
+    throw error;
+  }
+}
+
+function sameArchiveFile(a: ArchiveFileIdentity, b: ArchiveFileIdentity): boolean {
+  if (!a || !b) return a === b;
+  return a.ino === b.ino && a.size === b.size && a.mtimeMs === b.mtimeMs;
+}
+
+/** Remove archive temp files a crash left behind (shard cleanup only covers shards/). */
+async function cleanupStaleArchiveTemps(archivePath: string, now: number): Promise<void> {
+  const directory = dirname(archivePath);
+  let names: string[];
+  try {
+    names = await readdir(directory);
+  } catch {
+    return;
+  }
+  for (const name of names) {
+    if (!name.endsWith(".tmp")) continue;
+    const path = join(directory, name);
+    try {
+      const info = await lstat(path);
+      if (info.isFile() && !info.isSymbolicLink() && now - info.mtimeMs >= SHARD_TEMP_RETENTION_MS) await unlink(path);
+    } catch {}
+  }
+}
+
+async function writeStatsArchive(path: string, archive: StatsArchiveV1): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  const tempPath = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(tempPath, JSON.stringify(archive) + "\n", "utf8");
+    await renameWithRetry(tempPath, path);
+  } catch (error) {
+    await unlink(tempPath).catch(() => undefined);
+    throw error;
+  }
+  archiveCache.delete(path);
+}
+
+/** A shard maintenance may fold away: from a past day and no longer written to. */
+export function isArchivableShard(shard: PersistedStatsShardV7, now: number, today: string): boolean {
+  if (shard.day >= today) return false;
+  if (now - shard.lifecycle.updatedAt >= SHARD_RETENTION_MS) return false; // retention deletes it instead
+  if (shard.lifecycle.state === "closed") return true;
+  return now - shard.lifecycle.updatedAt >= ARCHIVE_ABANDONED_ACTIVE_MS && !isProcessAlive(shard.process.pid);
+}
+
+export type CompactionResult = {
+  /** Raw shards newly folded into the archive. */
+  absorbed: number;
+  /** Raw shard files deleted (newly or previously absorbed). */
+  removed: number;
+  /** Archive entries dropped for an old epoch or past retention. */
+  dropped: number;
+  /** True when an unreadable archive blocked compaction. */
+  skippedCorrupt: boolean;
+  /**
+   * True when the archive, an epoch, or lock ownership changed while this
+   * pass ran (e.g. a stale-lock takeover or a concurrent reset). Nothing was
+   * published or deleted; the next pass retries.
+   */
+  skippedConcurrent: boolean;
+};
+
+export type CompactionOptions = {
+  /** Owner token from acquireStatsMaintenanceLock; ownership is re-checked before publishing. */
+  lockToken?: string;
+  /** Test hook: runs after the plan is built, right before the publish checks. */
+  beforePublish?: () => Promise<void>;
+};
+
+const bucketId = (b: Pick<StatsArchiveBucketV1, "globalEpoch" | "sessionHash" | "modelKey" | "modelEpoch" | "day">): string =>
+  JSON.stringify([b.globalEpoch, b.sessionHash, b.modelKey, b.modelEpoch, b.day]);
+const routedId = (r: Pick<StatsArchiveRoutedV1, "globalEpoch" | "sessionHash">): string =>
+  JSON.stringify([r.globalEpoch, r.sessionHash]);
+
+/**
+ * Fold archivable shards into the archive, then delete them. Call only while
+ * holding the maintenance lock. Order matters for crash safety: the archive
+ * (with the absorbed ids) is published before any raw shard is deleted.
+ */
+export async function compactStatsShardsV7(
+  now = Date.now(),
+  directory: string = SHARD_FILES_DIR,
+  archivePath: string = SHARD_ARCHIVE_PATH,
+  options: CompactionOptions = {},
+): Promise<CompactionResult> {
+  const result: CompactionResult = { absorbed: 0, removed: 0, dropped: 0, skippedCorrupt: false, skippedConcurrent: false };
+  await cleanupStaleArchiveTemps(archivePath, now);
+  clearStatsArchiveCache();
+  const identityAtRead = await archiveFileIdentity(archivePath);
+  const loaded = await readStatsArchive(archivePath);
+  if (loaded.corrupt) {
+    result.skippedCorrupt = true;
+    return result;
+  }
+  const previous = loaded.archive;
+  const today = currentLocalDay();
+  // Strict reads: compaction permanently drops counters whose epoch differs,
+  // so an unreadable epoch file must abort instead of falling back to the
+  // initial epoch (which would discard every archived and absorbed counter).
+  const globalEpoch = await readGlobalStatsEpochStrict();
+  const modelEpochs = new Map<string, string>();
+  const currentModelEpoch = async (key: string): Promise<string> => {
+    let epoch = modelEpochs.get(key);
+    if (!epoch) {
+      epoch = await readModelStatsEpochStrict(key);
+      modelEpochs.set(key, epoch);
+    }
+    return epoch;
+  };
+
+  // Carry forward live entries; drop ones a reset or retention invalidated.
+  const buckets = new Map<string, StatsArchiveBucketV1>();
+  for (const bucket of previous.buckets) {
+    if (
+      bucket.globalEpoch !== globalEpoch ||
+      bucket.modelEpoch !== await currentModelEpoch(bucket.modelKey) ||
+      now - bucket.updatedAt >= SHARD_RETENTION_MS
+    ) {
+      result.dropped++;
+      continue;
+    }
+    buckets.set(bucketId(bucket), { ...bucket, stats: cloneCacheStats(bucket.stats) });
+  }
+  const routed = new Map<string, StatsArchiveRoutedV1>();
+  for (const entry of previous.routed) {
+    if (entry.globalEpoch !== globalEpoch || now - entry.updatedAt >= SHARD_RETENTION_MS) {
+      result.dropped++;
+      continue;
+    }
+    routed.set(routedId(entry), { ...entry, model: { ...entry.model } });
+  }
+
+  const shards = await readValidStatsShardsV7(directory);
+  const presentIds = new Set(shards.map((shard) => shard.instanceId));
+  const absorbed: Record<string, number> = {};
+  // Keep ids whose raw file still exists (a crash or a failed delete left it).
+  for (const [id, at] of Object.entries(previous.absorbed)) if (presentIds.has(id)) absorbed[id] = at;
+
+  for (const shard of shards) {
+    if (shard.instanceId in absorbed || !isArchivableShard(shard, now, today)) continue;
+    absorbed[shard.instanceId] = now;
+    result.absorbed++;
+    if (shard.globalEpoch !== globalEpoch) continue; // reset away: nothing to keep
+    if (shard.lastRoutedModel) {
+      const id = routedId({ globalEpoch, sessionHash: shard.sessionHash });
+      const existing = routed.get(id);
+      if (!existing || existing.updatedAt < shard.lifecycle.updatedAt) {
+        routed.set(id, { globalEpoch, sessionHash: shard.sessionHash, day: shard.day, updatedAt: shard.lifecycle.updatedAt, model: { ...shard.lastRoutedModel } });
+      }
+    }
+    for (const [key, entry] of Object.entries(shard.models)) {
+      if (entry.modelEpoch !== await currentModelEpoch(key)) continue;
+      const id = bucketId({ globalEpoch, sessionHash: shard.sessionHash, modelKey: key, modelEpoch: entry.modelEpoch, day: shard.day });
+      const existing = buckets.get(id);
+      const newer = !existing || existing.updatedAt <= shard.lifecycle.updatedAt;
+      buckets.set(id, {
+        globalEpoch,
+        sessionHash: shard.sessionHash,
+        modelKey: key,
+        modelEpoch: entry.modelEpoch,
+        day: shard.day,
+        provider: newer ? entry.provider : existing.provider,
+        modelId: newer ? entry.modelId : existing.modelId,
+        ...((newer ? entry.modelName : existing.modelName) ? { modelName: newer ? entry.modelName : existing.modelName } : {}),
+        ...((newer ? entry.api : existing.api) ? { api: newer ? entry.api : existing.api } : {}),
+        updatedAt: Math.max(existing?.updatedAt ?? 0, shard.lifecycle.updatedAt),
+        stats: mergeStatsAcrossDays(existing?.stats, entry.stats, null),
+      });
+    }
+  }
+
+  const absorbedChanged =
+    Object.keys(absorbed).length !== Object.keys(previous.absorbed).length ||
+    Object.keys(absorbed).some((id) => !(id in previous.absorbed));
+  if (result.absorbed > 0 || result.dropped > 0 || absorbedChanged) {
+    await options.beforePublish?.();
+    // Publish only if nothing this plan depends on moved underneath it: a
+    // reset during the pass (counters written under the new epoch would be
+    // dropped as stale), or another compactor after a stale-lock takeover
+    // (overwriting its archive would lose shards it already deleted).
+    let concurrent =
+      !(await holdsStatsMaintenanceLock(options.lockToken)) ||
+      !sameArchiveFile(identityAtRead, await archiveFileIdentity(archivePath)) ||
+      (await readGlobalStatsEpochStrict()) !== globalEpoch;
+    for (const [key, epoch] of modelEpochs) {
+      if (concurrent) break;
+      if ((await readModelStatsEpochStrict(key)) !== epoch) concurrent = true;
+    }
+    if (concurrent) {
+      result.skippedConcurrent = true;
+      result.absorbed = 0;
+      result.dropped = 0;
+      return result;
+    }
+    await writeStatsArchive(archivePath, {
+      version: 1,
+      kind: "pi-cache-optimizer-archive",
+      updatedAt: now,
+      absorbed,
+      buckets: Array.from(buckets.values()),
+      routed: Array.from(routed.values()),
+    });
+  }
+
+  // Only now that the archive covering them is durable, delete the originals.
+  const toDelete = Object.keys(absorbed);
+  const outcomes = await mapWithConcurrency(toDelete, SHARD_IO_CONCURRENCY, async (id): Promise<number> => {
+    try {
+      await unlink(join(directory, `${id}.json`));
+      return 1;
+    } catch {
+      // ENOENT: already gone. Anything else: the id stays absorbed (so the
+      // file is ignored by readers) and the next compaction retries.
+      return 0;
+    }
+  });
+  result.removed = outcomes.reduce((sum, value) => sum + value, 0);
+  return result;
+}
+
+export type StatsMaintenanceResult = { cleanupRemoved: number; compaction?: CompactionResult };
+
+/**
+ * Due retention cleanup and archive compaction under one lock. Each has its
+ * own marker so the first run after upgrading compacts immediately even when
+ * cleanup ran recently.
+ */
+export async function maybeRunStatsMaintenance(now = Date.now()): Promise<StatsMaintenanceResult> {
+  const result: StatsMaintenanceResult = { cleanupRemoved: 0 };
+  await mkdir(SHARD_MAINTENANCE_DIR, { recursive: true });
+  const cleanupDue = await isMaintenanceMarkerDue(SHARD_CLEANUP_MARKER_PATH, now);
+  const compactionDue = await isMaintenanceMarkerDue(SHARD_COMPACTION_MARKER_PATH, now);
+  if (!cleanupDue && !compactionDue) return result;
+  const lockToken = await acquireStatsMaintenanceLock(now);
+  if (!lockToken) return result;
+  try {
+    if (cleanupDue) {
+      result.cleanupRemoved = await cleanupStatsShardsV7(now);
+      await writeMaintenanceMarker(SHARD_CLEANUP_MARKER_PATH, now);
+    }
+    if (compactionDue) {
+      try {
+        result.compaction = await compactStatsShardsV7(now, SHARD_FILES_DIR, SHARD_ARCHIVE_PATH, { lockToken });
+        if (result.compaction.skippedCorrupt) {
+          console.warn(`${LOG_PREFIX}: stats archive ${SHARD_ARCHIVE_PATH} is unreadable; keeping raw shards`);
+        }
+      } catch (error) {
+        console.warn(`${LOG_PREFIX}: stats archive compaction failed`, error);
+      }
+      // A pass that lost a race published nothing; let the next startup retry
+      // instead of waiting out the full interval.
+      if (!result.compaction?.skippedConcurrent) await writeMaintenanceMarker(SHARD_COMPACTION_MARKER_PATH, now);
+    }
+  } finally {
+    await releaseStatsMaintenanceLock(lockToken);
+  }
+  return result;
+}

package/src/stats-store.ts

@@ -234,6 +234,25 @@ export async function readEpochFile(path: string, fallback: string): Promise<str
   }
 }
 
+/**
+ * Like readEpochFile, but only a missing file maps to the fallback. Any other
+ * read or parse failure throws, so destructive maintenance (archive
+ * compaction drops counters whose epoch differs) never mistakes a transient
+ * Windows EPERM/EBUSY or a damaged epoch file for "no reset ever happened".
+ */
+export async function readEpochFileStrict(path: string, fallback: string): Promise<string> {
+  let text: string;
+  try {
+    text = await readFile(path, "utf8");
+  } catch (error) {
+    if (getErrorCode(error) === "ENOENT") return fallback;
+    throw error;
+  }
+  const epoch = parseEpochRecord(JSON.parse(text));
+  if (!epoch) throw new Error(`invalid stats epoch file ${path}`);
+  return epoch;
+}
+
 export async function readGlobalStatsEpoch(): Promise<string> {
   return readEpochFile(SHARD_GLOBAL_EPOCH_PATH, initialEpoch("global"));
 }
@@ -242,6 +261,14 @@ export async function readModelStatsEpoch(modelKeyValue: string): Promise<string
   return readEpochFile(modelEpochPath(modelKeyValue), initialEpoch(`model:${modelKeyValue}`));
 }
 
+export async function readGlobalStatsEpochStrict(): Promise<string> {
+  return readEpochFileStrict(SHARD_GLOBAL_EPOCH_PATH, initialEpoch("global"));
+}
+
+export async function readModelStatsEpochStrict(modelKeyValue: string): Promise<string> {
+  return readEpochFileStrict(modelEpochPath(modelKeyValue), initialEpoch(`model:${modelKeyValue}`));
+}
+
 export async function writeStatsEpoch(path: string, epoch: string): Promise<void> {
   await mkdir(dirname(path), { recursive: true });
   const tempPath = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
@@ -584,35 +611,111 @@ export async function cleanupStatsShardsV7(now = Date.now(), directory: string =
   return outcomes.reduce((sum, value) => sum + value, 0);
 }
 
-/** Run due retention maintenance; resolves to the number of files removed (0 when skipped). */
-export async function maybeCleanupStatsShardsV7(now = Date.now()): Promise<number> {
-  await mkdir(SHARD_MAINTENANCE_DIR, { recursive: true });
-  try {
-    const marker = await stat(SHARD_CLEANUP_MARKER_PATH);
-    if (now - marker.mtimeMs < SHARD_CLEANUP_INTERVAL_MS) return 0;
-  } catch {}
+const SHARD_CLEANUP_LOCK_OWNER_FILE = "owner";
 
+/**
+ * Take the cross-process maintenance lock (a directory, so creation is atomic
+ * on every platform). A lock older than SHARD_CLEANUP_LOCK_STALE_MS is
+ * treated as abandoned by a crashed process and replaced. Resolves to an
+ * owner token (truthy) on success, undefined otherwise. The token lets a
+ * holder whose lock was taken over avoid releasing the new owner's lock.
+ */
+export async function acquireStatsMaintenanceLock(now = Date.now()): Promise<string | undefined> {
+  await mkdir(SHARD_MAINTENANCE_DIR, { recursive: true });
+  const token = `${process.pid}-${randomUUID()}`;
+  const claim = async (): Promise<string> => {
+    // Best effort: an older version (or a failed write) leaves no owner file,
+    // which only disables the ownership check, never the lock itself.
+    await writeFile(join(SHARD_CLEANUP_LOCK_PATH, SHARD_CLEANUP_LOCK_OWNER_FILE), token, "utf8").catch(() => undefined);
+    return token;
+  };
   try {
     await mkdir(SHARD_CLEANUP_LOCK_PATH);
+    return await claim();
   } catch (error) {
-    if (getErrorCode(error) !== "EEXIST") return 0;
+    if (getErrorCode(error) !== "EEXIST") return undefined;
     try {
       const lock = await stat(SHARD_CLEANUP_LOCK_PATH);
-      if (now - lock.mtimeMs <= SHARD_CLEANUP_LOCK_STALE_MS) return 0;
+      if (now - lock.mtimeMs <= SHARD_CLEANUP_LOCK_STALE_MS) return undefined;
       await rm(SHARD_CLEANUP_LOCK_PATH, { recursive: true, force: true });
       await mkdir(SHARD_CLEANUP_LOCK_PATH);
+      return await claim();
     } catch {
-      return 0;
+      return undefined;
     }
   }
+}
 
+/** False only when the lock directory now records a different owner (or is gone). */
+export async function holdsStatsMaintenanceLock(token: string | undefined): Promise<boolean> {
+  if (!token) return true;
+  try {
+    const owner = await readFile(join(SHARD_CLEANUP_LOCK_PATH, SHARD_CLEANUP_LOCK_OWNER_FILE), "utf8");
+    return owner === token;
+  } catch (error) {
+    // Lock directory gone: someone treated us as stale. Owner file missing
+    // inside an existing lock: written by a version without tokens; assume ours.
+    if (getErrorCode(error) !== "ENOENT") return true;
+    try {
+      await stat(SHARD_CLEANUP_LOCK_PATH);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}
+
+export async function releaseStatsMaintenanceLock(token?: string): Promise<void> {
+  // Do not delete a lock another process took over after ours went stale.
+  if (token && !(await holdsStatsMaintenanceLock(token))) return;
+  await rm(SHARD_CLEANUP_LOCK_PATH, { recursive: true, force: true });
+}
+
+/** True when the marker file is missing or older than `intervalMs`. */
+export async function isMaintenanceMarkerDue(path: string, now = Date.now(), intervalMs = SHARD_CLEANUP_INTERVAL_MS): Promise<boolean> {
+  try {
+    const marker = await stat(path);
+    return now - marker.mtimeMs >= intervalMs;
+  } catch {
+    return true;
+  }
+}
+
+export async function writeMaintenanceMarker(path: string, now = Date.now()): Promise<void> {
+  const tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
+  await writeFile(tempPath, `${now}\n`, "utf8");
+  await renameWithRetry(tempPath, path);
+}
+
+const RENAME_RETRY_CODES = new Set(["EPERM", "EBUSY", "EACCES"]);
+
+/**
+ * rename() that retries briefly on the transient errors Windows raises while
+ * another process (or an antivirus scanner) holds the target open.
+ */
+export async function renameWithRetry(from: string, to: string, attempts = 5): Promise<void> {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await rename(from, to);
+      return;
+    } catch (error) {
+      if (attempt >= attempts - 1 || !RENAME_RETRY_CODES.has(getErrorCode(error) ?? "")) throw error;
+      await new Promise((resolve) => setTimeout(resolve, 20 * 2 ** attempt));
+    }
+  }
+}
+
+/** Run due retention maintenance; resolves to the number of files removed (0 when skipped). */
+export async function maybeCleanupStatsShardsV7(now = Date.now()): Promise<number> {
+  await mkdir(SHARD_MAINTENANCE_DIR, { recursive: true });
+  if (!(await isMaintenanceMarkerDue(SHARD_CLEANUP_MARKER_PATH, now))) return 0;
+  const lockToken = await acquireStatsMaintenanceLock(now);
+  if (!lockToken) return 0;
   try {
     const removed = await cleanupStatsShardsV7(now);
-    const tempPath = `${SHARD_CLEANUP_MARKER_PATH}.${process.pid}.${randomUUID()}.tmp`;
-    await writeFile(tempPath, `${now}\n`, "utf8");
-    await rename(tempPath, SHARD_CLEANUP_MARKER_PATH);
+    await writeMaintenanceMarker(SHARD_CLEANUP_MARKER_PATH, now);
     return removed;
   } finally {
-    await rm(SHARD_CLEANUP_LOCK_PATH, { recursive: true, force: true });
+    await releaseStatsMaintenanceLock(lockToken);
   }
 }