pi-cache-optimizer 2.9.0 → 2.9.1

package/src/stats-archive.ts

@@ -6,8 +6,8 @@
  * conversations, which left ~1,500 files to stat and parse on every refresh.
  *
  * Past-day shards that are closed (or abandoned by a dead process) never
- * change again, so maintenance folds them into one archive file and deletes
- * the originals. The directory then only holds today's shards.
+ * change again, so maintenance folds them into one archive snapshot and
+ * deletes the originals. The directory then only holds today's shards.
  *
  * Correctness rules:
  * - Buckets keep the global and model epochs they were written under. Readers
@@ -20,18 +20,38 @@
  * - Archived data only feeds the all-days (session-scoped) aggregate. Its
  *   buckets keep their original day, so the current-day aggregate filters
  *   them out exactly as it filters the raw past-day shards they replace.
- *   Per-instance counters in the all-days aggregate count archive buckets,
- *   not processes; only `bySession` and `lastRoutedModelBySession` are read
- *   from it.
+ *
+ * Storage (generations): each snapshot is an immutable file
+ * `archive/archive.<generation>.json`. A compactor that read generation N
+ * publishes N+1 with link(temp, target), which fails with EEXIST when anyone
+ * else already published N+1. That makes publishing a compare-and-swap on
+ * Windows and POSIX alike: a holder that stalled past the stale-lock window
+ * (laptop sleep) and lost its lock can never replace a newer snapshot, and it
+ * deletes no raw shard unless its own snapshot is the newest one. Readers use
+ * the highest generation; old generations are removed one generation late
+ * and readers retry when the file they listed is gone.
+ *
+ * 2.9.0 compatibility: 2.9.0 kept a single `archive/archive.json`. It is still
+ * read while no generation exists and migrated by the next compaction, which
+ * then replaces it with a directory of the same name. 2.9.0 treats that as an
+ * unreadable archive: it never compacts (so it cannot overwrite or delete
+ * anything) and its footer shows only history that is not archived, exactly
+ * like the versions before 2.9.0. A file that 2.9.0 still manages to write
+ * there afterwards (only if that directory is removed) is never counted: it is
+ * set aside as `archive.legacy-<time>.json`, trading the counters it absorbed
+ * for a guarantee against double counting.
  */
-import { randomUUID } from "node:crypto";
-import { lstat, mkdir, readFile, readdir, unlink, writeFile } from "node:fs/promises";
+import { createHash, randomUUID } from "node:crypto";
+import { link, lstat, mkdir, open, readFile, readdir, rename, unlink, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
 import { LOG_PREFIX, asRecord, getErrorCode, isNonEmptyString, isProcessAlive } from "./common.ts";
 import {
   type CacheStats,
   type PersistedRoutedModelRef,
   type PersistedStatsShardV7,
+  type ShardAggregate,
+  type ShardAggregator,
+  type ShardContribution,
   SHARD_CLEANUP_MARKER_PATH,
   SHARD_FILES_DIR,
   SHARD_IO_CONCURRENCY,
@@ -42,6 +62,7 @@ import {
   acquireStatsMaintenanceLock,
   cleanupStatsShardsV7,
   cloneCacheStats,
+  createShardAggregatorV7,
   currentLocalDay,
   isMaintenanceMarkerDue,
   mapWithConcurrency,
@@ -54,10 +75,16 @@ import {
   readValidStatsShardsV7,
   releaseStatsMaintenanceLock,
   renameWithRetry,
+  touchStatsMaintenanceLock,
   writeMaintenanceMarker,
 } from "./stats-store.ts";
 
 export const SHARD_ARCHIVE_DIR = join(SHARD_STATE_DIR, "archive");
+/**
+ * 2.9.0's single-file archive. Read (and migrated) while no generation file
+ * exists; afterwards a directory blocks 2.9.0 from writing it. The other
+ * archive functions take this path and keep generations next to it.
+ */
 export const SHARD_ARCHIVE_PATH = join(SHARD_ARCHIVE_DIR, "archive.json");
 export const SHARD_COMPACTION_MARKER_PATH = join(SHARD_MAINTENANCE_DIR, "last-compaction");
 
@@ -68,6 +95,9 @@ export const SHARD_COMPACTION_MARKER_PATH = join(SHARD_MAINTENANCE_DIR, "last-co
  */
 export const ARCHIVE_ABANDONED_ACTIVE_MS = 24 * 60 * 60 * 1000;
 
+/** How often a running maintenance pass refreshes the lock's mtime. */
+export const STATS_MAINTENANCE_HEARTBEAT_MS = 5 * 60 * 1000;
+
 export type StatsArchiveBucketV1 = {
   globalEpoch: string;
   sessionHash: string;
@@ -92,6 +122,7 @@ export type StatsArchiveRoutedV1 = {
   model: PersistedRoutedModelRef;
 };
 
+/** In-memory archive contents (and 2.9.0's on-disk format). */
 export type StatsArchiveV1 = {
   version: 1;
   kind: "pi-cache-optimizer-archive";
@@ -102,7 +133,31 @@ export type StatsArchiveV1 = {
   routed: StatsArchiveRoutedV1[];
 };
 
+/**
+ * On-disk generation format: the same entries as v1 with every string
+ * interned in one table, so a bucket is one short row of numbers.
+ * models rows: [provider, modelId, modelName | -1, api | -1] (string indexes).
+ * buckets rows: [globalEpoch, sessionHash, model, modelEpoch, day, updatedAt,
+ *   stats.day, totalRequests, hitRequests, cachedInputTokens,
+ *   cacheWriteInputTokens, totalInputTokens] (strings and model are indexes).
+ * routed rows: [globalEpoch, sessionHash, day, updatedAt, provider, id, name | -1].
+ */
+export type StatsArchiveV2Document = {
+  version: 2;
+  kind: "pi-cache-optimizer-archive";
+  generation: number;
+  updatedAt: number;
+  /** sha256 of the 2.9.0 archive.json this archive was migrated from. */
+  legacySha256?: string;
+  absorbed: Record<string, number>;
+  strings: string[];
+  models: number[][];
+  buckets: number[][];
+  routed: number[][];
+};
+
 const DAY_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
+const GENERATION_FILE_PATTERN = /^archive\.(\d{1,15})\.json$/;
 
 export function emptyStatsArchive(now = Date.now()): StatsArchiveV1 {
   return { version: 1, kind: "pi-cache-optimizer-archive", updatedAt: now, absorbed: {}, buckets: [], routed: [] };
@@ -112,59 +167,71 @@ function isFiniteNumber(value: unknown): value is number {
   return typeof value === "number" && Number.isFinite(value);
 }
 
-/** Strict parse: returns undefined when the document is not a v1 archive. Invalid entries are dropped. */
-export function parseStatsArchive(value: unknown): StatsArchiveV1 | undefined {
-  const record = asRecord(value);
-  if (!record || record.version !== 1 || record.kind !== "pi-cache-optimizer-archive") return undefined;
-  const rawAbsorbed = asRecord(record.absorbed);
-  if (!rawAbsorbed || !Array.isArray(record.buckets) || !Array.isArray(record.routed)) return undefined;
+function parseArchiveBucket(raw: unknown): StatsArchiveBucketV1 | undefined {
+  const entry = asRecord(raw);
+  const stats = parseCacheStats(entry?.stats);
+  if (
+    !entry || !stats ||
+    !isNonEmptyString(entry.globalEpoch) || !isNonEmptyString(entry.sessionHash) ||
+    !isNonEmptyString(entry.modelKey) || !isNonEmptyString(entry.modelEpoch) ||
+    !isNonEmptyString(entry.provider) || !isNonEmptyString(entry.modelId) ||
+    typeof entry.day !== "string" || !DAY_PATTERN.test(entry.day) ||
+    !isFiniteNumber(entry.updatedAt) ||
+    entry.modelKey !== `${entry.provider}/${entry.modelId}`
+  ) return undefined;
+  return {
+    globalEpoch: entry.globalEpoch,
+    sessionHash: entry.sessionHash,
+    modelKey: entry.modelKey,
+    modelEpoch: entry.modelEpoch,
+    day: entry.day,
+    provider: entry.provider,
+    modelId: entry.modelId,
+    ...(isNonEmptyString(entry.modelName) ? { modelName: entry.modelName } : {}),
+    ...(isNonEmptyString(entry.api) ? { api: entry.api } : {}),
+    updatedAt: entry.updatedAt,
+    stats,
+  };
+}
+
+function parseArchiveRouted(raw: unknown): StatsArchiveRoutedV1 | undefined {
+  const entry = asRecord(raw);
+  const model = parsePersistedRoutedModelRef(entry?.model);
+  if (
+    !entry || !model ||
+    !isNonEmptyString(entry.globalEpoch) || !isNonEmptyString(entry.sessionHash) ||
+    typeof entry.day !== "string" || !DAY_PATTERN.test(entry.day) ||
+    !isFiniteNumber(entry.updatedAt)
+  ) return undefined;
+  return { globalEpoch: entry.globalEpoch, sessionHash: entry.sessionHash, day: entry.day, updatedAt: entry.updatedAt, model };
+}
 
+function parseAbsorbed(value: unknown): Record<string, number> | undefined {
+  const rawAbsorbed = asRecord(value);
+  if (!rawAbsorbed) return undefined;
   const absorbed: Record<string, number> = {};
   for (const [id, at] of Object.entries(rawAbsorbed)) {
     if (isNonEmptyString(id) && isFiniteNumber(at)) absorbed[id] = at;
   }
+  return absorbed;
+}
 
+/** Strict parse of 2.9.0's format: undefined when the document is not a v1 archive. Invalid entries are dropped. */
+export function parseStatsArchive(value: unknown): StatsArchiveV1 | undefined {
+  const record = asRecord(value);
+  if (!record || record.version !== 1 || record.kind !== "pi-cache-optimizer-archive") return undefined;
+  const absorbed = parseAbsorbed(record.absorbed);
+  if (!absorbed || !Array.isArray(record.buckets) || !Array.isArray(record.routed)) return undefined;
   const buckets: StatsArchiveBucketV1[] = [];
   for (const raw of record.buckets) {
-    const entry = asRecord(raw);
-    const stats = parseCacheStats(entry?.stats);
-    if (
-      !entry || !stats ||
-      !isNonEmptyString(entry.globalEpoch) || !isNonEmptyString(entry.sessionHash) ||
-      !isNonEmptyString(entry.modelKey) || !isNonEmptyString(entry.modelEpoch) ||
-      !isNonEmptyString(entry.provider) || !isNonEmptyString(entry.modelId) ||
-      typeof entry.day !== "string" || !DAY_PATTERN.test(entry.day) ||
-      !isFiniteNumber(entry.updatedAt) ||
-      entry.modelKey !== `${entry.provider}/${entry.modelId}`
-    ) continue;
-    buckets.push({
-      globalEpoch: entry.globalEpoch,
-      sessionHash: entry.sessionHash,
-      modelKey: entry.modelKey,
-      modelEpoch: entry.modelEpoch,
-      day: entry.day,
-      provider: entry.provider,
-      modelId: entry.modelId,
-      ...(isNonEmptyString(entry.modelName) ? { modelName: entry.modelName } : {}),
-      ...(isNonEmptyString(entry.api) ? { api: entry.api } : {}),
-      updatedAt: entry.updatedAt,
-      stats,
-    });
+    const bucket = parseArchiveBucket(raw);
+    if (bucket) buckets.push(bucket);
   }
-
   const routed: StatsArchiveRoutedV1[] = [];
   for (const raw of record.routed) {
-    const entry = asRecord(raw);
-    const model = parsePersistedRoutedModelRef(entry?.model);
-    if (
-      !entry || !model ||
-      !isNonEmptyString(entry.globalEpoch) || !isNonEmptyString(entry.sessionHash) ||
-      typeof entry.day !== "string" || !DAY_PATTERN.test(entry.day) ||
-      !isFiniteNumber(entry.updatedAt)
-    ) continue;
-    routed.push({ globalEpoch: entry.globalEpoch, sessionHash: entry.sessionHash, day: entry.day, updatedAt: entry.updatedAt, model });
+    const entry = parseArchiveRouted(raw);
+    if (entry) routed.push(entry);
   }
-
   return {
     version: 1,
     kind: "pi-cache-optimizer-archive",
@@ -175,28 +242,172 @@ export function parseStatsArchive(value: unknown): StatsArchiveV1 | undefined {
   };
 }
 
-export type LoadedStatsArchive = {
-  /** False when no archive file exists; true for a parsed archive. */
-  present: boolean;
-  /** True when a file exists but cannot be parsed; compaction then refuses to overwrite it. */
-  corrupt: boolean;
-  archive: StatsArchiveV1;
-  absorbedIds: ReadonlySet<string>;
-  /** Archive buckets/routed entries as synthetic shards for aggregateStatsShardsV7. Do not mutate. */
-  shards: readonly PersistedStatsShardV7[];
-};
+/** Encode an archive in the compact generation format. */
+export function encodeStatsArchiveV2(archive: StatsArchiveV1, generation: number, legacySha256?: string): StatsArchiveV2Document {
+  const strings: string[] = [];
+  const stringIndex = new Map<string, number>();
+  const str = (value: string): number => {
+    let index = stringIndex.get(value);
+    if (index === undefined) {
+      index = strings.length;
+      strings.push(value);
+      stringIndex.set(value, index);
+    }
+    return index;
+  };
+  const optional = (value: string | undefined): number => (value === undefined ? -1 : str(value));
+  const models: number[][] = [];
+  const modelIndex = new Map<string, number>();
+  const model = (bucket: StatsArchiveBucketV1): number => {
+    const row = [str(bucket.provider), str(bucket.modelId), optional(bucket.modelName), optional(bucket.api)];
+    const id = row.join(",");
+    let index = modelIndex.get(id);
+    if (index === undefined) {
+      index = models.length;
+      models.push(row);
+      modelIndex.set(id, index);
+    }
+    return index;
+  };
+  const buckets = archive.buckets.map((bucket) => [
+    str(bucket.globalEpoch), str(bucket.sessionHash), model(bucket), str(bucket.modelEpoch), str(bucket.day), bucket.updatedAt,
+    str(bucket.stats.day), bucket.stats.totalRequests, bucket.stats.hitRequests, bucket.stats.cachedInputTokens,
+    bucket.stats.cacheWriteInputTokens, bucket.stats.totalInputTokens,
+  ]);
+  const routed = archive.routed.map((entry) => [
+    str(entry.globalEpoch), str(entry.sessionHash), str(entry.day), entry.updatedAt,
+    str(entry.model.provider), str(entry.model.id), optional(entry.model.name),
+  ]);
+  return {
+    version: 2,
+    kind: "pi-cache-optimizer-archive",
+    generation,
+    updatedAt: archive.updatedAt,
+    ...(legacySha256 ? { legacySha256 } : {}),
+    absorbed: archive.absorbed,
+    strings,
+    models,
+    buckets,
+    routed,
+  };
+}
 
-type CachedArchive = { ino: number | bigint; size: number; mtimeMs: number; loaded: LoadedStatsArchive };
-const archiveCache = new Map<string, CachedArchive>();
+/**
+ * Strict parse of a generation file: undefined when the document itself is
+ * not a v2 archive. Rows with bad indexes or values are dropped, applying the
+ * same per-entry validation as v1.
+ */
+export function parseStatsArchiveV2(value: unknown): { archive: StatsArchiveV1; generation: number; legacySha256?: string } | undefined {
+  const record = asRecord(value);
+  if (!record || record.version !== 2 || record.kind !== "pi-cache-optimizer-archive") return undefined;
+  if (!Number.isSafeInteger(record.generation) || (record.generation as number) < 1) return undefined;
+  const absorbed = parseAbsorbed(record.absorbed);
+  const { strings, models, buckets: rawBuckets, routed: rawRouted } = record;
+  if (
+    !absorbed || !Array.isArray(strings) || !Array.isArray(models) || !Array.isArray(rawBuckets) || !Array.isArray(rawRouted) ||
+    !strings.every((entry) => typeof entry === "string")
+  ) return undefined;
+  const str = (index: unknown): string | undefined =>
+    Number.isInteger(index) && (index as number) >= 0 && (index as number) < strings.length ? strings[index as number] as string : undefined;
+  const row = (value: unknown, length: number): unknown[] | undefined => (Array.isArray(value) && value.length === length ? value : undefined);
 
-export function clearStatsArchiveCache(): void {
-  archiveCache.clear();
+  const modelTable = models.map((raw) => {
+    const entry = row(raw, 4);
+    return entry ? { provider: str(entry[0]), modelId: str(entry[1]), modelName: str(entry[2]), api: str(entry[3]) } : undefined;
+  });
+  const buckets: StatsArchiveBucketV1[] = [];
+  for (const raw of rawBuckets) {
+    const entry = row(raw, 12);
+    if (!entry) continue;
+    const modelEntry = Number.isInteger(entry[2]) ? modelTable[entry[2] as number] : undefined;
+    if (!modelEntry) continue;
+    const bucket = parseArchiveBucket({
+      globalEpoch: str(entry[0]),
+      sessionHash: str(entry[1]),
+      modelKey: `${modelEntry.provider}/${modelEntry.modelId}`,
+      modelEpoch: str(entry[3]),
+      day: str(entry[4]),
+      provider: modelEntry.provider,
+      modelId: modelEntry.modelId,
+      modelName: modelEntry.modelName,
+      api: modelEntry.api,
+      updatedAt: entry[5],
+      stats: {
+        day: str(entry[6]), totalRequests: entry[7], hitRequests: entry[8], cachedInputTokens: entry[9],
+        cacheWriteInputTokens: entry[10], totalInputTokens: entry[11],
+      },
+    });
+    if (bucket) buckets.push(bucket);
+  }
+  const routed: StatsArchiveRoutedV1[] = [];
+  for (const raw of rawRouted) {
+    const entry = row(raw, 7);
+    if (!entry) continue;
+    const parsed = parseArchiveRouted({
+      globalEpoch: str(entry[0]),
+      sessionHash: str(entry[1]),
+      day: str(entry[2]),
+      updatedAt: entry[3],
+      model: { provider: str(entry[4]), id: str(entry[5]), name: str(entry[6]) },
+    });
+    if (parsed) routed.push(parsed);
+  }
+  return {
+    archive: {
+      version: 1,
+      kind: "pi-cache-optimizer-archive",
+      updatedAt: isFiniteNumber(record.updatedAt) ? record.updatedAt : 0,
+      absorbed,
+      buckets,
+      routed,
+    },
+    generation: record.generation as number,
+    ...(isNonEmptyString(record.legacySha256) ? { legacySha256: record.legacySha256 } : {}),
+  };
 }
 
-/** Convert archive entries into synthetic closed shards so the existing aggregation applies unchanged. */
-export function archiveToSyntheticShards(archive: StatsArchiveV1): PersistedStatsShardV7[] {
-  const shards: PersistedStatsShardV7[] = [];
-  const base = (id: string, sessionHash: string, day: string, globalEpoch: string, updatedAt: number): PersistedStatsShardV7 => ({
+export function statsArchiveGenerationPath(archivePath: string, generation: number): string {
+  return join(dirname(archivePath), `archive.${String(generation).padStart(12, "0")}.json`);
+}
+
+/** Published generations next to `archivePath`, ascending. */
+export async function listStatsArchiveGenerations(archivePath: string = SHARD_ARCHIVE_PATH): Promise<number[]> {
+  let names: string[];
+  try {
+    names = await readdir(dirname(archivePath));
+  } catch (error) {
+    if (getErrorCode(error) === "ENOENT") return [];
+    throw error;
+  }
+  const generations: number[] = [];
+  for (const name of names) {
+    const match = GENERATION_FILE_PATTERN.exec(name);
+    if (match) generations.push(Number(match[1]));
+  }
+  return generations.sort((a, b) => a - b);
+}
+
+function bucketToSyntheticShard(bucket: StatsArchiveBucketV1, index: number): PersistedStatsShardV7 {
+  const shard = syntheticShardBase(`archive-bucket-${index}`, bucket.sessionHash, bucket.day, bucket.globalEpoch, bucket.updatedAt);
+  shard.models[bucket.modelKey] = {
+    modelEpoch: bucket.modelEpoch,
+    provider: bucket.provider,
+    modelId: bucket.modelId,
+    ...(bucket.modelName ? { modelName: bucket.modelName } : {}),
+    ...(bucket.api ? { api: bucket.api } : {}),
+    stats: cloneCacheStats(bucket.stats),
+  };
+  return shard;
+}
+
+function routedToSyntheticShard(entry: StatsArchiveRoutedV1, index: number): PersistedStatsShardV7 {
+  const shard = syntheticShardBase(`archive-routed-${index}`, entry.sessionHash, entry.day, entry.globalEpoch, entry.updatedAt);
+  shard.lastRoutedModel = { ...entry.model };
+  return shard;
+}
+
+function syntheticShardBase(id: string, sessionHash: string, day: string, globalEpoch: string, updatedAt: number): PersistedStatsShardV7 {
+  return {
     version: 7,
     kind: "pi-cache-optimizer-shard",
     instanceId: id,
@@ -206,103 +417,442 @@ export function archiveToSyntheticShards(archive: StatsArchiveV1): PersistedStat
     day,
     globalEpoch,
     models: {},
-  });
-  archive.buckets.forEach((bucket, index) => {
-    const shard = base(`archive-bucket-${index}`, bucket.sessionHash, bucket.day, bucket.globalEpoch, bucket.updatedAt);
-    shard.models[bucket.modelKey] = {
-      modelEpoch: bucket.modelEpoch,
-      provider: bucket.provider,
-      modelId: bucket.modelId,
-      ...(bucket.modelName ? { modelName: bucket.modelName } : {}),
-      ...(bucket.api ? { api: bucket.api } : {}),
-      stats: cloneCacheStats(bucket.stats),
-    };
-    shards.push(shard);
-  });
-  archive.routed.forEach((entry, index) => {
-    const shard = base(`archive-routed-${index}`, entry.sessionHash, entry.day, entry.globalEpoch, entry.updatedAt);
-    shard.lastRoutedModel = { ...entry.model };
-    shards.push(shard);
-  });
-  return shards;
+  };
 }
 
-function buildLoaded(archive: StatsArchiveV1, present: boolean, corrupt: boolean): LoadedStatsArchive {
-  return {
-    present,
-    corrupt,
-    archive,
-    absorbedIds: new Set(Object.keys(archive.absorbed)),
-    shards: archiveToSyntheticShards(archive),
-  };
+/**
+ * Archive entries as synthetic closed shards: the reference semantics.
+ * Aggregating these (after the raw shards) is what the archive must be
+ * equivalent to; readers use StatsArchiveView's reduced forms instead.
+ */
+export function archiveToSyntheticShards(archive: StatsArchiveV1): PersistedStatsShardV7[] {
+  return [
+    ...archive.buckets.map(bucketToSyntheticShard),
+    ...archive.routed.map(routedToSyntheticShard),
+  ];
 }
 
+export type LoadedStatsArchive = {
+  /** False when no archive exists; true for a parsed (or unreadable) one. */
+  present: boolean;
+  /** True when an archive exists but cannot be parsed; compaction then refuses to overwrite it. */
+  corrupt: boolean;
+  /** Generation this snapshot came from; 0 for 2.9.0's archive.json or no archive. */
+  generation: number;
+  archive: StatsArchiveV1;
+  absorbedIds: ReadonlySet<string>;
+  /** Every archive entry as a synthetic shard (reference form, built on first access). Do not mutate. */
+  readonly shards: readonly PersistedStatsShardV7[];
+};
+
+type BucketGroup = {
+  /** Position in ReducedArchive.groups. */
+  order: number;
+  globalEpoch: string;
+  sessionHash: string;
+  key: string;
+  modelEpoch: string;
+  /** The single bucket's own stats until a second bucket is merged (then a private copy). */
+  stats: CacheStats;
+  count: number;
+  /** First bucket (in archive order) with the group's newest updatedAt. */
+  bestUpdatedAt: number;
+  bestIndex: number;
+  /** Index of the first bucket an aggregation could record a ref from (updatedAt > -1); Infinity if none. */
+  firstEligibleIndex: number;
+};
+
+type RoutedGroup = {
+  globalEpoch: string;
+  sessionHash: string;
+  bestUpdatedAt: number;
+  model: PersistedRoutedModelRef;
+  /** Index of the first entry an aggregation could pick (updatedAt > -1); Infinity if none. */
+  firstEligibleIndex: number;
+};
+
+type ReducedArchive = {
+  /** Buckets folded per (globalEpoch, session, model, modelEpoch), in order of first appearance. */
+  groups: BucketGroup[];
+  routed: RoutedGroup[];
+  /** The same groups per session (each list in archive order). */
+  groupsBySession: Map<string, BucketGroup[]>;
+  routedBySession: Map<string, RoutedGroup[]>;
+  /** Model keys with buckets under each global epoch, sorted. */
+  keysByGlobalEpoch: Map<string, string[]>;
+};
+
 /**
- * Read the archive, re-parsing only when inode/size/mtime changed. A missing
- * file is an empty archive; an invalid one is reported as corrupt and treated
- * as empty, so raw shards stay the source of truth. A transient I/O failure
- * (Windows EPERM/EBUSY while another process or a scanner holds the file) is
- * never cached: the last good parse is returned when there is one, otherwise
- * an uncached corrupt result, so the next refresh retries.
+ * One parsed archive snapshot, cached per file identity. Besides the raw
+ * entries it lazily builds what a refresh needs: the per-day synthetic shards
+ * for the current-day view (archived days are past days, so normally none)
+ * and, per set of current epochs, the all-days aggregate pre-reduced to one
+ * entry per (session, model).
  */
-export async function readStatsArchive(path: string = SHARD_ARCHIVE_PATH): Promise<LoadedStatsArchive> {
-  const transientFailure = (): LoadedStatsArchive => {
-    const previous = archiveCache.get(path);
-    return previous && !previous.loaded.corrupt ? previous.loaded : buildLoaded(emptyStatsArchive(0), true, true);
-  };
-  let info: import("node:fs").Stats;
-  try {
-    info = await lstat(path);
-  } catch (error) {
-    if (getErrorCode(error) === "ENOENT") {
-      archiveCache.delete(path);
-      return buildLoaded(emptyStatsArchive(0), false, false);
+export class StatsArchiveView implements LoadedStatsArchive {
+  readonly absorbedIds: ReadonlySet<string>;
+  private syntheticShards?: PersistedStatsShardV7[];
+  private dayIndex?: Map<string, { buckets: number[]; routed: number[] }>;
+  private dayShards?: { day: string; shards: readonly PersistedStatsShardV7[] };
+  private reduced?: ReducedArchive;
+  private contributionCache?: { fingerprint: string; contribution: ShardContribution };
+
+  constructor(
+    readonly archive: StatsArchiveV1,
+    readonly present: boolean,
+    readonly corrupt: boolean,
+    readonly generation: number,
+    /** For 2.9.0's archive.json: its sha256; for a generation: the legacy file it was migrated from. */
+    readonly legacySha256?: string,
+  ) {
+    this.absorbedIds = new Set(Object.keys(archive.absorbed));
+  }
+
+  get shards(): readonly PersistedStatsShardV7[] {
+    return this.syntheticShards ??= archiveToSyntheticShards(this.archive);
+  }
+
+  /** The synthetic shards of `day`, in archive order. Do not mutate. */
+  shardsForDay(day: string): readonly PersistedStatsShardV7[] {
+    if (this.dayShards?.day === day) return this.dayShards.shards;
+    if (!this.dayIndex) {
+      this.dayIndex = new Map();
+      const slot = (entryDay: string) => {
+        let entry = this.dayIndex!.get(entryDay);
+        if (!entry) {
+          entry = { buckets: [], routed: [] };
+          this.dayIndex!.set(entryDay, entry);
+        }
+        return entry;
+      };
+      this.archive.buckets.forEach((bucket, index) => slot(bucket.day).buckets.push(index));
+      this.archive.routed.forEach((entry, index) => slot(entry.day).routed.push(index));
     }
-    return transientFailure();
+    const indexes = this.dayIndex.get(day);
+    const shards = indexes
+      ? [
+        ...indexes.buckets.map((index) => bucketToSyntheticShard(this.archive.buckets[index]!, index)),
+        ...indexes.routed.map((index) => routedToSyntheticShard(this.archive.routed[index]!, index)),
+      ]
+      : [];
+    this.dayShards = { day, shards };
+    return shards;
+  }
+
+  private reduce(): ReducedArchive {
+    if (this.reduced) return this.reduced;
+    const groups = new Map<string, BucketGroup>();
+    const keys = new Map<string, Set<string>>();
+    this.archive.buckets.forEach((bucket, index) => {
+      const id = JSON.stringify([bucket.globalEpoch, bucket.sessionHash, bucket.modelKey, bucket.modelEpoch]);
+      const group = groups.get(id);
+      if (!group) {
+        groups.set(id, {
+          order: groups.size,
+          globalEpoch: bucket.globalEpoch,
+          sessionHash: bucket.sessionHash,
+          key: bucket.modelKey,
+          modelEpoch: bucket.modelEpoch,
+          stats: bucket.stats,
+          count: 1,
+          bestUpdatedAt: bucket.updatedAt,
+          bestIndex: index,
+          firstEligibleIndex: bucket.updatedAt > -1 ? index : Infinity,
+        });
+        let set = keys.get(bucket.globalEpoch);
+        if (!set) keys.set(bucket.globalEpoch, set = new Set());
+        set.add(bucket.modelKey);
+        return;
+      }
+      if (group.count === 1) group.stats = cloneCacheStats(group.stats);
+      mergeStatsAcrossDays(group.stats, bucket.stats, null);
+      group.count++;
+      if (group.firstEligibleIndex === Infinity && bucket.updatedAt > -1) group.firstEligibleIndex = index;
+      if (bucket.updatedAt > group.bestUpdatedAt) {
+        group.bestUpdatedAt = bucket.updatedAt;
+        group.bestIndex = index;
+      }
+    });
+    const routed = new Map<string, RoutedGroup>();
+    this.archive.routed.forEach((entry, index) => {
+      const id = JSON.stringify([entry.globalEpoch, entry.sessionHash]);
+      const group = routed.get(id);
+      const eligible = entry.updatedAt > -1 ? index : Infinity;
+      if (!group) {
+        routed.set(id, { globalEpoch: entry.globalEpoch, sessionHash: entry.sessionHash, bestUpdatedAt: entry.updatedAt, model: entry.model, firstEligibleIndex: eligible });
+        return;
+      }
+      if (group.firstEligibleIndex === Infinity) group.firstEligibleIndex = eligible;
+      if (entry.updatedAt > group.bestUpdatedAt) {
+        group.bestUpdatedAt = entry.updatedAt;
+        group.model = entry.model;
+      }
+    });
+    const bySession = <T extends { sessionHash: string }>(items: T[]): Map<string, T[]> => {
+      const map = new Map<string, T[]>();
+      for (const item of items) {
+        let list = map.get(item.sessionHash);
+        if (!list) map.set(item.sessionHash, list = []);
+        list.push(item);
+      }
+      return map;
+    };
+    const groupList = Array.from(groups.values());
+    const routedList = Array.from(routed.values());
+    this.reduced = {
+      groups: groupList,
+      routed: routedList,
+      groupsBySession: bySession(groupList),
+      routedBySession: bySession(routedList),
+      keysByGlobalEpoch: new Map(Array.from(keys, ([epoch, set]) => [epoch, Array.from(set).sort()])),
+    };
+    return this.reduced;
   }
-  if (!info.isFile() || info.isSymbolicLink()) {
-    archiveCache.delete(path);
-    return buildLoaded(emptyStatsArchive(0), true, true);
+
+  /**
+   * The archive's share of an all-days aggregate under the aggregator's
+   * epochs. Equivalent to adding `shards` (only those of `sessions`, when
+   * given) at that point of the sequence; cached until an epoch, the
+   * session selection, or the archive changes.
+   */
+  async contribution(aggregator: Pick<ShardAggregator, "globalEpoch" | "modelEpoch">, only?: ReadonlySet<string>): Promise<ShardContribution> {
+    const reduced = this.reduce();
+    const globalEpoch = aggregator.globalEpoch;
+    let groupList = reduced.groups;
+    let routedList = reduced.routed;
+    let keys = reduced.keysByGlobalEpoch.get(globalEpoch) ?? [];
+    if (only) {
+      groupList = Array.from(only, (session) => reduced.groupsBySession.get(session) ?? []).flat()
+        .sort((a, b) => a.order - b.order);
+      routedList = Array.from(only, (session) => reduced.routedBySession.get(session) ?? []).flat();
+      keys = Array.from(new Set(groupList.filter((group) => group.globalEpoch === globalEpoch).map((group) => group.key))).sort();
+    }
+    const epochs = new Map<string, string>();
+    for (const key of keys) epochs.set(key, await aggregator.modelEpoch(key));
+    const fingerprint = JSON.stringify([globalEpoch, ...keys.flatMap((key) => [key, epochs.get(key)]), only ? Array.from(only).sort() : null]);
+    if (this.contributionCache?.fingerprint === fingerprint) return this.contributionCache.contribution;
+
+    type SessionOut = { sessionHash: string; instances: number; models: Array<{ key: string; stats: CacheStats; instances: number }> };
+    type ModelOut = { key: string; stats: CacheStats; instances: number; sessions: string[]; refUpdatedAt: number; refIndex: number; firstEligibleIndex: number };
+    const sessions = new Map<string, SessionOut>();
+    const models = new Map<string, ModelOut>();
+    for (const group of groupList) {
+      if (group.globalEpoch !== globalEpoch || group.modelEpoch !== epochs.get(group.key)) continue;
+      let session = sessions.get(group.sessionHash);
+      if (!session) sessions.set(group.sessionHash, session = { sessionHash: group.sessionHash, instances: 0, models: [] });
+      session.models.push({ key: group.key, stats: group.stats, instances: group.count });
+      session.instances += group.count;
+      const model = models.get(group.key);
+      if (!model) {
+        models.set(group.key, {
+          key: group.key,
+          stats: cloneCacheStats(group.stats),
+          instances: group.count,
+          sessions: [group.sessionHash],
+          refUpdatedAt: group.bestUpdatedAt,
+          refIndex: group.bestIndex,
+          firstEligibleIndex: group.firstEligibleIndex,
+        });
+        continue;
+      }
+      model.firstEligibleIndex = Math.min(model.firstEligibleIndex, group.firstEligibleIndex);
+      mergeStatsAcrossDays(model.stats, group.stats, null);
+      model.instances += group.count;
+      model.sessions.push(group.sessionHash);
+      // The sequential aggregation keeps the first shard with the newest updatedAt.
+      if (group.bestUpdatedAt > model.refUpdatedAt || (group.bestUpdatedAt === model.refUpdatedAt && group.bestIndex < model.refIndex)) {
+        model.refUpdatedAt = group.bestUpdatedAt;
+        model.refIndex = group.bestIndex;
+      }
+    }
+    const routed = routedList
+      .filter((group) => group.globalEpoch === globalEpoch && group.firstEligibleIndex !== Infinity)
+      .sort((a, b) => a.firstEligibleIndex - b.firstEligibleIndex)
+      .map((group) => ({ sessionHash: group.sessionHash, updatedAt: group.bestUpdatedAt, model: group.model }));
+    const contribution: ShardContribution = {
+      sessions: Array.from(sessions.values()),
+      models: Array.from(models.values(), ({ key, stats, instances, sessions: list }) => ({ key, stats, instances, sessions: list })),
+      refs: Array.from(models.values())
+        .filter((model) => model.firstEligibleIndex !== Infinity)
+        .sort((a, b) => a.firstEligibleIndex - b.firstEligibleIndex)
+        .map((model) => {
+          const bucket = this.archive.buckets[model.refIndex]!;
+          return { key: model.key, updatedAt: model.refUpdatedAt, ref: { provider: bucket.provider, id: bucket.modelId, name: bucket.modelName ?? bucket.modelId } };
+        }),
+      routed,
+    };
+    this.contributionCache = { fingerprint, contribution };
+    return contribution;
   }
-  const cached = archiveCache.get(path);
-  if (cached && cached.ino === info.ino && cached.size === info.size && cached.mtimeMs === info.mtimeMs) return cached.loaded;
-  let text: string;
-  try {
-    text = await readFile(path, "utf8");
-  } catch (error) {
-    // A file renamed away between lstat and read is simply gone.
-    if (getErrorCode(error) === "ENOENT") {
-      archiveCache.delete(path);
-      return buildLoaded(emptyStatsArchive(0), false, false);
+}
+
+/**
+ * Aggregate `leading` shards, then the archive, then `trailing` shards: the
+ * same result as aggregateStatsShardsV7([...leading, ...archive.shards,
+ * ...trailing], day), without materializing the archive. With `sessions`,
+ * only shards of those sessions take part. The current-day view only sees
+ * archive entries of that day; the all-days view merges the archive's
+ * pre-reduced contribution.
+ */
+export async function aggregateStatsWithArchiveV7(
+  leading: readonly PersistedStatsShardV7[],
+  archive: StatsArchiveView,
+  trailing: readonly PersistedStatsShardV7[] = [],
+  day: string | null = currentLocalDay(),
+  sessions?: ReadonlySet<string>,
+): Promise<ShardAggregate> {
+  const aggregator = await createShardAggregatorV7(day);
+  const include = (shard: PersistedStatsShardV7) => !sessions || sessions.has(shard.sessionHash);
+  for (const shard of leading) if (include(shard)) await aggregator.add(shard);
+  if (day === null) aggregator.addContribution(await archive.contribution(aggregator, sessions));
+  else for (const shard of archive.shardsForDay(day)) if (include(shard)) await aggregator.add(shard);
+  for (const shard of trailing) if (include(shard)) await aggregator.add(shard);
+  return aggregator.finish();
+}
+
+type ArchiveSource =
+  | { kind: "none" }
+  | { kind: "invalid"; path: string }
+  | { kind: "legacy"; path: string; info: import("node:fs").Stats }
+  | { kind: "generation"; path: string; generation: number; info: import("node:fs").Stats };
+
+const SOURCE_ATTEMPTS = 5;
+
+/**
+ * The newest snapshot: the highest generation, or 2.9.0's archive.json while
+ * no generation exists. Retries when a listed generation disappears (it was
+ * superseded and cleaned up meanwhile). Throws on other I/O errors.
+ */
+async function locateArchiveSource(archivePath: string): Promise<ArchiveSource> {
+  for (let attempt = 0; attempt < SOURCE_ATTEMPTS; attempt++) {
+    const generations = await listStatsArchiveGenerations(archivePath);
+    const generation = generations.at(-1);
+    if (generation !== undefined) {
+      const path = statsArchiveGenerationPath(archivePath, generation);
+      try {
+        const info = await lstat(path);
+        if (!info.isFile() || info.isSymbolicLink()) return { kind: "invalid", path };
+        return { kind: "generation", path, generation, info };
+      } catch (error) {
+        if (getErrorCode(error) === "ENOENT") continue;
+        throw error;
+      }
+    }
+    try {
+      const info = await lstat(archivePath);
+      if (info.isFile() && !info.isSymbolicLink()) return { kind: "legacy", path: archivePath, info };
+      // The migration tombstone is created only after a generation exists:
+      // the listing raced a cleanup, or the generations were removed by hand.
+      if (info.isDirectory()) continue;
+      return { kind: "invalid", path: archivePath };
+    } catch (error) {
+      if (getErrorCode(error) === "ENOENT") return { kind: "none" };
+      throw error;
     }
-    return transientFailure();
   }
-  let loaded: LoadedStatsArchive;
+  return { kind: "none" };
+}
+
+type CachedArchive = { path: string; ino: number | bigint; size: number; mtimeMs: number; view: StatsArchiveView };
+const archiveCache = new Map<string, CachedArchive>();
+
+export function clearStatsArchiveCache(): void {
+  archiveCache.clear();
+}
+
+const emptyView = (): StatsArchiveView => new StatsArchiveView(emptyStatsArchive(0), false, false, 0);
+const corruptView = (): StatsArchiveView => new StatsArchiveView(emptyStatsArchive(0), true, true, 0);
+
+function sha256(text: string | Buffer): string {
+  return createHash("sha256").update(text).digest("hex");
+}
+
+function parseArchiveFile(source: Extract<ArchiveSource, { kind: "legacy" | "generation" }>, bytes: Buffer): StatsArchiveView {
+  let value: unknown;
   try {
-    const parsed = parseStatsArchive(JSON.parse(text));
-    loaded = parsed ? buildLoaded(parsed, true, false) : buildLoaded(emptyStatsArchive(0), true, true);
+    value = JSON.parse(bytes.toString("utf8"));
   } catch {
-    loaded = buildLoaded(emptyStatsArchive(0), true, true);
+    return corruptView();
+  }
+  if (source.kind === "legacy") {
+    const archive = parseStatsArchive(value);
+    return archive ? new StatsArchiveView(archive, true, false, 0, sha256(bytes)) : corruptView();
   }
-  archiveCache.set(path, { ino: info.ino, size: info.size, mtimeMs: info.mtimeMs, loaded });
-  return loaded;
+  const parsed = parseStatsArchiveV2(value);
+  if (!parsed || parsed.generation !== source.generation) return corruptView();
+  return new StatsArchiveView(parsed.archive, true, false, parsed.generation, parsed.legacySha256);
 }
 
-type ArchiveFileIdentity = { ino: number | bigint; size: number; mtimeMs: number } | undefined;
+export type ReadStatsArchiveOptions = {
+  /** Throw on a transient I/O failure instead of serving the last good snapshot (compaction). */
+  strict?: boolean;
+};
 
-async function archiveFileIdentity(path: string): Promise<ArchiveFileIdentity> {
-  try {
-    const info = await lstat(path);
-    return { ino: info.ino, size: info.size, mtimeMs: info.mtimeMs };
-  } catch (error) {
-    if (getErrorCode(error) === "ENOENT") return undefined;
-    throw error;
+/**
+ * Read the newest archive snapshot, re-parsing only when the file (path,
+ * inode, size, mtime) changed. No archive is an empty one; an invalid one is
+ * reported as corrupt and treated as empty, so raw shards stay the source of
+ * truth. A transient I/O failure (Windows EPERM/EBUSY while another process
+ * or a scanner holds the file) is never cached: the last good snapshot is
+ * returned when there is one, otherwise an uncached corrupt result, so the
+ * next refresh retries.
+ */
+export async function readStatsArchiveView(archivePath: string = SHARD_ARCHIVE_PATH, options: ReadStatsArchiveOptions = {}): Promise<StatsArchiveView> {
+  const transientFailure = (error: unknown): StatsArchiveView => {
+    if (options.strict) throw error;
+    const previous = archiveCache.get(archivePath);
+    return previous && !previous.view.corrupt ? previous.view : corruptView();
+  };
+  for (let attempt = 0; attempt < SOURCE_ATTEMPTS; attempt++) {
+    let source: ArchiveSource;
+    try {
+      source = await locateArchiveSource(archivePath);
+    } catch (error) {
+      return transientFailure(error);
+    }
+    if (source.kind === "none") {
+      archiveCache.delete(archivePath);
+      return emptyView();
+    }
+    if (source.kind === "invalid") {
+      archiveCache.delete(archivePath);
+      return corruptView();
+    }
+    const { info } = source;
+    const cached = archiveCache.get(archivePath);
+    // Generations are immutable, so (path, identity) is a safe cache key. 2.9.0
+    // rewrites archive.json in place of the old one, and on filesystems with
+    // ino 0 and coarse mtimes (FAT/SMB on Windows) a rewrite of the same size
+    // keeps the same identity. Compaction migrates from it and then deletes
+    // shards, so it always re-reads that file instead of trusting the cache.
+    const cacheable = !(options.strict && source.kind === "legacy");
+    if (cacheable && cached && cached.path === source.path && cached.ino === info.ino && cached.size === info.size && cached.mtimeMs === info.mtimeMs) {
+      return cached.view;
+    }
+    let bytes: Buffer;
+    try {
+      bytes = await readFile(source.path);
+    } catch (error) {
+      // Superseded and cleaned up (or migrated) between listing and reading.
+      if (getErrorCode(error) === "ENOENT") continue;
+      // On Windows a generation that is being deleted while someone holds it
+      // open fails with EPERM instead; a newer one exists by then.
+      if (source.kind === "generation") {
+        const newest = await listStatsArchiveGenerations(archivePath).then((list) => list.at(-1), () => undefined);
+        if (newest !== undefined && newest > source.generation) continue;
+      }
+      return transientFailure(error);
+    }
+    const view = parseArchiveFile(source, bytes);
+    archiveCache.set(archivePath, { path: source.path, ino: info.ino, size: info.size, mtimeMs: info.mtimeMs, view });
+    return view;
   }
+  return transientFailure(new Error(`stats archive ${archivePath} kept changing while being read`));
 }
 
-function sameArchiveFile(a: ArchiveFileIdentity, b: ArchiveFileIdentity): boolean {
-  if (!a || !b) return a === b;
-  return a.ino === b.ino && a.size === b.size && a.mtimeMs === b.mtimeMs;
+/** Reference-form read (used by tests and tools): the snapshot with synthetic shards available. */
+export async function readStatsArchive(archivePath: string = SHARD_ARCHIVE_PATH, options: ReadStatsArchiveOptions = {}): Promise<LoadedStatsArchive> {
+  return readStatsArchiveView(archivePath, options);
 }
 
 /** Remove archive temp files a crash left behind (shard cleanup only covers shards/). */
@@ -324,17 +874,126 @@ async function cleanupStaleArchiveTemps(archivePath: string, now: number): Promi
   }
 }
 
-async function writeStatsArchive(path: string, archive: StatsArchiveV1): Promise<void> {
-  await mkdir(dirname(path), { recursive: true });
-  const tempPath = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
+/** Identity of whatever sits at 2.9.0's archive.json path (undefined when nothing does). */
+async function legacyArchiveIdentity(archivePath: string): Promise<string | undefined> {
   try {
-    await writeFile(tempPath, JSON.stringify(archive) + "\n", "utf8");
-    await renameWithRetry(tempPath, path);
+    const info = await lstat(archivePath);
+    return JSON.stringify([String(info.ino), info.size, info.mtimeMs, info.isFile()]);
   } catch (error) {
+    if (getErrorCode(error) === "ENOENT") return undefined;
+    throw error;
+  }
+}
+
+/** Test hooks for the publish protocol; production never sets them. */
+export type ArchivePublishHooks = {
+  /** Runs after every publish check passed, right before the new generation becomes visible. */
+  beforeCommit?: () => Promise<void>;
+};
+
+/**
+ * Publish `generation` atomically and exclusively: the snapshot is written
+ * and flushed to a private temp file, then hard-linked to its final name.
+ * link() fails with EEXIST when that generation already exists, on Windows
+ * and POSIX alike, so of all writers that read generation N exactly one can
+ * publish N+1. Resolves false when another writer won.
+ */
+async function commitStatsArchiveGeneration(archivePath: string, generation: number, text: string, hooks: ArchivePublishHooks): Promise<boolean> {
+  await mkdir(dirname(archivePath), { recursive: true });
+  const target = statsArchiveGenerationPath(archivePath, generation);
+  const tempPath = `${target}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`;
+  try {
+    const handle = await open(tempPath, "wx");
+    try {
+      await handle.writeFile(text, "utf8");
+      await handle.sync();
+    } finally {
+      await handle.close();
+    }
+    await hooks.beforeCommit?.();
+    try {
+      await link(tempPath, target);
+    } catch (error) {
+      if (getErrorCode(error) === "EEXIST") return false;
+      // A filesystem without hard links (FAT/exFAT) cannot publish safely;
+      // the caller reports it and raw shards simply stay where they are.
+      throw error;
+    }
+    return true;
+  } finally {
     await unlink(tempPath).catch(() => undefined);
+  }
+}
+
+/**
+ * Drop generations older than the one before `current`. The previous one is
+ * kept for readers that listed the directory just before `current` appeared;
+ * a reader that still finds its file gone re-lists and reads a newer one.
+ */
+async function removeSupersededGenerations(archivePath: string, current: number): Promise<void> {
+  for (const generation of await listStatsArchiveGenerations(archivePath).catch(() => [] as number[])) {
+    if (generation >= current - 1) continue;
+    await unlink(statsArchiveGenerationPath(archivePath, generation)).catch(() => undefined);
+  }
+}
+
+const LEGACY_TOMBSTONE_README =
+  "pi-cache-optimizer stores its stats archive in the archive.<generation>.json files next to this directory.\n" +
+  "This directory replaces the single archive.json of version 2.9.0 so that 2.9.0 can no longer write it\n" +
+  "(it would publish without a compare-and-swap). 2.9.0 reports the archive as unreadable and keeps raw shards.\n" +
+  "Do not remove it while any Pi process might still run pi-cache-optimizer 2.9.0.\n";
+
+/**
+ * After migrating (or whenever generations exist), make sure 2.9.0 cannot
+ * write archive.json: move a file there aside and put a directory in its
+ * place. A file whose content is the one this archive was migrated from is
+ * simply deleted. Anything else was written by 2.9.0 after the migration
+ * (only possible if the directory was removed); it is never counted, because
+ * merging it could double count, and is kept for inspection.
+ */
+async function ensureLegacyArchiveTombstone(archivePath: string, migratedFromSha256: string | undefined): Promise<void> {
+  let info: import("node:fs").Stats | undefined;
+  try {
+    info = await lstat(archivePath);
+  } catch (error) {
+    if (getErrorCode(error) !== "ENOENT") return;
+  }
+  if (info?.isDirectory()) return;
+  if (info) {
+    const retired = `${archivePath}.${process.pid}.${randomUUID()}.retired.tmp`;
+    try {
+      await renameWithRetry(archivePath, retired);
+    } catch {
+      return; // retried by the next pass; readers ignore archive.json once a generation exists
+    }
+    let sha: string | undefined;
+    try {
+      sha = sha256(await readFile(retired));
+    } catch {}
+    if (sha !== undefined && sha === migratedFromSha256) {
+      await unlink(retired).catch(() => undefined);
+    } else {
+      const orphan = join(dirname(archivePath), `archive.legacy-${Date.now()}.json`);
+      await rename(retired, orphan).catch(() => undefined);
+      console.warn(`${LOG_PREFIX}: ignored a stats archive written by an older pi-cache-optimizer after migration; kept as ${orphan}`);
+    }
+  }
+  try {
+    await mkdir(archivePath);
+    await writeFile(join(archivePath, "README.txt"), LEGACY_TOMBSTONE_README, "utf8");
+  } catch {}
+}
+
+/** Instance ids of every shard-named file in `directory`, readable or not. */
+async function listShardInstanceIds(directory: string): Promise<Set<string>> {
+  let names: string[];
+  try {
+    names = await readdir(directory);
+  } catch (error) {
+    if (getErrorCode(error) === "ENOENT") return new Set();
     throw error;
   }
-  archiveCache.delete(path);
+  return new Set(names.filter((name) => /^[0-9a-f-]+\.json$/i.test(name)).map((name) => name.slice(0, -".json".length)));
 }
 
 /** A shard maintenance may fold away: from a past day and no longer written to. */
@@ -356,10 +1015,13 @@ export type CompactionResult = {
   skippedCorrupt: boolean;
   /**
    * True when the archive, an epoch, or lock ownership changed while this
-   * pass ran (e.g. a stale-lock takeover or a concurrent reset). Nothing was
-   * published or deleted; the next pass retries.
+   * pass ran (e.g. a stale-lock takeover or a concurrent reset), or another
+   * compactor published a newer generation first. No raw shard was deleted
+   * and no visible archive was replaced; the next pass retries.
    */
   skippedConcurrent: boolean;
+  /** Generation this pass published, when it published one. */
+  generation?: number;
 };
 
 export type CompactionOptions = {
@@ -367,17 +1029,27 @@ export type CompactionOptions = {
   lockToken?: string;
   /** Test hook: runs after the plan is built, right before the publish checks. */
   beforePublish?: () => Promise<void>;
-};
+} & ArchivePublishHooks;
 
-const bucketId = (b: Pick<StatsArchiveBucketV1, "globalEpoch" | "sessionHash" | "modelKey" | "modelEpoch" | "day">): string =>
-  JSON.stringify([b.globalEpoch, b.sessionHash, b.modelKey, b.modelEpoch, b.day]);
+/**
+ * Buckets expire by their newest folded shard. Shards of one day can be
+ * closed much later (an idle process closes yesterday's shard at its next
+ * rollover, possibly weeks later), so buckets are also split by the UTC day
+ * of `updatedAt`. Otherwise the earliest shard would outlive its own 60-day
+ * retention by as long as the latest one, and archived totals would exceed
+ * what the raw shards would have shown. The remaining error is under a day.
+ */
+const ARCHIVE_EXPIRY_GRANULARITY_MS = 24 * 60 * 60 * 1000;
+const bucketId = (b: Pick<StatsArchiveBucketV1, "globalEpoch" | "sessionHash" | "modelKey" | "modelEpoch" | "day" | "updatedAt">): string =>
+  JSON.stringify([b.globalEpoch, b.sessionHash, b.modelKey, b.modelEpoch, b.day, Math.floor(b.updatedAt / ARCHIVE_EXPIRY_GRANULARITY_MS)]);
 const routedId = (r: Pick<StatsArchiveRoutedV1, "globalEpoch" | "sessionHash">): string =>
   JSON.stringify([r.globalEpoch, r.sessionHash]);
 
 /**
  * Fold archivable shards into the archive, then delete them. Call only while
  * holding the maintenance lock. Order matters for crash safety: the archive
- * (with the absorbed ids) is published before any raw shard is deleted.
+ * (with the absorbed ids) is published before any raw shard is deleted, and
+ * only by a pass whose generation is the newest one.
  */
 export async function compactStatsShardsV7(
   now = Date.now(),
@@ -387,14 +1059,20 @@ export async function compactStatsShardsV7(
 ): Promise<CompactionResult> {
   const result: CompactionResult = { absorbed: 0, removed: 0, dropped: 0, skippedCorrupt: false, skippedConcurrent: false };
   await cleanupStaleArchiveTemps(archivePath, now);
-  clearStatsArchiveCache();
-  const identityAtRead = await archiveFileIdentity(archivePath);
-  const loaded = await readStatsArchive(archivePath);
-  if (loaded.corrupt) {
+  // Taken before the read so a 2.9.0 write racing it is noticed below.
+  const legacyIdentityAtRead = await legacyArchiveIdentity(archivePath);
+  // The snapshot this pass builds on. Generations are immutable, so the
+  // readers' cached parse is safe to reuse; the publish below is a
+  // compare-and-swap against exactly this generation.
+  const base = await readStatsArchiveView(archivePath, { strict: true });
+  if (base.corrupt) {
     result.skippedCorrupt = true;
     return result;
   }
-  const previous = loaded.archive;
+  // 2.9.0's archive.json: publish it as the first generation even when
+  // nothing else changed, so 2.9.0 can be locked out of writing it.
+  const migrating = base.present && base.generation === 0;
+  const previous = base.archive;
   const today = currentLocalDay();
   // Strict reads: compaction permanently drops counters whose epoch differs,
   // so an unreadable epoch file must abort instead of falling back to the
@@ -433,9 +1111,14 @@ export async function compactStatsShardsV7(
   }
 
   const shards = await readValidStatsShardsV7(directory);
-  const presentIds = new Set(shards.map((shard) => shard.instanceId));
-  const absorbed: Record<string, number> = {};
   // Keep ids whose raw file still exists (a crash or a failed delete left it).
+  // Presence is decided by the directory listing, not by a successful parse:
+  // a leftover that is only transiently unreadable (Windows EPERM/EBUSY while
+  // a scanner or sync client holds it) must stay absorbed, or it would be
+  // counted raw again once readable and then folded into the archive twice.
+  const presentIds = await listShardInstanceIds(directory);
+  for (const shard of shards) presentIds.add(shard.instanceId);
+  const absorbed: Record<string, number> = {};
   for (const [id, at] of Object.entries(previous.absorbed)) if (presentIds.has(id)) absorbed[id] = at;
 
   for (const shard of shards) {
@@ -452,7 +1135,7 @@ export async function compactStatsShardsV7(
     }
     for (const [key, entry] of Object.entries(shard.models)) {
       if (entry.modelEpoch !== await currentModelEpoch(key)) continue;
-      const id = bucketId({ globalEpoch, sessionHash: shard.sessionHash, modelKey: key, modelEpoch: entry.modelEpoch, day: shard.day });
+      const id = bucketId({ globalEpoch, sessionHash: shard.sessionHash, modelKey: key, modelEpoch: entry.modelEpoch, day: shard.day, updatedAt: shard.lifecycle.updatedAt });
       const existing = buckets.get(id);
       const newer = !existing || existing.updatedAt <= shard.lifecycle.updatedAt;
       buckets.set(id, {
@@ -471,38 +1154,61 @@ export async function compactStatsShardsV7(
     }
   }
 
+  await touchStatsMaintenanceLock(options.lockToken);
+
   const absorbedChanged =
     Object.keys(absorbed).length !== Object.keys(previous.absorbed).length ||
     Object.keys(absorbed).some((id) => !(id in previous.absorbed));
-  if (result.absorbed > 0 || result.dropped > 0 || absorbedChanged) {
+  const skip = (): CompactionResult => {
+    result.skippedConcurrent = true;
+    result.absorbed = 0;
+    result.dropped = 0;
+    return result;
+  };
+  if (result.absorbed > 0 || result.dropped > 0 || absorbedChanged || migrating) {
     await options.beforePublish?.();
-    // Publish only if nothing this plan depends on moved underneath it: a
-    // reset during the pass (counters written under the new epoch would be
-    // dropped as stale), or another compactor after a stale-lock takeover
-    // (overwriting its archive would lose shards it already deleted).
+    // Cheap early exits: lost the lock, someone published meanwhile, or a
+    // reset landed during the pass (counters written under the new epoch
+    // would be dropped as stale). The compare-and-swap below is what makes
+    // a stale holder safe; these only avoid wasted work.
     let concurrent =
       !(await holdsStatsMaintenanceLock(options.lockToken)) ||
-      !sameArchiveFile(identityAtRead, await archiveFileIdentity(archivePath)) ||
+      ((await listStatsArchiveGenerations(archivePath)).at(-1) ?? 0) !== base.generation ||
+      (base.generation === 0 && (await legacyArchiveIdentity(archivePath)) !== legacyIdentityAtRead) ||
       (await readGlobalStatsEpochStrict()) !== globalEpoch;
     for (const [key, epoch] of modelEpochs) {
       if (concurrent) break;
       if ((await readModelStatsEpochStrict(key)) !== epoch) concurrent = true;
     }
-    if (concurrent) {
-      result.skippedConcurrent = true;
-      result.absorbed = 0;
-      result.dropped = 0;
-      return result;
-    }
-    await writeStatsArchive(archivePath, {
+    if (concurrent) return skip();
+    const next: StatsArchiveV1 = {
       version: 1,
       kind: "pi-cache-optimizer-archive",
       updatedAt: now,
       absorbed,
       buckets: Array.from(buckets.values()),
       routed: Array.from(routed.values()),
-    });
+    };
+    const generation = base.generation + 1;
+    const text = JSON.stringify(encodeStatsArchiveV2(next, generation, base.legacySha256)) + "\n";
+    if (!(await commitStatsArchiveGeneration(archivePath, generation, text, options))) return skip();
+    // A generation above ours means ours is invisible to readers (only
+    // possible after our base was superseded and cleaned up while this pass
+    // was suspended): it must not delete what its snapshot covers.
+    if (((await listStatsArchiveGenerations(archivePath)).at(-1) ?? 0) !== generation) return skip();
+    result.generation = generation;
+    try {
+      const info = await lstat(statsArchiveGenerationPath(archivePath, generation));
+      archiveCache.set(archivePath, {
+        path: statsArchiveGenerationPath(archivePath, generation),
+        ino: info.ino, size: info.size, mtimeMs: info.mtimeMs,
+        view: new StatsArchiveView(next, true, false, generation, base.legacySha256),
+      });
+    } catch {}
+    await removeSupersededGenerations(archivePath, generation);
   }
+  if (result.generation !== undefined || base.generation > 0) await ensureLegacyArchiveTombstone(archivePath, base.legacySha256);
+  await touchStatsMaintenanceLock(options.lockToken);
 
   // Only now that the archive covering them is durable, delete the originals.
   const toDelete = Object.keys(absorbed);
@@ -535,6 +1241,11 @@ export async function maybeRunStatsMaintenance(now = Date.now()): Promise<StatsM
   if (!cleanupDue && !compactionDue) return result;
   const lockToken = await acquireStatsMaintenanceLock(now);
   if (!lockToken) return result;
+  // Heartbeat so a long pass (a huge archive on a slow, scanned disk) is not
+  // taken over as stale. It stops while the process is suspended; the
+  // archive compare-and-swap covers that case.
+  const heartbeat = setInterval(() => void touchStatsMaintenanceLock(lockToken), STATS_MAINTENANCE_HEARTBEAT_MS);
+  heartbeat.unref?.();
   try {
     if (cleanupDue) {
       result.cleanupRemoved = await cleanupStatsShardsV7(now);
@@ -554,6 +1265,7 @@ export async function maybeRunStatsMaintenance(now = Date.now()): Promise<StatsM
       if (!result.compaction?.skippedConcurrent) await writeMaintenanceMarker(SHARD_COMPACTION_MARKER_PATH, now);
     }
   } finally {
+    clearInterval(heartbeat);
     await releaseStatsMaintenanceLock(lockToken);
   }
   return result;