session-orchestrator 5.2.0 → 5.3.0

package/scripts/lib/events-rotation.mjs

@@ -9,24 +9,182 @@
  * per-append overhead is wasteful given ~6 KiB/day growth.
  *
  * Rename safety (POSIX): atomic rename is safe with in-flight writers. Old fds
- * continue writing to the original inode (now `events.jsonl.1`); new writers
- * will open the new file on next append.
+ * continue writing to the original inode (now the archive); new writers will
+ * open the new file on next append.
+ *
+ * ## The ledger carries its own break (#1401)
+ *
+ * Rotation used to be INVISIBLE. It wrote no event and no durable log line —
+ * success surfaced only as a `console.error` from the SessionStart hook, whose
+ * stderr the harness discards. So a rotation and a DELETED archive produced
+ * byte-identical evidence: an events.jsonl that simply starts later than it
+ * used to. Measured 2026-09-19: `events.jsonl.1` (53,896 lines, 2026-04-12 →
+ * 2026-09-18) was destroyed by a wave subagent's `touch <path> && rm -f <path>`
+ * ignore-probe that adopted the existing 10 MB file, and nothing anywhere
+ * recorded that the file had ever existed.
+ *
+ * Since #1401 the FIRST line of every new active file is an
+ * `orchestrator.events.rotated` record naming the archive, its size, its line
+ * count and its timestamp range. That record is the tombstone: it is what lets
+ * {@link readEventsWithRotations} in `events.mjs` report a MISSING archive as a
+ * finding instead of silently returning a shorter history.
+ *
+ * ## Why `_archive/<name>` and not the `.1`..`.N` ring
+ *
+ * The ring was replaced in #1401, for a structural reason rather than taste:
+ * its shift step RENAMES every surviving archive on each rotation, so the
+ * `archived_as` pointer above would go stale the moment the next rotation ran
+ * and every reader would report a phantom gap. A durable pointer needs a
+ * durable name. Three further consequences, all measured or direct:
+ *
+ *   - `_archive/events-<first>_<last>.jsonl` says what it holds; `.1` does not,
+ *     which is how a 5-month ledger read as scratch to the agent that deleted it.
+ *   - `.orchestrator/metrics/_archive/` is already gitignored (`.gitignore`) and
+ *     already the repo's archive convention for ledger data.
+ *   - Census 2026-09-19 @ 8f15f77b — `rg -n 'jsonl\.1|jsonl\.[0-9]' scripts/ hooks/`
+ *     (excluding tests) found ZERO code readers of the ring, so nothing in the
+ *     codebase breaks. Two live fleet repos DO carry a ~10 MB `events.jsonl.1`
+ *     on disk, which is why the READER still reads the legacy ring; this writer
+ *     never creates, shifts or prunes one.
  */
 
-import { statSync, renameSync, unlinkSync, existsSync } from 'node:fs';
+import {
+  statSync,
+  renameSync,
+  unlinkSync,
+  existsSync,
+  readFileSync,
+  appendFileSync,
+  mkdirSync,
+  readdirSync,
+} from 'node:fs';
+import path from 'node:path';
+import {
+  ARCHIVE_DIR_NAME,
+  ARCHIVE_NAME_RE,
+  ROTATION_EVENT,
+  parseEventLines,
+  stampEventSchemaVersion,
+  summarizeEventRecords,
+  validateEventRecord,
+} from './events-schema.mjs';
+
+// The archive naming contract (`ROTATION_EVENT`, `ARCHIVE_DIR_NAME`,
+// `ARCHIVE_NAME_RE`, `LEGACY_RING_MAX`) is defined in `events-schema.mjs`, the
+// zero-fs module the reader in `events.mjs` also loads — see the comment there
+// for the hook-closure measurement that put it on that side.
+
+/**
+ * `2026-04-12T06:33:01.123Z` → `20260412T063301Z` — filename-safe, and
+ * lexicographically ordered, which is what lets the pruner sort archives
+ * chronologically by name alone.
+ * @param {string} iso
+ * @returns {string}
+ */
+function compactStamp(iso) {
+  return iso.replace(/\.\d+/, '').replace(/[-:]/g, '');
+}
+
+/**
+ * An archive path inside `dir` that does not yet exist.
+ *
+ * `renameSync` overwrites its destination SILENTLY, so a name collision would
+ * destroy an existing archive — the exact loss class #1401 exists to prevent.
+ * Exhausting the suffix range therefore THROWS rather than returning a path
+ * that would overwrite: the caller's catch turns that into `reason: 'error'`,
+ * leaving the active log in place and losing nothing.
+ *
+ * @param {string} dir
+ * @param {string|null} firstTs
+ * @param {string|null} lastTs
+ * @returns {string}
+ */
+function uniqueArchivePath(dir, firstTs, lastTs) {
+  const from = firstTs ? compactStamp(firstTs) : 'unknown';
+  const to = lastTs ? compactStamp(lastTs) : compactStamp(new Date().toISOString());
+  const base = `events-${from}_${to}`;
+  let candidate = path.join(dir, `${base}.jsonl`);
+  // Ceiling (BV-004): 999 same-range archives. Rotation fires 2-3x/year at the
+  // measured ~6 KiB/day growth, and the range is content-derived, so a second
+  // collision already implies something is re-rotating identical content.
+  // Revisit if a `pruned`-less archive dir is ever seen holding a `-3` suffix.
+  for (let n = 2; existsSync(candidate); n += 1) {
+    if (n > 999) {
+      throw new Error(`events-rotation: no free archive name for ${base} in ${dir}`);
+    }
+    candidate = path.join(dir, `${base}-${n}.jsonl`);
+  }
+  return candidate;
+}
+
+/**
+ * Enforce `maxBackups` over the archives in `dir`, oldest first.
+ *
+ * The cap is kept rather than dropped because `events-rotation.max-backups` is
+ * a live, documented config key: a key whose reader silently stops enforcing it
+ * is the "config key nobody produces" defect in reverse. At the measured
+ * rotation rate `max-backups: 5` is roughly two years of history.
+ *
+ * Only names matching {@link ARCHIVE_NAME_RE} are eligible, and the archive
+ * just written is never a candidate. A failed unlink is swallowed: leaving one
+ * archive too many is strictly better than aborting a completed rotation.
+ *
+ * @param {string} dir
+ * @param {number} maxBackups
+ * @param {string} keepPath — absolute path of the archive written by this run.
+ * @returns {string[]} absolute paths actually deleted.
+ */
+function pruneArchives(dir, maxBackups, keepPath) {
+  const pruned = [];
+  let names;
+  try {
+    names = readdirSync(dir).filter((name) => ARCHIVE_NAME_RE.test(name)).sort();
+  } catch {
+    return pruned;
+  }
+  // `sort()` orders by the leading `YYYYMMDDTHHMMSSZ` first stamp, i.e. oldest
+  // first. An `unknown_` prefix sorts AFTER every digit, so an archive whose
+  // range could not be derived is treated as newest and outlives the dated ones
+  // — conservative on purpose: never delete the file you understand least.
+  const excess = names.length - maxBackups;
+  for (let i = 0; i < excess; i += 1) {
+    const victim = path.join(dir, names[i]);
+    if (victim === keepPath) continue;
+    try {
+      unlinkSync(victim);
+      pruned.push(victim);
+    } catch {
+      /* keep it — an un-prunable archive is not a rotation failure */
+    }
+  }
+  return pruned;
+}
 
 /**
  * Rotate the events log if it exceeds `maxSizeMb`.
  *
- * Shift scheme: `.N-1` → `.N`, …, `.1` → `.2`, active `events.jsonl` → `.1`.
- * The oldest backup (`events.jsonl.{maxBackups}`) is deleted before shifting.
+ * On rotation the active file is renamed into
+ * `<dir>/_archive/events-<firstTs>_<lastTs>.jsonl` and an
+ * `orchestrator.events.rotated` record is appended to the now-absent active
+ * path, making it that file's first line. Archives beyond `maxBackups` are
+ * pruned (oldest first) and named in the record's `pruned` field, so the
+ * deletion is itself in the ledger.
+ *
+ * The record is written SYNCHRONOUSLY here rather than via `emitEvent()` for
+ * two reasons: it must land between the rename and any other writer's first
+ * append to be the first line, and `emitEvent()`'s async correlation lookups
+ * (session lock, wave manifest) describe a session, not a file operation. It
+ * still goes through the schema module's own stamper and validator, so it is
+ * not a raw writer inventing a shape.
  *
  * @param {object} opts
  * @param {string}  opts.logPath     — absolute path to `events.jsonl`
  * @param {number}  opts.maxSizeMb   — integer 1..1024
  * @param {number}  opts.maxBackups  — integer 1..20
  * @param {boolean} opts.enabled     — if false, returns early
- * @returns {{rotated: boolean, reason?: string, archivedAs?: string, sizeBefore?: number, maxBackups?: number, error?: string}}
+ * @returns {{rotated: boolean, reason?: string, archivedAs?: string, sizeBefore?: number,
+ *            maxBackups?: number, lines?: number, firstTs?: string|null, lastTs?: string|null,
+ *            malformedLines?: number, pruned?: string[], recordWritten?: boolean, error?: string}}
  */
 export function maybeRotate({ logPath, maxSizeMb, maxBackups, enabled } = {}) {
   // --- Input validation (throw — programmer error, not runtime fs failure) ---
@@ -44,43 +202,81 @@ export function maybeRotate({ logPath, maxSizeMb, maxBackups, enabled } = {}) {
     return { rotated: false, reason: 'disabled' };
   }
 
+  // Set once the rename has happened. After that point the 10 MB HAS moved, so
+  // a later failure must never be reported as `rotated: false` — a caller that
+  // believes nothing happened is exactly how a rotation goes unnoticed.
+  let archivedAs = null;
+  let sizeBefore = 0;
+
   try {
     if (!existsSync(logPath)) {
       return { rotated: false, reason: 'no-file' };
     }
 
-    const sizeBefore = statSync(logPath).size;
+    sizeBefore = statSync(logPath).size;
     const threshold = maxSizeMb * 1024 * 1024;
     if (sizeBefore < threshold) {
       return { rotated: false, reason: 'under-threshold' };
     }
 
-    // Drop the oldest backup if it exists — this keeps the ring bounded.
-    const oldestPath = `${logPath}.${maxBackups}`;
-    if (existsSync(oldestPath)) {
-      unlinkSync(oldestPath);
-    }
+    // Read BEFORE the rename: the content is what names the archive. Cost
+    // ceiling (BV-004): one full read of a file at the rotation threshold —
+    // ~50 ms and ~40 MB transient at the default 10 MB cap, paid 2-3x/year.
+    // Revisit if `max-size-mb` is ever raised past ~100.
+    const { records, malformedLines } = parseEventLines(readFileSync(logPath, 'utf8'));
+    const { firstTs, lastTs } = summarizeEventRecords(records);
+    const lines = records.length + malformedLines;
 
-    // Shift `.N-1` → `.N`, …, `.1` → `.2`.
-    for (let i = maxBackups - 1; i >= 1; i--) {
-      const src = `${logPath}.${i}`;
-      const dst = `${logPath}.${i + 1}`;
-      if (existsSync(src)) {
-        renameSync(src, dst);
-      }
-    }
+    const archiveDir = path.join(path.dirname(logPath), ARCHIVE_DIR_NAME);
+    mkdirSync(archiveDir, { recursive: true });
+    const destination = uniqueArchivePath(archiveDir, firstTs, lastTs);
 
-    // Rotate active file to `.1`. POSIX atomic rename.
-    renameSync(logPath, `${logPath}.1`);
+    renameSync(logPath, destination);
+    archivedAs = destination;
+
+    const pruned = pruneArchives(archiveDir, maxBackups, archivedAs);
+
+    // The tombstone. `first_ts` / `last_ts` are `null` — present, never absent —
+    // when the archive held no parseable timestamp: a reader must be able to
+    // tell "range unknown" from "field not written by this version".
+    const record = stampEventSchemaVersion({
+      timestamp: new Date().toISOString(),
+      event: ROTATION_EVENT,
+      archived_as: archivedAs,
+      size_before: sizeBefore,
+      lines,
+      first_ts: firstTs,
+      last_ts: lastTs,
+      malformed_lines: malformedLines,
+      ...(pruned.length > 0 ? { pruned } : {}),
+    });
+    const verdict = validateEventRecord(record);
+    let recordWritten = false;
+    if (verdict.valid) {
+      appendFileSync(logPath, `${JSON.stringify(record)}\n`, 'utf8');
+      recordWritten = true;
+    }
 
     return {
       rotated: true,
-      archivedAs: `${logPath}.1`,
+      archivedAs,
       sizeBefore,
       maxBackups,
+      lines,
+      firstTs,
+      lastTs,
+      malformedLines,
+      pruned,
+      recordWritten,
+      ...(recordWritten ? {} : { error: `invalid rotation record: ${verdict.errors.join('; ')}` }),
     };
   } catch (err) {
-    // Never throw — rotation failure must not break session-start.
-    return { rotated: false, reason: 'error', error: err?.message ?? String(err) };
+    const message = err?.message ?? String(err);
+    // Never throw — rotation failure must not break session-start. But once the
+    // rename landed, the archive EXISTS and the caller must hear about it.
+    if (archivedAs !== null) {
+      return { rotated: true, archivedAs, sizeBefore, maxBackups, recordWritten: false, error: message };
+    }
+    return { rotated: false, reason: 'error', error: message };
   }
 }

package/scripts/lib/events-schema.mjs

@@ -63,6 +63,42 @@ export function stampEventSchemaVersion(record) {
 /** ISO-8601 UTC timestamp with trailing Z (e.g. 2026-05-28T14:35:13.123Z). */
 const ISO_8601_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/;
 
+// ---------------------------------------------------------------------------
+// Rotation-archive naming contract (#1401)
+// ---------------------------------------------------------------------------
+//
+// These four live HERE, in the pure module, and not beside the rotation writer
+// that produces them — deliberately, and measured. `events-rotation.mjs`
+// imports `node:fs`; when `events.mjs` (the reader) imported the constants
+// from it, `generate-hook-import-set.mjs` went from `reachable_from:
+// ["on-session-start.mjs"]` to SEVENTEEN hooks, including the per-Edit/per-Bash
+// hot paths (`enforce-scope`, `pre-task-scope-disjoint`, `post-edit-validate`).
+// A naming CONTRACT shared by a writer and a reader belongs in the zero-fs
+// module both already load; only the fs code stays behind. Same reasoning as
+// `session-lock-shape.mjs` (`.claude/rules/identity-and-locks.md`).
+
+/** Event name written as the first line of the new active file after a rotation. */
+export const ROTATION_EVENT = 'orchestrator.events.rotated';
+
+/** Directory (beside the active log) that holds rotated archives. */
+export const ARCHIVE_DIR_NAME = '_archive';
+
+/**
+ * Exact shape of a rotation archive: `events-<first>_<last>.jsonl`, each stamp
+ * `YYYYMMDDTHHMMSSZ`, with an optional `-<n>` collision suffix.
+ *
+ * Deliberately tight, because the same regex decides what the pruner may
+ * DELETE. `_archive/` is a shared human-facing directory — this repo's own copy
+ * already holds a hand-placed `events-worktree-vault-session-analysis-<date>.jsonl`
+ * — and a loose `^events-.*\.jsonl$` would both merge that file into the ledger
+ * timeline and offer it up for pruning. Rotation deletes only what it made.
+ */
+export const ARCHIVE_NAME_RE =
+  /^events-(?:\d{8}T\d{6}Z|unknown)_\d{8}T\d{6}Z(?:-\d+)?\.jsonl$/;
+
+/** Upper bound of the legacy `.1`..`.N` ring (`max-backups` is capped at 20). */
+export const LEGACY_RING_MAX = 20;
+
 /** Prefix marking an orchestrator-owned event. */
 export const ORCHESTRATOR_PREFIX = 'orchestrator.';
 
@@ -89,6 +125,84 @@ export function isIso8601(value) {
   );
 }
 
+/**
+ * Split raw JSONL text into records, COUNTING the lines that could not be read.
+ *
+ * The counting is the point (#1401). Skipping an unreadable line is right — a
+ * torn tail from a killed writer must not abort a whole window read — but
+ * skipping it SILENTLY turns a partial result into a clean verdict: a join over
+ * the ledger then reports "everything matched" in the very instrument built to
+ * surface silent failure. So the count travels with the records and every
+ * consumer is expected to carry it into its own report and telemetry (HR-105).
+ *
+ * What counts as malformed here is UNREADABLE, not invalid: a line that is not
+ * JSON, or that parses to something other than a plain object. A readable
+ * record that `validateEventRecord()` would reject (bad timestamp, illegal
+ * event name) is a DIFFERENT class and is returned in `records` — the two must
+ * not be conflated, or a schema violation would hide inside a corruption count.
+ * Blank lines are neither: a trailing newline is the normal shape of a JSONL
+ * file, so an empty line is skipped without being counted.
+ *
+ * @param {string} text — raw file contents.
+ * @returns {{records: object[], malformedLines: number}}
+ */
+export function parseEventLines(text) {
+  const records = [];
+  let malformedLines = 0;
+  for (const line of String(text ?? '').split('\n')) {
+    if (line.trim() === '') continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      malformedLines += 1;
+      continue;
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+      malformedLines += 1;
+      continue;
+    }
+    records.push(parsed);
+  }
+  return { records, malformedLines };
+}
+
+/**
+ * Earliest and latest VALID timestamp across `records` — `null` for each when
+ * no record carries a parseable one.
+ *
+ * Compared on `Date.parse`, not lexicographically: ISO-8601 UTC strings sort by
+ * string order ONLY while their millisecond part is uniform, and this stream
+ * carries both spellings (`…:13Z` and `…:13.123Z`). `Z` (0x5A) sorts after `.`
+ * (0x2E), so the millisecond-free form of the SAME second would compare as the
+ * later instant.
+ *
+ * `null` means "no parseable timestamp in this set" — an honest absence, never
+ * a fabricated epoch.
+ *
+ * @param {object[]} records
+ * @returns {{firstTs: string|null, lastTs: string|null}}
+ */
+export function summarizeEventRecords(records) {
+  let firstTs = null;
+  let lastTs = null;
+  let firstMs = Infinity;
+  let lastMs = -Infinity;
+  for (const record of records ?? []) {
+    if (!isIso8601(record?.timestamp)) continue;
+    const ms = Date.parse(record.timestamp);
+    if (ms < firstMs) {
+      firstMs = ms;
+      firstTs = record.timestamp;
+    }
+    if (ms > lastMs) {
+      lastMs = ms;
+      lastTs = record.timestamp;
+    }
+  }
+  return { firstTs, lastTs };
+}
+
 /**
  * Validate a single events.jsonl record against the canonical schema.
  *