session-orchestrator 5.1.0 → 5.3.0

package/scripts/lib/sessions-staleness-banner.mjs

@@ -30,29 +30,43 @@
  *                           newest-across-all, not genuine-first" for why the
  *                           two kinds are maxed rather than ranked.
  *   - `cutoff`            = the CURRENT session's `session.lock`
- *                           `started_at` (via `readLock()`); when no lock is
- *                           readable, `cutoff = now` (all events count).
+ *                           `started_at` (via `readLock()`), pulled back to
+ *                           that lock id's FIRST `lock.acquired` event
+ *                           (`ownGenesisMs()`, a compact re-acquire rewrites
+ *                           `started_at`); when no lock is readable,
+ *                           `cutoff = now` (all events count).
  *   - `lastForeignEventAt`= the NEWEST `events.jsonl` line whose `timestamp`
  *                           is STRICTLY BEFORE `cutoff` — this structurally
  *                           excludes the current session's own events without
  *                           needing a `session_id` filter (most event lines
  *                           don't carry one; see the mission-log sample in
- *                           `.orchestrator/metrics/events.jsonl`).
+ *                           `.orchestrator/metrics/events.jsonl`). A previous
+ *                           session's own teardown events are skipped
+ *                           (`CLOSING_DIAGNOSTIC_EVENTS`): they postdate its
+ *                           ledger record by construction.
  *   - `deltaHours`        = (lastForeignEventAt − lastLedgerAt) / 1h, only
  *                           meaningful when > 0 (foreign activity happened
  *                           AFTER the last ledger entry).
  *
- * Backfill-stub self-erasure fix — the anchor axis: a backfill-produced
- * `sessions.jsonl` record (`_backfill_source` / `status: 'abandoned'`, see
- * `session-close-backfill.mjs` `synthesizeRecord()`) sets `completed_at =
- * max(started_at, lastTerminalMs ?? nowMs)`. When the abandoned session
- * never emitted a STOPPED/ENDED event — the COMMON case, since that is
- * *why* it is "abandoned" — `completed_at` silently becomes the BACKFILL
- * RUN's own wall-clock instant, not a measurement of when the session
- * actually ended. Anchoring `lastLedgerEntry()` on that value means a
- * backfill run can retroactively erase a multi-day staleness gap just by
- * writing a stub today (observed: a 92.5h gap to the last GENUINE record
- * collapsed to 0.6h the moment a backfill stub landed).
+ * Backfill-stub self-erasure fix — the anchor axis: a backfill STUB
+ * (`status: 'abandoned'`, see `session-close-backfill.mjs` `synthesizeRecord()`)
+ * is a reconstruction, not a measured close. Before #914 R1 its `completed_at`
+ * could be the BACKFILL RUN's own wall-clock; since #914 R1 it is
+ * events-derived (terminal event, else last life-sign, else `started_at`) and
+ * flagged in `_backfill_incomplete_fields` whenever it is an estimate. A stub's
+ * `completed_at` is still refused as an anchor: an abandoned session proves
+ * the ledger alive only at its start. Anchoring on it once let a backfill run
+ * retroactively erase a multi-day staleness gap just by writing a stub today
+ * (observed: a 92.5h gap to the last GENUINE record collapsed to 0.6h).
+ *
+ * A `state-md-completed` backfill (status `completed`, #429/#1068) is NOT a
+ * stub: its status is the session's own truth claim, and its timestamps are
+ * events-derived like any other. Treating every `_backfill_source` as a stub
+ * kept the probe warning after a stub had been superseded, and recommended a
+ * backfill run with nothing left to do (live 2026-09-18). Independently of the
+ * kind, a timestamp named in a record's `_backfill_incomplete_fields` was not
+ * measured (e.g. `started_at` fell back to the backfill run's `now`) and is
+ * never an anchor candidate.
  *
  * Two axes were available to fix this: (a) skip stub records when scanning
  * for the ledger anchor, keeping `completed_at` as the anchor field; or (b)
@@ -63,10 +77,9 @@
  * (a session's own mid-session events would newly count as "after" the
  * anchor), reintroducing false positives on the opposite side. (a) is
  * chosen: a stub's `completed_at` is NEVER an anchor candidate. Stub
- * recognition uses EITHER marker (OR, not AND) deliberately — both are set by
- * the same producer today, but requiring both would silently stop matching the
- * day a future backfill variant drops one of them while keeping the other; OR
- * degrades gracefully (still catches it), AND does not.
+ * recognition: `status: 'abandoned'` alone suffices, and so does a
+ * `_backfill_source` on any record whose status is not `completed` — OR, not
+ * AND, so a future backfill variant that drops one marker is still caught.
  *
  * #1125 — newest-across-ALL records, not genuine-first: excluding the stub's
  * `completed_at` (above) is correct; excluding the whole STUB from the anchor
@@ -146,6 +159,64 @@ export const WARN_THRESHOLD_HOURS = DEFAULT_TTL_HOURS * 2;
 /** alert threshold: a full day behind is a strong close-through signal. */
 export const ALERT_THRESHOLD_HOURS = 24;
 
+/**
+ * A session's own CLOSING diagnostics — events emitted at teardown, i.e. AFTER
+ * that session's `/close` has already written its ledger record. They measure
+ * when the process exited, not when work happened, so they are never "foreign
+ * activity": counting them turned every terminal left open after `/close` into
+ * a close-through warning (live 2026-09-19: a 9.6h warn whose only post-ledger
+ * events were one SessionEnd's `wave.final_refused`, `session.ended`,
+ * `session.backfill_completed` and `telemetry.flush`, while
+ * `backfill-abandoned-sessions.mjs --dry-run` found 0 sessions to backfill).
+ * Map value = the `emitted_by` an event must carry to be
+ * skipped; `null` = skipped from every emitter.
+ *
+ * Deliberately NOT listed, and still counted: `orchestrator.turn.stopped`, its
+ * deprecated alias `orchestrator.session.stopped`, `orchestrator.agent.stopped`
+ * and every other event. A process killed by OOM or `kill -9` never runs its
+ * SessionEnd hook but has left those behind — that is the close-through gap
+ * this probe exists to catch. Skipping teardown events loses nothing there:
+ * a session that exits WITHOUT `/close` still leaves its turn/agent events,
+ * and only the idle tail between its last turn and the exit goes uncounted.
+ */
+const CLOSING_DIAGNOSTIC_EVENTS = new Map([
+  // Only emitter: hooks/on-session-end.mjs — the SessionEnd breadcrumb itself.
+  ['orchestrator.session.ended', null],
+  // Only emitter: hooks/on-session-end.mjs — the outcome of the exit-time telemetry flush.
+  ['orchestrator.telemetry.flush', null],
+  // hooks/on-session-end.mjs + scripts/backfill-abandoned-sessions.mjs — a backfill run
+  // is only ever ABOUT an already-dead session, and writes that session's ledger stub itself.
+  ['orchestrator.session.backfill_completed', null],
+  // Lock-teardown breadcrumbs. Every emitter is a session teardown: hooks/on-session-end.mjs,
+  // scripts/lib/session-transition.mjs (#1069 process boundary), and
+  // scripts/lib/autopilot/worktree-pipeline.mjs (release_failed/released only).
+  ['orchestrator.session.lock.released', null],
+  ['orchestrator.session.lock.release_failed', null],
+  // Only emitter: hooks/on-session-end.mjs — reading the lock during teardown.
+  ['orchestrator.session.lock.read_anomaly', null],
+  // Wave lifecycle: hooks/post-tool-batch-wave-signal.mjs emits wave.completed MID-session
+  // (real activity, still counted); only the SessionEnd final-wave stamp is a closing
+  // diagnostic. final_refused has on-session-end as its sole emitter today, keyed the
+  // same way so a future mid-session emitter stays counted by default.
+  ['orchestrator.wave.completed', 'on-session-end'],
+  ['orchestrator.wave.final_refused', 'on-session-end'],
+]);
+
+/**
+ * True when an events.jsonl record is one of the session's own closing
+ * diagnostics (`CLOSING_DIAGNOSTIC_EVENTS`). An unknown or missing `event`
+ * counts as activity — the failure direction stays "warn", never "silence".
+ * Caller guarantees `record` is already a non-null object.
+ *
+ * @param {object} record
+ * @returns {boolean}
+ */
+function isClosingDiagnostic(record) {
+  if (typeof record.event !== 'string' || !CLOSING_DIAGNOSTIC_EVENTS.has(record.event)) return false;
+  const requiredEmitter = CLOSING_DIAGNOSTIC_EVENTS.get(record.event);
+  return requiredEmitter === null || record.emitted_by === requiredEmitter;
+}
+
 /**
  * Read a JSONL file's non-empty lines. Returns `null` when the file is
  * absent or unreadable, `[]` when it exists but has no non-empty lines.
@@ -166,26 +237,44 @@ function readJsonlLines(filePath) {
 }
 
 /**
- * True when a `sessions.jsonl` record's `completed_at` was SYNTHESIZED by
- * the backfill engine (`scripts/lib/session-close-backfill.mjs`
- * `synthesizeRecord()`) rather than measured at real session-close time —
- * see the module-header "Backfill-stub self-erasure fix" section above for
- * the full reasoning behind the OR (not AND) combination of the two markers.
- *
- * Reuses `isRealSession()` from `./session-schema/filters.mjs` — its own doc
- * names `status: 'abandoned'` "the canonical marker" for exactly this phantom
- * class, so this is the SAME predicate every other real/phantom-aware
- * consumer in this repo already relies on, not a hand-rolled duplicate of it.
- * `_backfill_source` is layered on top as the second, independent signal.
- * Caller guarantees `record` is already a non-null object (see
- * `lastLedgerEntry()`'s guard above the call site).
+ * True when a `sessions.jsonl` record is a backfill STUB — a reconstruction
+ * whose `completed_at` is never an anchor candidate (module header,
+ * "Backfill-stub self-erasure fix").
+ *
+ * `isRealSession()` (`./session-schema/filters.mjs`) catches `status:
+ * 'abandoned'`, the canonical phantom marker. A `_backfill_source` counts only
+ * when the status is not `completed`: the `state-md-completed` supersede record
+ * is the session's own completion claim. Same exclusion as
+ * `isSupersedableStub()` in `session-close-backfill.mjs` (which keeps
+ * `completed` out of its stub statuses); not imported from there because that
+ * predicate demands BOTH markers before a record may be OVERWRITTEN, where
+ * this one accepts EITHER — a record only has to be doubtful to be
+ * disqualified as a time anchor. The two therefore disagree by design on
+ * `{_backfill_source, status: 'interrupted'}`, and the names say which
+ * question each answers.
+ * Caller guarantees `record` is already a non-null object.
  *
  * @param {object} record
  * @returns {boolean}
  */
-function isBackfillStub(record) {
+function isNonAnchorStub(record) {
   if (!isRealSession(record)) return true;
-  return typeof record._backfill_source === 'string' && record._backfill_source.length > 0;
+  return typeof record._backfill_source === 'string' && record._backfill_source.length > 0
+    && record.status !== 'completed';
+}
+
+/**
+ * Anchor candidate for one timestamp field of a record — `null` when the
+ * backfill engine listed that field in `_backfill_incomplete_fields` (it was
+ * not measured, e.g. a `started_at` that fell back to the backfill run's now).
+ *
+ * @param {object} record
+ * @param {'started_at'|'completed_at'} field
+ * @returns {{iso: string, ms: number}|null}
+ */
+function fieldCandidate(record, field) {
+  if (Array.isArray(record._backfill_incomplete_fields) && record._backfill_incomplete_fields.includes(field)) return null;
+  return tsCandidate(record[field]);
 }
 
 /**
@@ -225,15 +314,16 @@ function keepNewer(current, candidate) {
  *
  * Which timestamp each record kind contributes, and why (full reasoning in the
  * module header, "#1125 — newest-across-ALL records"):
- *   - GENUINE (`!isBackfillStub()`) → `completed_at`, written by the real
+ *   - GENUINE (`!isNonAnchorStub()`) → `completed_at`, written by the real
  *     session-end path; falling back to `started_at` when `completed_at` is
  *     absent or unparseable, so a truncated/in-flight record still contributes
  *     the coverage it does prove. Taking the newer of the two can only move the
  *     anchor forward, never backward, so it does not inflate `deltaHours`.
- *   - BACKFILL STUB → `started_at` ONLY. Its `completed_at` may be the backfill
- *     RUN's own wall-clock rather than a measurement of when the session ended
- *     (see `synthesizeRecord()`), and anchoring on that would let a backfill run
- *     retroactively erase a real multi-day gap.
+ *   - BACKFILL STUB → `started_at` ONLY. An abandoned session proves the
+ *     ledger alive at its start, not at a reconstructed end; anchoring on the
+ *     stub's `completed_at` would let a backfill run erase a real gap.
+ *   - Either kind: a field listed in `_backfill_incomplete_fields` was not
+ *     measured and contributes nothing (`fieldCandidate()`).
  *
  * The two kinds are MAXED, not ranked: an older genuine record must not outrank
  * a newer stub (that priority order was the #1125 false-alert defect). When the
@@ -258,13 +348,13 @@ function lastLedgerEntry(records) {
   for (const record of records) {
     if (!record || typeof record !== 'object') continue;
 
-    if (isBackfillStub(record)) {
-      newestStub = keepNewer(newestStub, tsCandidate(record.started_at));
+    if (isNonAnchorStub(record)) {
+      newestStub = keepNewer(newestStub, fieldCandidate(record, 'started_at'));
       continue;
     }
 
-    let own = tsCandidate(record.completed_at);
-    const started = tsCandidate(record.started_at);
+    let own = fieldCandidate(record, 'completed_at');
+    const started = fieldCandidate(record, 'started_at');
     // `completed_at` is the normal anchor and wins ties; `started_at` only
     // takes over when it is genuinely newer or `completed_at` is unusable.
     if (started !== null && (own === null || started.ms > own.ms)) own = started;
@@ -284,7 +374,23 @@ function lastLedgerEntry(records) {
  * current session's own events (all >= cutoff, since cutoff is this
  * session's lock `started_at`) are structurally excluded without needing a
  * `session_id` filter. Malformed lines and lines with a missing/invalid
- * `timestamp` are skipped.
+ * `timestamp` are skipped, and so is a previous session's own teardown output
+ * (`isClosingDiagnostic()`), which postdates that session's ledger record by
+ * construction.
+ *
+ * Known ceiling (BV-004) — post-close TURNS still count: a session that keeps
+ * talking after its `/close` wrote the ledger record emits
+ * `orchestrator.turn.stopped` (and the deprecated `orchestrator.session.stopped`)
+ * at the end of every further turn, so a post-close conversation spanning more
+ * than `WARN_THRESHOLD_HOURS` warns although that session IS recorded. Kept on
+ * purpose: those same events are the only trace an OOM-/`kill -9`-ended session
+ * leaves. Size (W1 census 2026-09-19 over this repo's events.jsonl(.1), not
+ * re-measured here): 4 of 10 resolvable firings were this class. Revisit trigger: the HR-105 firing-rate
+ * audit shows this class alone above ~10% of sessions that ran the probe
+ * (HR-101) — then skip a stop event whose `semantic_session_id` matches a
+ * GENUINE ledger record (that session is recorded, so its post-close turns are
+ * not a close-through gap), which needs `newestForeignEvent` to receive the
+ * canonical records.
  *
  * @param {string[]} lines
  * @param {number} cutoffMs
@@ -303,6 +409,7 @@ function newestForeignEvent(lines, cutoffMs) {
     const ms = Date.parse(record.timestamp);
     if (!Number.isFinite(ms)) continue;
     if (ms >= cutoffMs) continue; // not "foreign" — belongs to (or postdates) the current session
+    if (isClosingDiagnostic(record)) continue; // a previous session's teardown, not activity
     if (best === null || ms > best.ms) best = { iso: record.timestamp, ms };
   }
   return best;
@@ -325,18 +432,53 @@ function newestForeignEvent(lines, cutoffMs) {
  *
  * @param {string} repoRoot
  * @param {number} nowMs
- * @returns {number}
+ * @returns {{cutoffMs: number, ownSessionId: string|null}}
  */
-function resolveCutoffMs(repoRoot, nowMs) {
+function resolveCutoff(repoRoot, nowMs) {
   let lock;
   try {
     lock = readLock({ repoRoot });
   } catch {
-    return nowMs;
+    return { cutoffMs: nowMs, ownSessionId: null };
+  }
+  if (!lock) return { cutoffMs: nowMs, ownSessionId: null };
+  const ownSessionId = typeof lock.session_id === 'string' && lock.session_id !== '' ? lock.session_id : null;
+  const parsed = typeof lock.started_at === 'string' ? Date.parse(lock.started_at) : NaN;
+  return { cutoffMs: Number.isFinite(parsed) ? parsed : nowMs, ownSessionId };
+}
+
+/**
+ * Pull the cutoff back to the FIRST `orchestrator.session.lock.acquired` event
+ * carrying the current lock's raw `session_id`. A re-acquire under the same id
+ * (the SessionStart hook on a context compact) rewrites `started_at`, which
+ * moved the cutoff past the session's own earlier events — and not all of them
+ * carry a `session_id` to filter on (`secret_masker.applied` does not).
+ * Measured 2026-09-19: a 2h15m session warned "11.9h behind" about itself on
+ * compact. Raw id only, never the semantic label (two sessions can mint the
+ * same label, #1066). Linear scan over events.jsonl — the same pass
+ * `newestForeignEvent()` already makes.
+ *
+ * @param {string[]} lines
+ * @param {string|null} ownSessionId
+ * @param {number} cutoffMs
+ * @returns {number}
+ */
+function ownGenesisMs(lines, ownSessionId, cutoffMs) {
+  if (ownSessionId === null) return cutoffMs;
+  let genesis = cutoffMs;
+  for (const line of lines) {
+    if (!line.includes(ownSessionId)) continue;
+    let record;
+    try {
+      record = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (record?.event !== 'orchestrator.session.lock.acquired' || record.session_id !== ownSessionId) continue;
+    const ms = Date.parse(record.timestamp);
+    if (Number.isFinite(ms) && ms < genesis) genesis = ms;
   }
-  if (!lock || typeof lock.started_at !== 'string') return nowMs;
-  const parsed = Date.parse(lock.started_at);
-  return Number.isFinite(parsed) ? parsed : nowMs;
+  return genesis;
 }
 
 /**
@@ -386,9 +528,9 @@ export function checkSessionsStaleness({ repoRoot, now = Date.now() } = {}) {
     const eventLines = readJsonlLines(path.join(repoRoot, EVENTS_PATH));
     if (eventLines === null || eventLines.length === 0) return null;
 
-    const cutoffMs = resolveCutoffMs(repoRoot, nowMs);
+    const { cutoffMs, ownSessionId } = resolveCutoff(repoRoot, nowMs);
 
-    const foreign = newestForeignEvent(eventLines, cutoffMs);
+    const foreign = newestForeignEvent(eventLines, ownGenesisMs(eventLines, ownSessionId, cutoffMs));
     if (foreign === null) return null;
 
     const deltaMs = foreign.ms - ledger.ms;