session-orchestrator 5.3.0 → 5.4.0

package/scripts/lib/session-record-repair.mjs

@@ -36,11 +36,17 @@
  *    no key-order churn.
  *
  * ── PROVENANCE ───────────────────────────────────────────────────────────────
- * Every repaired record carries `_backfill_source` + `_backfill_incomplete_fields`
- * (both already schema-accepted by `_validateOptionalFields`, so no schema bump
- * is needed). The incomplete-fields list is the EXACT set of fields this record
- * had defaulted, which is what lets a downstream consumer tell a measured zero
- * apart from a repaired-to-zero.
+ * Every repaired record carries `_repair_source` (`REPAIR_SOURCE`) +
+ * `_backfill_incomplete_fields`; `_backfill_source` is set to `REPAIR_SOURCE`
+ * only when the record had no origin of its own. An existing origin
+ * (`state-md-completed`, `events-jsonl`, `abandoned`) is KEPT, because
+ * `sessions-canonical.mjs`'s stub guard reads it: overwriting it promoted a
+ * repaired zero-work stub to a real record that shadowed the real one (#1443
+ * F2). `_backfill_*` is schema-accepted by `_validateOptionalFields`, and
+ * `validateSession` ignores undeclared keys (as it does the `_<field>_raw`
+ * sidecars), so no schema bump is needed. The incomplete-fields list is the
+ * EXACT set of fields this record had defaulted, which is what lets a
+ * downstream consumer tell a measured zero apart from a repaired-to-zero.
  *
  * It also never DISCARDS what it replaces: any field whose present value is
  * overwritten by a default is first copied to a `_<field>_raw` sidecar
@@ -70,7 +76,10 @@ import { checkSessionsIntegrity as defaultCheckIntegrity } from './sessions-inte
 // Constants
 // ---------------------------------------------------------------------------
 
-/** Value written to `_backfill_source` on every record this module repairs. */
+/**
+ * Value written to `_repair_source` on every record this module repairs, and to
+ * `_backfill_source` only on a record that had no origin there (#1443 F2).
+ */
 export const REPAIR_SOURCE = 'repair-invalid-sessions/1004';
 
 /** Ledger path (relative to repoRoot) that `checkSessionsIntegrity` inspects. */
@@ -98,6 +107,28 @@ const INCOMPLETE_FIELD_ORDER = Object.freeze([
   'total_agents',
   'total_files_changed',
 ]);
+// `timestamp_precision` (#1442) deliberately adds NOTHING to the list above: a
+// truncated timestamp is still a measurement (exact to the millisecond), not a
+// default. Listing it would make consumers discard a real value —
+// `sessions-staleness-banner.mjs` returns null for any timestamp named in
+// `_backfill_incomplete_fields`, `session-close-backfill.mjs` keys on
+// `includes('started_at')`. The sub-millisecond original survives in the
+// `_<field>_raw` sidecar instead.
+
+/**
+ * Timestamp fields `validateSession` holds to `ISO_8601_UTC_MS_RE`
+ * (`session-schema/validator.mjs`, not exported): exactly three fractional
+ * digits or none.
+ */
+const MS_PRECISION_FIELDS = Object.freeze(['started_at', 'completed_at', 'lease_acquired_at']);
+
+/**
+ * ISO-8601 UTC with MORE than three fractional digits (`.994600Z`) — the one
+ * rejected shape that has an unambiguous canonical form. Group 1 is the value
+ * cut at the millisecond. Offsets, `.3Z` and non-ISO strings do not match and
+ * stay for the validator to reject: there is no defensible default for them.
+ */
+const ISO_UTC_SUB_MS_RE = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3})\d+Z$/;
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -112,6 +143,20 @@ function isCount(v) {
   return typeof v === 'number' && Number.isFinite(v) && v >= 0;
 }
 
+/**
+ * Cut a sub-millisecond ISO-8601 UTC timestamp to milliseconds. TRUNCATES,
+ * never rounds: `.9999Z` must not roll into the next second, and truncation is
+ * monotone, so `completed_at >= started_at` survives on every pair it held on.
+ *
+ * @param {unknown} value
+ * @returns {string|null} the truncated value, or null when there is nothing to cut
+ */
+function truncateToMs(value) {
+  if (typeof value !== 'string') return null;
+  const m = ISO_UTC_SUB_MS_RE.exec(value);
+  return m ? `${m[1]}Z` : null;
+}
+
 function orderIncompleteFields(fields) {
   const rank = (f) => {
     const i = INCOMPLETE_FIELD_ORDER.indexOf(f);
@@ -321,6 +366,19 @@ export function repairRecord(record) {
     incomplete.add('total_files_changed');
   }
 
+  // -- timestamp precision (#1442) -------------------------------------------
+  // Runs BEFORE the completed_at default below, because that default copies
+  // `started_at` — a microsecond `started_at` would otherwise be copied into
+  // `completed_at` and spread the defect instead of fixing it.
+  let precisionCut = false;
+  for (const key of MS_PRECISION_FIELDS) {
+    const truncated = truncateToMs(out[key]);
+    if (truncated === null) continue;
+    preserveRaw(out, key, record[key], rescued);
+    out[key] = truncated;
+    precisionCut = true;
+  }
+
   // -- completed_at ---------------------------------------------------------
   // Legacy `ended_at` is the same fact under its old name (emit-session.mjs
   // aliasLegacyEndedAt precedent) — prefer it when parseable and monotonic,
@@ -332,13 +390,19 @@ export function repairRecord(record) {
     preserveRaw(out, 'completed_at', record.completed_at, rescued);
     const endedMs = typeof out.ended_at === 'string' ? Date.parse(out.ended_at) : NaN;
     const startedMs = Date.parse(out.started_at);
-    out.completed_at =
-      Number.isFinite(endedMs) && Number.isFinite(startedMs) && endedMs >= startedMs
-        ? out.ended_at
-        : out.started_at;
+    if (Number.isFinite(endedMs) && Number.isFinite(startedMs) && endedMs >= startedMs) {
+      // The COPY is cut to milliseconds; `ended_at` itself is not a validated
+      // field and keeps its original bytes, so nothing needs a sidecar here.
+      const endedTruncated = truncateToMs(out.ended_at);
+      if (endedTruncated !== null) precisionCut = true;
+      out.completed_at = endedTruncated ?? out.ended_at;
+    } else {
+      out.completed_at = out.started_at;
+    }
     defects.push('completed_at_missing');
     incomplete.add('completed_at');
   }
+  if (precisionCut) defects.push('timestamp_precision');
 
   if (defects.length === 0) {
     return { record, defects: [], incompleteFields: [], changed: false };
@@ -349,7 +413,14 @@ export function repairRecord(record) {
     ? record._backfill_incomplete_fields.filter((f) => typeof f === 'string')
     : [];
   const merged = orderIncompleteFields(incomplete);
-  out._backfill_source = REPAIR_SOURCE;
+  // The repair is recorded in its own field; `_backfill_source` keeps the
+  // record's ORIGIN. Overwriting a synthesised origin ('state-md-completed',
+  // 'events-jsonl', 'abandoned') made a repaired zero-work stub read as a real
+  // record, and it shadowed the real one for its id (#1443 F2).
+  out._repair_source = REPAIR_SOURCE;
+  if (typeof record._backfill_source !== 'string' || record._backfill_source.length === 0) {
+    out._backfill_source = REPAIR_SOURCE;
+  }
   out._backfill_incomplete_fields = [...new Set([...prior, ...merged])];
 
   return { record: out, defects, incompleteFields: merged, changed: true };
@@ -487,6 +558,11 @@ export function repairText(raw, deps = {}) {
       repaired,
       // Projected: every line that was invalid and did NOT get repaired.
       invalid_after: invalidBefore - repaired,
+      // Lines with no defensible repair (`repairLine` status `error`). They are
+      // emitted byte-identically and are EXPECTED to stay invalid after a
+      // write — `verifyWritten` tolerates exactly these line numbers (#1442).
+      // Same population as `errors`, which carries the per-line detail.
+      unrepairable: errors.length,
       duplicate_ids_observed: duplicates,
       defects_by_class: defectsByClass,
       errors,
@@ -507,9 +583,32 @@ export function repairText(raw, deps = {}) {
  * any other target it is reported as skipped rather than quietly measuring the
  * wrong file.
  *
- * @returns {{ok: boolean, invalid_after: number, invalid_lines: object[], integrity: string|object}}
+ * ── WHAT "OK" MEANS (#1442) ──────────────────────────────────────────────────
+ * Not "zero invalid lines". A ledger that carries lines this module has no
+ * defensible repair for could never pass that bar, so one unrepairable record
+ * rolled back every repair beside it (all-or-nothing). `ok` now means NO
+ * WORSENING and ALL REPAIRABLE VALID:
+ *   - every invalid line in the written file is one of `expected.unrepairableLines`
+ *     (so every repaired AND every already-valid line validates), and
+ *   - `invalid_after <= expected.invalidBefore`, and
+ *   - the integrity probe flags no line it did not already flag in
+ *     `expected.integrityBefore` (compared by line number — the repair never
+ *     adds or removes lines, so positions are stable across the write).
+ * With no `expected`, the defaults reproduce the old strict bar: no invalid
+ * line tolerated, and a clean pre-write integrity baseline.
+ *
+ * NAMED CEILING (BV-004): integrity is compared per line, not per reason — a
+ * line flagged before AND after for a DIFFERENT reason is not "worse". The
+ * per-line `validateSession` pass above still catches it for the schema half.
+ *
+ * @param {object} args
+ * @param {object} [args.expected]
+ * @param {number[]} [args.expected.unrepairableLines] 1-based lines allowed to stay invalid
+ * @param {number}   [args.expected.invalidBefore]      invalid count before the write
+ * @param {object|null} [args.expected.integrityBefore] pre-write `checkIntegrity` banner (null = clean)
+ * @returns {{ok: boolean, invalid_after: number, invalid_lines: object[], unexpected_invalid: object[], integrity: string|object, integrity_worsened: boolean}}
  */
-export function verifyWritten({ file, repoRoot, deps = {} }) {
+export function verifyWritten({ file, repoRoot, deps = {}, expected = {} }) {
   const {
     readFileSync = fs.readFileSync,
     validateSession = defaultValidateSession,
@@ -539,17 +638,50 @@ export function verifyWritten({ file, repoRoot, deps = {} }) {
     }
   }
 
+  const { unrepairableLines = [], invalidBefore = 0, integrityBefore = null } = expected;
+  const tolerated = new Set(unrepairableLines);
+  const unexpectedInvalid = invalidLines.filter((l) => !tolerated.has(l.line));
+
   let integrity = 'skipped-not-canonical-path';
-  if (typeof repoRoot === 'string' && repoRoot.length > 0) {
-    const canonical = path.join(repoRoot, CANONICAL_LEDGER_REL);
-    if (path.resolve(file) === path.resolve(canonical)) {
-      const banner = checkIntegrity({ repoRoot });
-      integrity = banner === null ? 'clean' : banner;
-    }
+  let worsened = false;
+  if (isCanonicalLedger(file, repoRoot)) {
+    const banner = checkIntegrity({ repoRoot });
+    integrity = banner === null ? 'clean' : banner;
+    worsened = integrityWorsened(integrityBefore, banner);
   }
 
-  const ok = invalidLines.length === 0 && (integrity === 'clean' || integrity === 'skipped-not-canonical-path');
-  return { ok, invalid_after: invalidLines.length, invalid_lines: invalidLines, integrity };
+  const ok = unexpectedInvalid.length === 0 && invalidLines.length <= invalidBefore && !worsened;
+  return {
+    ok,
+    invalid_after: invalidLines.length,
+    invalid_lines: invalidLines,
+    unexpected_invalid: unexpectedInvalid,
+    integrity,
+    integrity_worsened: worsened,
+  };
+}
+
+/** True when `file` IS `<repoRoot>/.orchestrator/metrics/sessions.jsonl` — the only file `checkIntegrity` reads. */
+function isCanonicalLedger(file, repoRoot) {
+  if (typeof repoRoot !== 'string' || repoRoot.length === 0) return false;
+  return path.resolve(file) === path.resolve(path.join(repoRoot, CANONICAL_LEDGER_REL));
+}
+
+/**
+ * Did the integrity probe flag a line after the write that it did not flag
+ * before? Fail-safe on an unreadable banner shape: a non-null banner without
+ * its `schemaInvalid` / `mirrorSkipped` arrays counts as worse.
+ */
+function integrityWorsened(before, after) {
+  if (after === null || after === undefined) return false;
+  for (const key of ['schemaInvalid', 'mirrorSkipped']) {
+    const now = after && Array.isArray(after[key]) ? after[key] : null;
+    if (now === null) return true;
+    const prior = before && Array.isArray(before[key]) ? before[key] : [];
+    const priorLines = new Set(prior.map((e) => e?.line));
+    if (now.some((e) => !priorLines.has(e?.line))) return true;
+  }
+  return false;
 }
 
 /**
@@ -560,7 +692,10 @@ export function verifyWritten({ file, repoRoot, deps = {} }) {
  * `<file>.tmp-<pid>` in the same directory and renames it over the target
  * (atomic within a filesystem), then re-verifies the written file. A failed
  * verification restores the backup byte-identically and reports `ok: false` —
- * the caller maps that to exit 3.
+ * the caller maps that to exit 3. Lines with no defensible repair are kept
+ * byte-identical and counted in `summary.unrepairable`; they are NOT a
+ * verification failure (see `verifyWritten` § WHAT "OK" MEANS) — the caller
+ * maps an ok run with an unrepairable remainder to exit 4.
  *
  * Never throws for a defective RECORD (those land in `summary.errors`); a
  * genuine I/O failure DOES throw and is the caller's exit-2 case.
@@ -584,6 +719,8 @@ export function repairLedger({ file, repoRoot = null, apply = false, backup = tr
     existsSync = fs.existsSync,
   } = deps;
 
+  const { checkIntegrity = defaultCheckIntegrity } = deps;
+
   const raw = readFileSync(file, 'utf8');
   const { text, summary: base } = repairText(raw, deps);
 
@@ -594,10 +731,17 @@ export function repairLedger({ file, repoRoot = null, apply = false, backup = tr
     backup_path: null,
     post_verify: null,
     ok: true,
+    restored: false,
   };
 
   if (!apply) return summary;
 
+  // -- integrity baseline (#1442) --------------------------------------------
+  // Measured on the ORIGINAL bytes, before anything is written, so the
+  // post-write probe can be judged "not worse" instead of "perfectly clean" —
+  // unrepairable lines keep the banner non-null on both sides by construction.
+  const integrityBefore = isCanonicalLedger(file, repoRoot) ? checkIntegrity({ repoRoot }) : null;
+
   // -- backup FIRST (unconditional unless explicitly opted out) --------------
   let backupPath = null;
   if (backup) {
@@ -621,14 +765,27 @@ export function repairLedger({ file, repoRoot = null, apply = false, backup = tr
   }
 
   // -- post-verification ----------------------------------------------------
-  const verdict = verifyWritten({ file, repoRoot, deps });
+  const verdict = verifyWritten({
+    file,
+    repoRoot,
+    deps,
+    expected: {
+      unrepairableLines: base.errors.map((e) => e.line),
+      invalidBefore: base.invalid_before,
+      integrityBefore,
+    },
+  });
   summary.post_verify = verdict;
   summary.invalid_after = verdict.invalid_after;
 
   if (!verdict.ok) {
-    // Restore byte-identically. The backup is preferred (it is the on-disk
-    // artefact an operator can inspect); `raw` is the in-memory fallback for
-    // `--no-backup`, and both are the same bytes.
+    // Restore byte-identically — ONLY for a repaired (or previously valid) line
+    // that fails verification, or an integrity probe that got worse. An
+    // unrepairable remainder alone is not a failure (#1442): those lines were
+    // emitted byte-identically and keep the ledger exactly as broken as it was.
+    // The backup is preferred (it is the on-disk artefact an operator can
+    // inspect); `raw` is the in-memory fallback for `--no-backup`, and both are
+    // the same bytes.
     if (backupPath) {
       copyFileSync(backupPath, file);
     } else {

package/scripts/lib/session-registry.mjs

@@ -17,7 +17,9 @@
  *   {
  *     session_id, pid, platform, repo_path_hash, repo_name, branch,
  *     started_at, last_heartbeat, status ('active'|'wave'|'idle'),
- *     current_wave, host_class, mode (Epic #583 W2-I3)
+ *     current_wave, host_class, mode (Epic #583 W2-I3),
+ *     role ('navigator'|null, optional, #1462 — display only, never ownership;
+ *           set by heartbeat(patch.role), NOT by registerSelf)
  *   }
  *
  * Schema v2 (Epic #583, W2-I3): `mode` field added so the exclusivity-matrix
@@ -142,6 +144,14 @@ function _validEntry(obj) {
   if ('mode' in obj && obj.mode !== null && typeof obj.mode !== 'string') {
     return false;
   }
+  // #1462: `role` is optional and a DISPLAY field only — the registry shows who
+  // claims to be navigator, it never decides it. The authoritative claim is the
+  // navigator lease under `~/.config/navigator/leases/navigator.json`, and
+  // classifyMode ignores `role`. When present it MUST be null or a string (no
+  // number / object smuggling); absent is accepted like a v1 `mode`.
+  if ('role' in obj && obj.role !== null && typeof obj.role !== 'string') {
+    return false;
+  }
   return true;
 }
 
@@ -236,11 +246,25 @@ export async function registerSelf(opts) {
   return entry;
 }
 
+/** Roles `heartbeat(id, { role })` accepts besides `null` (#1462). */
+export const REGISTRY_ROLES = Object.freeze(['navigator']);
+
 /**
  * Refresh last_heartbeat on the existing entry (and optionally status /
- * current_wave). No-op if the entry is missing.
+ * current_wave / role). No-op if the entry is missing.
+ *
+ * `patch.role`: `undefined` keeps the stored role, `null` clears it, a member
+ * of `REGISTRY_ROLES` sets it. Anything else throws before the file is touched.
+ *
+ * @param {string} sessionId
+ * @param {{ status?: string, currentWave?: number, role?: string|null }} [patch]
+ * @returns {Promise<object|null>} the updated entry, or null if none exists.
+ * @throws {TypeError} when `patch.role` is neither undefined, null, nor a REGISTRY_ROLES member.
  */
 export async function heartbeat(sessionId, patch = {}) {
+  if (patch.role !== undefined && patch.role !== null && !REGISTRY_ROLES.includes(patch.role)) {
+    throw new TypeError('heartbeat: role must be "navigator" or null');
+  }
   const file = entryPath(sessionId);
   const existing = await _readJsonSafe(file);
   if (!_validEntry(existing)) return null;
@@ -249,6 +273,7 @@ export async function heartbeat(sessionId, patch = {}) {
     last_heartbeat: utcTimestamp(),
     ...(patch.status !== undefined ? { status: patch.status } : {}),
     ...(patch.currentWave !== undefined ? { current_wave: patch.currentWave } : {}),
+    ...(patch.role !== undefined ? { role: patch.role } : {}),
   };
   await _writeJsonAtomic(file, updated);
   return updated;

package/scripts/lib/session-schema/constants.mjs

@@ -219,11 +219,37 @@ export const OPTIONAL_FIELDS = Object.freeze([
   // #1339 P8 — HEAD sha at session start. Gives `/evolve analyze` an
   // attributable commit range instead of a time window that also catches
   // parallel sessions' commits. Absent = not measured. NOT guaranteed to be a
-  // full sha: scripts/emit-session.mjs adopts STATE.md `session-start-ref`
-  // only when it is a full 40/64-hex sha, but an explicit key already on the
-  // record wins unchecked (even `null`), and every record before that
-  // derivation was hand-composed — 19 with a 40-char and 9 with a 7-char
+  // full sha: scripts/emit-session.mjs now writes only a full 40/64-hex sha or
+  // an explicit `null` (a derived ref is adopted only in full form, and an
+  // explicit short ref on the record is dropped with a WARN), but every record
+  // before that derivation was hand-composed — 19 with a 40-char and 9 with a 7-char
   // value (measured 2026-09-19 @ 8f6ac022, jq over sessions.jsonl).
   // Consumers must accept a short sha or `null`, never assume a full one.
   'session_start_ref',
+  // #1457 point 7 — four fields that were written and (two of them) shape-
+  // checked while appearing in NEITHER list, the same "status only inferable
+  // from an `if`" class as `effectiveness` above (#964). Each one has a live
+  // writer:
+  //   `total_tokens`, `matched_records` — merged from the #1436 subagents.jsonl
+  //     rollup by scripts/emit-session.mjs (ROLLUP_KEYS) and produced by
+  //     scripts/lib/session-token-rollup.mjs; validator.mjs checks both as
+  //     non-negative integers.
+  //   `raw_session_id` — the native harness uuid beside the semantic
+  //     `session_id`, written by scripts/emit-session.mjs and by the SessionEnd
+  //     backfill in scripts/lib/session-close-backfill.mjs.
+  //   `_repair_source` — provenance marker every record repaired by
+  //     scripts/lib/session-record-repair.mjs carries (#1443 F2), distinct from
+  //     `_backfill_source`, which keeps the record's origin.
+  'total_tokens',
+  'matched_records',
+  'raw_session_id',
+  '_repair_source',
+  // #1475 — how many of the session's cost-relevant subagent records were
+  // priced, out of how many. Merged from the rollup by scripts/emit-session.mjs;
+  // non-negative integers, priced ≤ total. `total_cost_usd` is omitted when
+  // priced < total (some agent's cost is unknown) AND when total is 0 (nothing
+  // to price — 1223 of 2093 ledger×session pairs, 31 fleet ledgers,
+  // 2026-10-01), so the counters say which of the two it is.
+  'cost_records_priced',
+  'cost_records_total',
 ]);

package/scripts/lib/session-schema/validator.mjs

@@ -386,7 +386,9 @@ function _validateOptionalFields(entry) {
   }
   // #1244 — cache buckets, cost estimate and the token-contract marker.
   // Same non-negative-finite-or-null contract as the two totals above;
-  // total_cost_usd is fractional and null means "unknown model", never 0.
+  // total_cost_usd is fractional; null means "cost unknown" — at least one
+  // subagent record could not be priced (#1475: unknown model, no model, or no
+  // tokens; see cost_records_* below) — never 0.
   for (const field of [
     'total_token_input_uncached',
     'total_token_cache_read',
@@ -417,6 +419,31 @@ function _validateOptionalFields(entry) {
     }
   }
 
+  // #1436 — the session-level token total and the rollup's record count, merged
+  // by emit-session from subagents.jsonl. Same contract as subagents_with_tokens:
+  // a non-negative integer, null/absent = not measured. Before this, neither was
+  // checked and `total_tokens: "abc"` validated — the field the autopilot token
+  // budget reads. #1475 adds the rollup's cost-coverage counters under the same
+  // contract, plus the one relation between them: priced never exceeds total.
+  for (const field of ['total_tokens', 'matched_records', 'cost_records_priced', 'cost_records_total']) {
+    const value = entry[field];
+    if (value !== undefined && value !== null) {
+      if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
+        throw new ValidationError(`${field} must be a non-negative integer, got: ${value}`);
+      }
+    }
+  }
+  if (
+    Number.isInteger(entry.cost_records_priced) &&
+    Number.isInteger(entry.cost_records_total) &&
+    entry.cost_records_priced > entry.cost_records_total
+  ) {
+    throw new ValidationError(
+      `cost_records_priced must not exceed cost_records_total, got: ` +
+        `${entry.cost_records_priced} > ${entry.cost_records_total}`
+    );
+  }
+
   // Epic #724 C1 — SessionEnd close-through backfill provenance fields.
   // All additive-optional; null/absent passes without further checks so that
   // pre-#724 records (and every normally-closed record) validate cleanly.