session-orchestrator 3.22.0 → 3.24.0

package/scripts/lib/dispatcher/enumerate.mjs

@@ -29,12 +29,12 @@
  */
 
 import { readdirSync, existsSync } from 'node:fs';
-import os from 'node:os';
 import path from 'node:path';
 
 import { getConfinementRoot, getCrossRepoProjects } from '../config/cross-repo.mjs';
 import { validatePathInsideProject } from '../path-utils.mjs';
 import { readLock, isLockLive } from '../session-lock.mjs';
+import { expandTilde } from '../common.mjs';
 
 /**
  * @typedef {Object} Candidate
@@ -97,21 +97,6 @@ function shouldDescendInto(name) {
   return true;
 }
 
-/**
- * Expand a leading `~` to the current user's home directory. Mirrors the helper
- * in board-writer.mjs (a shared extraction is deferred to a later epic). Used to
- * normalise config-declared cross-repo paths that may begin with `~/`.
- *
- * @param {string} p
- * @returns {string}
- */
-function expandHome(p) {
-  if (typeof p !== 'string' || p.length === 0) return p;
-  if (p === '~') return os.homedir();
-  if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2));
-  return p;
-}
-
 /**
  * Derive free/busy status for a single repo from its session.lock lease.
  * Mirrors board-writer.mjs collectRows semantics exactly:
@@ -322,7 +307,7 @@ export async function enumerateCandidates({ startDir, now, deps } = {}) {
   if (Array.isArray(declared)) {
     for (const raw of declared) {
       if (typeof raw !== 'string' || raw.length === 0) continue;
-      const expanded = expandHome(raw);
+      const expanded = expandTilde(raw);
       const abs = path.resolve(expanded);
       // Confinement-filter against the same root as the FS scan.
       const guard = validatePathInsideProjectFn(abs, root);

package/scripts/lib/events-schema.mjs

@@ -12,6 +12,54 @@
  * Pure functions, no filesystem access — safe to import anywhere.
  */
 
+/**
+ * Current events.jsonl record schema version (#1177). Mirrors
+ * `subagents-schema.mjs § CURRENT_SCHEMA_VERSION`.
+ *
+ * Records written before #1177 carry NO `schema_version` key at all; an absent
+ * key therefore reads as "pre-versioned", never as version 0. Stamping is
+ * strictly additive — see `stampEventSchemaVersion()`.
+ */
+export const CURRENT_SCHEMA_VERSION = 1;
+
+/**
+ * Thrown by `emitEvent()` when a record fails `validateEventRecord()`.
+ *
+ * Carries the individual validator messages so a CLI/hook caller can surface
+ * them without re-running the validator.
+ */
+export class EventValidationError extends Error {
+  /**
+   * @param {string} message — human-readable summary.
+   * @param {string[]} [errors=[]] — the validator's individual error strings.
+   * @param {string} [eventType] — the offending event name, when known.
+   */
+  constructor(message, errors = [], eventType) {
+    super(message);
+    this.name = 'EventValidationError';
+    this.errors = errors;
+    this.eventType = eventType;
+  }
+}
+
+/**
+ * Return a shallow copy of `record` with `schema_version` stamped to
+ * `CURRENT_SCHEMA_VERSION` — but ONLY when the field is absent
+ * (`undefined`/`null`). An existing value is never overwritten, so a caller
+ * (or a migration re-writing historical records) keeps authority over its own
+ * version field.
+ *
+ * @param {object} record
+ * @returns {object} shallow copy, `schema_version` guaranteed present.
+ */
+export function stampEventSchemaVersion(record) {
+  const out = { ...record };
+  if (out.schema_version === undefined || out.schema_version === null) {
+    out.schema_version = CURRENT_SCHEMA_VERSION;
+  }
+  return out;
+}
+
 /** ISO-8601 UTC timestamp with trailing Z (e.g. 2026-05-28T14:35:13.123Z). */
 const ISO_8601_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/;
 

package/scripts/lib/events.mjs

@@ -7,12 +7,52 @@
  * Part of v3.0.0 migration (Epic #124, issue #133).
  * Issue #228: removed hardcoded personal-domain default URL. Clank Event Bus URL
  * must now be supplied explicitly via CLANK_EVENT_URL when CLANK_EVENT_SECRET is set.
+ *
+ * ## Correlation envelope (#1177 FA3)
+ *
+ * Measured 2026-09-02 @ c3ab480 over 33,608 ledger records: only 22.1% carry
+ * `session_id` and 4.8% carry `wave`, because filling them was every call
+ * site's own job and 32 of 34 call sites pass no options at all. `emitEvent()`
+ * now fills those keys itself — under three hard rules:
+ *
+ *   1. **Additive, never overriding.** The correlation keys are spread BEFORE
+ *      `payload`, so any caller-supplied `session_id` / `semantic_session_id` /
+ *      `wave` wins byte-for-byte. A payload that supplies EITHER session key
+ *      suppresses the session fill entirely (both keys), so a caller that
+ *      deliberately pins attribution elsewhere — `vault-mirror/telemetry.mjs`
+ *      pins to `SO_PROJECT_DIR` and passes both keys — is untouched.
+ *   2. **Omit, never fabricate.** When attribution cannot be PROVEN, both keys
+ *      are left ABSENT — never `null`, never `''`. An absent key is the only
+ *      honest encoding of "not attributable" (see `sessionAttribution()`).
+ *   3. **Never a peer's id (#1123).** A shared working copy means
+ *      `session.lock` can name a PEER session that won the acquire race. The
+ *      lock alone therefore does not prove ownership; the fill happens only
+ *      when a PROCESS-LOCAL id (`CLAUDE_CODE_SESSION_ID`, or a hook payload's
+ *      `session_id`) equals the lock's raw `session_id`. STATE.md is NOT a
+ *      witness here (#1177 FX1): it is a shared working-copy file written by
+ *      the lock holder, so under a peer-owned lock both agreed about the peer
+ *      and the union stamped the peer's ids. See {@link attributionForRecord}.
+ *
+ * The attribution root is the SAME root the ledger line is pinned to
+ * (`opts.repoRoot ?? SO_PROJECT_DIR`), never `process.cwd()`. Measured cost of
+ * the whole envelope (lock + wave manifest, 100 calls, this repo):
+ * 0.0961 ms/call.
  */
 
-import { promises as fs } from 'node:fs';
+import { promises as fs, existsSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 import { SO_PROJECT_DIR, SO_SHARED_DIR } from './platform.mjs';
 import { readLock } from './session-lock.mjs';
+import { resolveStateMdPath } from './state-md/frontmatter-mutators.mjs';
+import {
+  classifyManifestSession,
+  readProcessLocalSessionIds,
+} from './session-identity/own-session.mjs';
+import {
+  EventValidationError,
+  stampEventSchemaVersion,
+  validateEventRecord,
+} from './events-schema.mjs';
 
 // ---------------------------------------------------------------------------
 // Public API
@@ -69,16 +109,173 @@ export function sessionAttribution(repoRoot) {
   }
 }
 
+// ---------------------------------------------------------------------------
+// Correlation envelope (#1177 FA3)
+// ---------------------------------------------------------------------------
+
+/** State-dir candidates, in the same order `state-md` resolves them. */
+const STATE_DIR_CANDIDATES = ['.claude', '.codex', '.cursor', '.pi'];
+
+/**
+ * Session-correlation keys for a record pinned to `root` — `{}` when ownership
+ * is not provable.
+ *
+ * Decision, in one line: **only a PROCESS-LOCAL id may confirm the lock, and
+ * when one exists it decides alone.**
+ *
+ *   - No lock (CI, a bare script) → `{}`. Nothing to attribute to.
+ *   - No process-local id (`CLAUDE_CODE_SESSION_ID` absent) → `{}`. Ownership is
+ *     UNPROVEN, and an unproven attribution is exactly the peer-id write #1123
+ *     forbids; an absent key costs a correlation, a wrong key costs a false one.
+ *   - A process-local id that equals the lock's raw `session_id` → fill BOTH
+ *     keys, verbatim from the lock.
+ *   - A process-local id that DISAGREES → `{}` (the lock names a peer that won
+ *     the acquire race).
+ *
+ * **Why STATE.md is not a witness (#1177 FX1).** It used to be one, unioned
+ * with the env id — and the union was the bug: `.claude/STATE.md` is a SHARED
+ * working-copy artefact written by the session that OWNS the working copy,
+ * i.e. normally the lock holder. When a peer holds the lock, the peer also
+ * wrote STATE.md, so both "independent" witnesses name the PEER and a
+ * disagreeing process-local id could not veto them. Measured: lock=peer,
+ * STATE.md=peer, `CLAUDE_CODE_SESSION_ID`=me → the peer's ids were stamped on
+ * this session's records. A shared file cannot prove which PROCESS is emitting;
+ * see `readProcessLocalSessionIds()` for the tiering rationale (HR-102: a
+ * better signal replaces a worse one, it does not merely get outvoted by it).
+ *
+ * CEILING (BV-004): the comparison is against the lock's RAW `session_id`, so a
+ * harness that ROTATES its session id mid-session (see
+ * `tests/hooks/on-session-end.test.mjs` `new-rotated-uuid`) has an env id that
+ * no longer equals the lock's raw id, and BOTH keys are then omitted — honest
+ * absence, never misattribution. REVISIT when the rotation rate is measured in
+ * `events.jsonl` (count `orchestrator.session.started` against lock rewrites):
+ * if rotation is common, the lock must be refreshed on rotation rather than
+ * this comparison widened.
+ *
+ * @param {string} [root=SO_PROJECT_DIR] — the repo the record is pinned to.
+ * @returns {{session_id?: string, semantic_session_id?: string}}
+ */
+export function attributionForRecord(root = SO_PROJECT_DIR) {
+  const attribution = sessionAttribution(root);
+  const lockRawId =
+    typeof attribution.session_id === 'string' ? attribution.session_id.trim() : '';
+  if (!lockRawId) return {};
+  const processLocal = readProcessLocalSessionIds();
+  if (processLocal.length === 0) return {};
+  return processLocal.includes(lockRawId) ? { ...attribution } : {};
+}
+
+/**
+ * Absolute path of the active `wave-scope.json`, or `null` when none exists.
+ *
+ * The active platform's state dir is tried first (via `resolveStateMdPath()`,
+ * the repo's existing resolver), then the remaining candidates — so a Codex or
+ * Cursor run finds its own manifest rather than a stale `.claude/` one.
+ *
+ * @param {string} root
+ * @returns {string|null}
+ */
+function waveScopePath(root) {
+  const dirs = [];
+  try {
+    dirs.push(path.dirname(resolveStateMdPath(root)));
+  } catch {
+    /* fall through to the fixed candidate list */
+  }
+  for (const dir of STATE_DIR_CANDIDATES) {
+    const abs = path.join(root, dir);
+    if (!dirs.includes(abs)) dirs.push(abs);
+  }
+  for (const dir of dirs) {
+    const candidate = path.join(dir, 'wave-scope.json');
+    try {
+      if (existsSync(candidate)) return candidate;
+    } catch {
+      /* unreadable candidate — try the next one */
+    }
+  }
+  return null;
+}
+
+/**
+ * `{ wave }` from the live wave-scope manifest — `{}` when the manifest is
+ * missing, waveless, or belongs to another session.
+ *
+ * The manifest is a SHARED working-copy artefact (`.claude/wave-scope.json`),
+ * so a peer session's manifest is readable here and would stamp this session's
+ * events with a foreign wave number. Ownership is classified with
+ * `classifyManifestSession()` against the PROCESS-LOCAL id set plus whatever
+ * `attributionForRecord()` actually filled — the same tiering as the session
+ * keys, for the same reason (a shared file cannot prove which process emits):
+ *
+ *   - manifest classified `own` → fill (as a NUMBER, see below).
+ *   - anything else → omit. That includes an UNBOUND manifest (no `session` /
+ *     `semantic_session`): since #1123 BOTH writers stamp the binding, so a
+ *     manifest without one is a peer's or a stale artefact, never a legacy own
+ *     one. It also includes `unknown` because we cannot resolve our own
+ *     identity — stricter than `classifyManifestSession()`'s own `unknown`
+ *     doctrine on purpose: a wave number is data on the record, not a feature
+ *     gate, so "cannot tell" must not become "stamp it anyway".
+ *
+ * @param {string} root
+ * @param {{session_id?: string, semantic_session_id?: string}} attribution
+ * @returns {{wave?: number}}
+ */
+function waveForRecord(root, attribution) {
+  let scope;
+  try {
+    const file = waveScopePath(root);
+    if (!file) return {};
+    scope = JSON.parse(readFileSync(file, 'utf8'));
+  } catch {
+    return {};
+  }
+  // The ledger's `wave` is numeric in 1876 of 1876 live records; a manifest
+  // carrying `"wave": "3"` used to write the STRING through verbatim and split
+  // every downstream group-by. Coerce, and omit anything that is not an integer
+  // (`"abc"`, `2.5`) rather than writing a NaN or a fraction.
+  const waveNum = Number(scope?.wave);
+  if (scope?.wave === null || scope?.wave === '' || !Number.isInteger(waveNum)) return {};
+
+  const ownIds = new Set([
+    ...readProcessLocalSessionIds(),
+    ...[attribution.session_id, attribution.semantic_session_id].filter(Boolean),
+  ]);
+  const { verdict } = classifyManifestSession(scope, ownIds);
+  return verdict === 'own' ? { wave: waveNum } : {};
+}
+
 /**
  * Append a JSONL event record and optionally POST to the Clank Event Bus webhook.
  *
- * Writes `{ts, event, ...payload}` as a single JSON line to
- * `.orchestrator/metrics/events.jsonl` (creates parent directory if needed).
+ * Writes `{timestamp, event, schema_version, ...payload}` as a single JSON line
+ * to `.orchestrator/metrics/events.jsonl` (creates parent directory if needed).
  * If both `CLANK_EVENT_SECRET` and `CLANK_EVENT_URL` are set, fires an async
  * fire-and-forget POST to `CLANK_EVENT_URL` with a 3-second timeout. Network
  * errors are swallowed. Write errors propagate to the caller. No personal-domain
  * default URL exists — both vars must be set explicitly (#228).
  *
+ * Validation + versioning (#1177). Every record is stamped via
+ * `stampEventSchemaVersion()` (the schema module's own stamper — it fills the
+ * field only when absent, so a caller keeps authority over it) and run through
+ * `validateEventRecord()` BEFORE any side effect. An invalid record throws
+ * `EventValidationError` and produces NO ledger line and NO webhook POST —
+ * a malformed event is dropped at the producer rather than written and
+ * discovered by a downstream reader. The stamp is applied AFTER the payload
+ * spread, and still yields to it: the helper fills the field only when it is
+ * absent or null, so a caller supplying its own `schema_version` wins.
+ *
+ * Correlation envelope (#1177 FA3). When the payload carries neither session
+ * key, `session_id`/`semantic_session_id` are filled from
+ * {@link attributionForRecord}; when it carries no `wave`, `wave` is filled
+ * from the OWN wave-scope manifest. Both are additive and omitted whenever
+ * ownership is unproven — see the module header for the three rules.
+ *
+ * The webhook body deliberately stays `{ event_type, source, payload }` with the
+ * RAW payload — the wire format is a published contract with an external
+ * consumer; `schema_version` describes the JSONL record, not the webhook
+ * envelope, and is not added to it.
+ *
  * @param {string} type — event type (e.g. "orchestrator.session.started")
  * @param {object} [payload={}] — additional fields shallow-merged into the record
  * @param {object} [opts={}] — emission options.
@@ -95,8 +292,44 @@ export function sessionAttribution(repoRoot) {
  * @returns {Promise<void>}
  */
 export async function emitEvent(type, payload = {}, opts = {}) {
-  // Build the JSONL record: ts + event come first, payload spreads last.
-  const record = { timestamp: new Date().toISOString(), event: type, ...payload };
+  // Correlation envelope (#1177 FA3) — computed against the SAME root the line
+  // is pinned to. Both fills are gated on the payload NOT already carrying the
+  // key, and both spread BEFORE `payload`, so a caller always wins twice over.
+  // A payload that supplies EITHER session key suppresses BOTH: mixing a
+  // caller's `session_id` with a lock-derived `semantic_session_id` would
+  // silently produce a record whose two id fields name different sessions.
+  const attributionRoot = opts.repoRoot ?? SO_PROJECT_DIR;
+  const correlation = {};
+  if (payload.session_id === undefined && payload.semantic_session_id === undefined) {
+    Object.assign(correlation, attributionForRecord(attributionRoot));
+  }
+  if (payload.wave === undefined) {
+    Object.assign(correlation, waveForRecord(attributionRoot, correlation));
+  }
+
+  // Build the JSONL record: timestamp + event first, payload spreads last, and
+  // `stampEventSchemaVersion()` — the schema module's own stamper, which only
+  // fills an ABSENT/null field — adds the version. Routing through the helper
+  // instead of inlining `schema_version: CURRENT_SCHEMA_VERSION` keeps the
+  // stamp rule in ONE place: a caller-supplied version still wins, because the
+  // helper never overwrites a value the spread already put there.
+  const record = stampEventSchemaVersion({
+    timestamp: new Date().toISOString(),
+    event: type,
+    ...correlation,
+    ...payload,
+  });
+
+  // Validate BEFORE any side effect — no line, no directory, no webhook (#1177).
+  const verdict = validateEventRecord(record);
+  if (!verdict.valid) {
+    throw new EventValidationError(
+      `invalid event record for "${String(type)}": ${verdict.errors.join('; ')}`,
+      verdict.errors,
+      typeof type === 'string' ? type : undefined,
+    );
+  }
+
   const line = JSON.stringify(record) + '\n';
 
   // Ensure the destination directory exists before appending. Resolution order:

package/scripts/lib/evolve/autonomy-verdict.mjs

@@ -223,10 +223,15 @@ function readinessConfidence(autopilotSummary, judgmentSummary, score) {
 /**
  * Summarize autopilot run history plus type-8 mode effectiveness rollups.
  *
- * Abandoned-session filtering (#834): `sessions` is passed straight through
- * to `groupByMode()`, which filters phantom `status: 'abandoned'` stubs
- * before bucketing — this function inherits that guarantee transitively and
- * does not duplicate the filter. See `autopilot-effectiveness.mjs` `groupByMode()`.
+ * Abandoned-session filtering (#834) AND duplicate-identity collapse (#1167):
+ * `sessions` is passed straight through to `groupByMode()`, which canonicalizes
+ * the array (one record per physical session) and then filters phantom
+ * `status: 'abandoned'` stubs before bucketing — this function inherits BOTH
+ * guarantees transitively and duplicates neither. The abandoned filter alone
+ * was not enough: a `supersedes` pair and an exact same-id duplicate are two
+ * records of one session that both survive it. See
+ * `autopilot-effectiveness.mjs` `groupByMode()` (which canonicalizes via
+ * `canonicalizeSessions(sessions, { keepUnidentified: true })` before filtering).
  *
  * @param {Array} autopilotRuns
  * @param {Array} sessions

package/scripts/lib/evolve/autopilot-effectiveness.mjs

@@ -27,6 +27,7 @@
 import { randomUUID } from 'node:crypto';
 
 import { filterRealSessions } from '../session-schema.mjs';
+import { canonicalizeSessions } from '../sessions-canonical.mjs';
 
 // ---------------------------------------------------------------------------
 // Constants
@@ -181,7 +182,23 @@ export function groupByMode(autopilotRuns, sessions) {
   const out = new Map();
   if (!Array.isArray(sessions) || sessions.length === 0) return out;
 
-  const realSessions = filterRealSessions(sessions);
+  // Canonicalize BEFORE filtering (#1167). `filterRealSessions` drops phantom
+  // `abandoned` stubs, but two records of ONE physical session that are both
+  // real survive it — a `supersedes` pair (the backfilled stub plus the
+  // authoritative `completed` record that refutes it) and an exact same-id
+  // duplicate line both do. Each of those inflates `n_manual` / `n_autopilot`
+  // and skews every mean computed from the bucket, so the identity collapse
+  // has to happen first; the two compose in exactly this order (see
+  // `sessions-canonical.mjs` § "What this module does not do").
+  //
+  // `keepUnidentified: true` because effectiveness analysis must not LOSE the
+  // id-less rows `canonicalizeSessions` drops by default: legacy ledger rows
+  // and every in-repo fixture of the pre-id era carry `session_type` and
+  // metrics but no id, and dropping them would silently shrink `n_manual`
+  // instead of de-duplicating it.
+  const realSessions = filterRealSessions(
+    canonicalizeSessions(sessions, { keepUnidentified: true }),
+  );
   if (realSessions.length === 0) return out;
 
   // Optional: known autopilot_run_id set for stricter pairing. Empty set means