session-orchestrator 3.16.0 → 3.17.0

package/scripts/lib/dispatcher/enumerate.mjs

@@ -2,8 +2,9 @@
  * enumerate.mjs — Candidate-repo enumeration + free/busy resolution.
  *
  * Epic #673 Phase 2 (issue #676, PRD §2 P2.1+P2.2, §4). Enumerates candidate
- * repos one level below a confinement root and resolves each as free or busy
- * via its per-repo `session.lock` v2 lease (heartbeat-based liveness).
+ * repos below a confinement root (recursive walk, depth-capped — see
+ * {@link DEFAULT_MAX_DEPTH}) and resolves each as free or busy via its per-repo
+ * `session.lock` v2 lease (heartbeat-based liveness).
  *
  * Source of truth for free/busy: the same lease semantics as
  * `scripts/lib/vault-status/board-writer.mjs` collectRows —
@@ -15,8 +16,8 @@
  * the {@link Candidate} contract defined here.
  *
  * Exports:
- *   enumerateCandidates  — scan immediate children of a startDir, resolve each
- *                          repo's free/busy status from its lease.
+ *   enumerateCandidates  — walk a startDir up to `maxDepth` levels deep, resolve
+ *                          each repo's free/busy status from its lease.
  *   freeCandidates       — filter helper: keep only `free === true` candidates.
  *
  * No top-level side effects. All filesystem + lock access is dependency-injected
@@ -47,6 +48,67 @@ const STATUS_FREI = 'frei';
 const STATUS_IN_PROGRESS = 'in-progress';
 const STATUS_FORCE_CLOSED = 'force-closed';
 
+/**
+ * Default walk depth. `1` = immediate children of the scan root only (the
+ * pre-#832 behaviour); `2` additionally covers `<root>/<org>/<repo>`.
+ *
+ * Measured on the reference host (~/Projects, 2026-07-19; warm dentry cache —
+ * a cold first walk costs roughly 10x these figures at either depth):
+ *   depth 1 →  1 of 47 repos (2%)   — the real topology is `<org>/<repo>`,
+ *                                     so the scan missed 46 repos including
+ *                                     session-orchestrator itself.
+ *   depth 2 → 45 of 47 repos, ~0.9-1.9ms
+ *   depth 3 → 47 of 47 repos, ~8ms (node_modules is pruned, but the extra
+ *                                     level still multiplies the node count)
+ *                                     for two additional repos, both archived.
+ * Depth 2 is therefore the default: it recovers 96% of the host's repos at
+ * negligible cost, while depth 3 costs ~8x the walk for two dead repos.
+ */
+const DEFAULT_MAX_DEPTH = 2;
+
+/** Hard bounds for {@link clampMaxDepth}. Depth 3 is the ceiling by measurement. */
+const MIN_MAX_DEPTH = 1;
+const MAX_MAX_DEPTH = 3;
+
+/**
+ * Directory names never DESCENDED into during the walk.
+ *
+ * Applied to the descent decision ONLY — never to repo emission — so the
+ * depth-1 contract stays byte-identical to the pre-#832 scan (`.orchestrator`,
+ * `.claude` etc. are still probed for a `.git` marker at depth 1; they simply
+ * have none). `node_modules` is the dominant cost driver at depth 3 and can
+ * legitimately contain vendored `.git` directories that are not host repos.
+ *
+ * @param {string} name — a single path segment (Dirent.name).
+ * @returns {boolean} true iff the walk may recurse into this directory.
+ */
+function shouldDescendInto(name) {
+  if (typeof name !== 'string' || name.length === 0) return false;
+  if (name === 'node_modules') return false;
+  // Dot-directories (.git, .claude, .orchestrator, .venv, …) hold no host repos.
+  if (name.startsWith('.')) return false;
+  return true;
+}
+
+/**
+ * Normalise a caller-supplied `maxDepth` into the supported 1..3 range.
+ * Anything that is not a positive finite number falls back to
+ * {@link DEFAULT_MAX_DEPTH} — including `0`, negatives, `NaN`, and non-numbers
+ * such as the string `'3'` (no coercion: a string is a caller bug, and silently
+ * honouring it would make an unvalidated config value widen the walk).
+ *
+ * @param {unknown} value
+ * @returns {number} an integer in [MIN_MAX_DEPTH, MAX_MAX_DEPTH].
+ */
+function clampMaxDepth(value) {
+  if (typeof value !== 'number' || !Number.isFinite(value)) return DEFAULT_MAX_DEPTH;
+  const truncated = Math.trunc(value);
+  if (truncated <= 0) return DEFAULT_MAX_DEPTH;
+  if (truncated < MIN_MAX_DEPTH) return MIN_MAX_DEPTH;
+  if (truncated > MAX_MAX_DEPTH) return MAX_MAX_DEPTH;
+  return truncated;
+}
+
 /**
  * Expand a leading `~` to the current user's home directory. Mirrors the helper
  * in board-writer.mjs (a shared extraction is deferred to a later epic). Used to
@@ -126,20 +188,32 @@ function isGitRepo(childAbs, existsSyncFn) {
  * free or busy via its local lease.
  *
  * Algorithm:
- *   1. Scan the IMMEDIATE children (one level deep) of `startDir` that are git
- *      repos (a child is a repo iff `<child>/.git` exists — dir or file).
- *   2. Drop any child failing the confinement guard
- *      (`validatePathInsideProject(childAbs, startDir)`).
+ *   1. Depth-first walk of `startDir`, up to `maxDepth` levels deep (depth 1 =
+ *      immediate children). Every directory node is a repo candidate iff
+ *      `<node>/.git` exists (dir or file — the file form covers worktrees).
+ *   2. Confinement guard (`validatePathInsideProject(nodeAbs, startDir)`) runs
+ *      on EVERY node BEFORE it is emitted AND before it is opened — see the
+ *      security notes on the walk body below.
  *   3. OPTIONAL secondary source: union with `getCrossRepoProjects()`
  *      config-declared paths (leading `~/` expanded, then confinement-filtered),
  *      deduped by `path.resolve()`. Additive, applied AFTER the FS scan.
  *   4. Resolve free/busy per repo from its `session.lock` lease.
  *
+ * A git repo does NOT terminate the descent: on a measured reference host, an
+ * org-level directory one level under the confinement root was itself a git
+ * repo (a small umbrella notes repo) that CONTAINED 16 independent repos —
+ * this plugin's own checkout among them. "A repo's children are not separate
+ * repos" is empirically false there, and an early-exit-on-`.git` walk dropped
+ * 45 discoverable repos to 29. Pruning is therefore by NAME
+ * ({@link shouldDescendInto}), never by `.git` presence.
+ *
  * ALL repos are returned (busy ones LISTED, not dropped — downstream rank.mjs
  * filters). Returns a plain serialisable {@link Candidate}[].
  *
  * @param {object} [opts]
  * @param {string} [opts.startDir] — scan root; defaults to {@link getConfinementRoot}().
+ * @param {number} [opts.maxDepth] — walk depth, clamped to 1..3; defaults to
+ *   {@link DEFAULT_MAX_DEPTH} (2) for anything non-numeric, non-finite, or <= 0.
  * @param {number} [opts.now] — clock seam in ms; defaults to Date.now().
  * @param {object} [opts.deps] — dependency-injection seam (Wave-4 testability).
  * @param {Function} [opts.deps.readdirSync]   — node:fs readdirSync.
@@ -151,7 +225,7 @@ function isGitRepo(childAbs, existsSyncFn) {
  * @param {Function} [opts.deps.now] — () => ms (overridden by opts.now when set).
  * @returns {Promise<Candidate[]>}
  */
-export async function enumerateCandidates({ startDir, now, deps } = {}) {
+export async function enumerateCandidates({ startDir, now, maxDepth, deps } = {}) {
   const d = deps ?? {};
   const readdirSyncFn = d.readdirSync ?? readdirSync;
   const existsSyncFn = d.existsSync ?? existsSync;
@@ -165,6 +239,7 @@ export async function enumerateCandidates({ startDir, now, deps } = {}) {
     ? startDir
     : getConfinementRoot();
   const nowMs = typeof now === 'number' ? now : nowFn();
+  const depthCap = clampMaxDepth(maxDepth);
 
   // Dedup set keyed by resolved absolute path; preserves first-seen ordering
   // (FS-scan repos first, config-declared additions after).
@@ -179,32 +254,77 @@ export async function enumerateCandidates({ startDir, now, deps } = {}) {
     repoPaths.push(resolved);
   };
 
-  // ── 1+2. FS scan of immediate children, confinement-guarded. ──
-  let entries;
-  try {
-    entries = readdirSyncFn(root, { withFileTypes: true });
-  } catch {
-    // Unreadable/absent startDir → no FS-scanned repos. The config-declared
-    // secondary source below may still contribute.
-    entries = [];
-  }
+  // ── 1+2. Depth-capped FS walk, confinement-guarded at every node. ──
+  //
+  // SECURITY (three load-bearing invariants — do not relax without re-reading
+  // validatePathInsideProject at scripts/lib/path-utils.mjs):
+  //
+  //   (i)  Every node is validated against the ORIGINAL `root`, NEVER against
+  //        its own parent. The guard's Phase 2 calls realpathSync, which
+  //        resolves EVERY intermediate component — so validating a grandchild
+  //        against the original root is both sufficient and complete at any
+  //        depth. Re-rooting per level (`validate(grandchild, childDir)`) would
+  //        validate a symlinked subtree against ITSELF and defeat the guard.
+  //
+  //   (ii) The guard runs BEFORE `readdirSync`, not merely before emission.
+  //        Pre-#832 the guard ran only after `isGitRepo` passed, which was safe
+  //        because a non-repo directory was never opened. Under recursion an
+  //        unguarded non-repo directory WOULD be opened, so an `ok:false` node
+  //        must be refused for descent as well as for emission.
+  //
+  //  (iii) The guard call is wrapped in try/catch. path-utils.mjs rethrows any
+  //        non-ENOENT realpath error, so a single mode-000 directory under the
+  //        scan root would otherwise throw straight out of enumerateCandidates
+  //        — and runDispatch (scripts/lib/dispatcher/cli.mjs) has no try/catch
+  //        around this call. The walk now validates ~52 nodes instead of 1, so
+  //        a throwing guard is treated as "skip this node".
+  //
+  // Unbounded recursion is impossible: `Dirent.isDirectory()` is false for a
+  // symlink-to-directory (verified empirically), so symlink cycles never enter
+  // the walk — and `depthCap` bounds it regardless, including for stubbed
+  // entries that do not implement isDirectory().
+  const walk = (dirAbs, depth) => {
+    let entries;
+    try {
+      entries = readdirSyncFn(dirAbs, { withFileTypes: true });
+    } catch {
+      // Unreadable/absent directory → this subtree contributes nothing.
+      // Siblings and the config-declared secondary source are unaffected.
+      return;
+    }
 
-  for (const entry of entries) {
-    // Only directories can be repos. Dirent.isDirectory() guards against files,
-    // sockets, etc. A stubbed entry may not implement isDirectory — fall back
-    // to treating it as a directory candidate (existsSync('.git') gates anyway).
-    const isDir = typeof entry?.isDirectory === 'function' ? entry.isDirectory() : true;
-    if (!isDir) continue;
+    for (const entry of entries) {
+      // Only directories can be repos. Dirent.isDirectory() guards against
+      // files, sockets, etc. A stubbed entry may not implement isDirectory —
+      // fall back to treating it as a directory candidate (the depth cap and
+      // existsSync('.git') gate the consequences).
+      const isDir = typeof entry?.isDirectory === 'function' ? entry.isDirectory() : true;
+      if (!isDir) continue;
 
-    const childAbs = path.join(root, entry.name);
-    if (!isGitRepo(childAbs, existsSyncFn)) continue;
+      const childAbs = path.join(dirAbs, entry.name);
 
-    // Confinement guard: drop anything not strictly inside startDir.
-    const guard = validatePathInsideProjectFn(childAbs, root);
-    if (!guard || guard.ok !== true) continue;
+      // Confinement guard — invariants (i)+(ii)+(iii) above.
+      let guard;
+      try {
+        guard = validatePathInsideProjectFn(childAbs, root);
+      } catch {
+        continue;
+      }
+      if (!guard || guard.ok !== true) continue;
 
-    addRepo(childAbs);
-  }
+      if (isGitRepo(childAbs, existsSyncFn)) addRepo(childAbs);
+
+      // Descent is INDEPENDENT of repo-ness: a repo may contain further repos
+      // (the umbrella-repo case documented above). Prune by name only, and only
+      // for the descent decision — emission above is untouched, which keeps the
+      // depth-1 contract byte-identical to the pre-#832 scan.
+      if (depth < depthCap && shouldDescendInto(entry.name)) {
+        walk(childAbs, depth + 1);
+      }
+    }
+  };
+
+  walk(root, 1);
 
   // ── 3. Optional secondary source: config-declared cross-repo projects. ──
   let declared;

package/scripts/lib/dispatcher/rank.mjs

@@ -35,6 +35,7 @@ import path from 'node:path';
 import { scanBacklog } from '../backlog-scan.mjs';
 import { checkCiStatus as realCheckCiStatus } from '../ci-status-banner.mjs';
 import { probe as realProbe, evaluate as realEvaluate } from '../resource-probe.mjs';
+import { isRealSession } from '../session-schema/filters.mjs';
 
 /** Staleness cap (days). Beyond this, additional age does not raise the score. */
 export const STALENESS_CAP_DAYS = 90;
@@ -156,13 +157,21 @@ async function defaultFetchPriority(repoRoot, nowMs) {
 
 /**
  * Default STALENESS source: read `<repoRoot>/.orchestrator/metrics/sessions.jsonl`,
- * take the LAST record, and compute days since `completed_at` (fallback
- * `started_at`). No file / no parsable record / no timestamp ⇒
- * `STALENESS_CAP_DAYS` (treat as maximally stale = most worthwhile).
+ * find the last REAL (non-phantom) record scanning backward from the tail, and
+ * compute days since `completed_at` (fallback `started_at`). No file / no
+ * parsable REAL record / no timestamp ⇒ `STALENESS_CAP_DAYS` (treat as
+ * maximally stale = most worthwhile).
+ *
+ * Scans backward PAST any trailing `status: 'abandoned'` phantom stubs (#834)
+ * — session-close-backfill writes these for sessions that ended without a real
+ * close (0 waves, seconds of runtime). Stopping at the raw last LINE would let
+ * a single recent phantom make a genuinely neglected repo look freshly
+ * touched, defeating the dispatcher's whole purpose (this is the N=1 extreme
+ * case of the phantom-tail problem — one stub is enough to zero out staleness).
  *
  * @param {string} repoRoot
  * @param {number} nowMs
- * @returns {Promise<number>} days since last session (≥ 0)
+ * @returns {Promise<number>} days since last REAL session (≥ 0)
  */
 async function defaultStaleDaysFor(repoRoot, nowMs) {
   try {
@@ -171,14 +180,19 @@ async function defaultStaleDaysFor(repoRoot, nowMs) {
     const lines = raw.split('\n').map((l) => l.trim()).filter(Boolean);
     if (lines.length === 0) return STALENESS_CAP_DAYS;
 
-    // Last non-empty line = most recent session record.
+    // Scan backward for the last REAL (non-abandoned) session record, skipping
+    // both corrupt lines and phantom stubs.
     let last = null;
     for (let i = lines.length - 1; i >= 0; i -= 1) {
+      let parsed;
       try {
-        last = JSON.parse(lines[i]);
-        break;
+        parsed = JSON.parse(lines[i]);
       } catch {
-        // Skip a corrupt trailing line and try the previous one.
+        continue; // Skip a corrupt line and try the previous one.
+      }
+      if (isRealSession(parsed)) {
+        last = parsed;
+        break;
       }
     }
     if (!last || typeof last !== 'object') return STALENESS_CAP_DAYS;

package/scripts/lib/evolve/autonomy-verdict.mjs

@@ -223,6 +223,11 @@ function readinessConfidence(autopilotSummary, judgmentSummary, score) {
 /**
  * Summarize autopilot run history plus type-8 mode effectiveness rollups.
  *
+ * Abandoned-session filtering (#834): `sessions` is passed straight through
+ * to `groupByMode()`, which filters phantom `status: 'abandoned'` stubs
+ * before bucketing — this function inherits that guarantee transitively and
+ * does not duplicate the filter. See `autopilot-effectiveness.mjs` `groupByMode()`.
+ *
  * @param {Array} autopilotRuns
  * @param {Array} sessions
  * @returns {object}

package/scripts/lib/evolve/autopilot-effectiveness.mjs

@@ -26,6 +26,8 @@
 
 import { randomUUID } from 'node:crypto';
 
+import { filterRealSessions } from '../session-schema.mjs';
+
 // ---------------------------------------------------------------------------
 // Constants
 // ---------------------------------------------------------------------------
@@ -69,18 +71,40 @@ function isoPlusDays(nowIso, days) {
   }
 }
 
+/**
+ * Return the `effectiveness` sub-object of a session record, or an empty object
+ * when absent/malformed. Session records NEST their effectiveness metrics under
+ * this key — see `skills/session-end/metrics-collection.md` (the writer) and
+ * `scripts/lib/eval/session-resolve.mjs` (a sibling reader).
+ *
+ * @param {object} s
+ * @returns {object} the nested block, or `{}` when there is none
+ */
+function effectivenessOf(s) {
+  const e = s?.effectiveness;
+  return e && typeof e === 'object' && !Array.isArray(e) ? e : {};
+}
+
 /**
  * Extract a completion ratio from a session record. Sessions encode this in a
  * few historically-evolved shapes; we tolerate all of them and fall back to
  * `null` (excluded from the average) when nothing usable is present.
  *
+ * Shape precedence (#835): the NESTED `effectiveness.completion_rate` is the
+ * shape the session-end writer actually emits and is therefore read FIRST. The
+ * top-level read is retained as a fallback for legacy/hand-written records —
+ * dropping it would silently zero out any record predating the nesting.
+ *
  * @param {object} s
  * @returns {number|null} ratio in [0, 1], or null when unknown
  */
 function completionOf(s) {
   if (!s || typeof s !== 'object') return null;
-  // Direct ratio fields
-  const direct = num(s.completion_rate ?? s.completion_ratio);
+  const eff = effectivenessOf(s);
+  // Direct ratio fields — nested (canonical) first, then legacy top-level.
+  const direct = num(
+    eff.completion_rate ?? eff.completion_ratio ?? s.completion_rate ?? s.completion_ratio,
+  );
   if (direct !== null && direct >= 0 && direct <= 1) return direct;
   // Planned vs completed counts
   const planned = num(s.planned_count ?? s.tasks_planned);
@@ -93,15 +117,18 @@ function completionOf(s) {
 
 /**
  * Extract a carryover ratio from a session record. Carryover = work not closed
- * within the session that flowed to a follow-up. Same tolerance pattern as
- * `completionOf`.
+ * within the session that flowed to a follow-up. Same tolerance pattern —
+ * and the same nested-first precedence (#835) — as `completionOf`.
  *
  * @param {object} s
  * @returns {number|null} ratio in [0, 1], or null when unknown
  */
 function carryoverOf(s) {
   if (!s || typeof s !== 'object') return null;
-  const direct = num(s.carryover_ratio ?? s.carryover_rate);
+  const eff = effectivenessOf(s);
+  const direct = num(
+    eff.carryover_ratio ?? eff.carryover_rate ?? s.carryover_ratio ?? s.carryover_rate,
+  );
   if (direct !== null && direct >= 0 && direct <= 1) return direct;
   const planned = num(s.planned_count ?? s.tasks_planned);
   const carried = num(s.carryover_count ?? s.tasks_carried_over);
@@ -132,6 +159,18 @@ function mean(values) {
  * whose autopilot_run_id matches a known run). For the skeleton it is used
  * only as a non-emptiness signal.
  *
+ * Abandoned-session filtering (#834): `sessions` is filtered to REAL (non-
+ * phantom) records via `filterRealSessions()` before bucketing. Phantom
+ * `status: 'abandoned'` stubs are legitimate DATA but not legitimate SIGNAL —
+ * they must not inflate `n_manual`/`n_autopilot` or dilute the
+ * completion/carryover averages. See `scripts/lib/session-schema/filters.mjs`.
+ *
+ * Mode resolution (#834): `mode` is a LEGACY alias for the canonical
+ * `session_type` field (`session-schema/constants.mjs` `SESSION_KEY_ALIASES`).
+ * `session_type` is read first; `mode` is a fallback for legacy records that
+ * predate the rename (production ledgers overwhelmingly carry `session_type`,
+ * not `mode` — reading `mode` alone left this analyzer largely inert).
+ *
  * @param {Array} autopilotRuns
  * @param {Array} sessions
  * @returns {Map<string, {n_manual:number, n_autopilot:number,
@@ -142,6 +181,9 @@ export function groupByMode(autopilotRuns, sessions) {
   const out = new Map();
   if (!Array.isArray(sessions) || sessions.length === 0) return out;
 
+  const realSessions = filterRealSessions(sessions);
+  if (realSessions.length === 0) return out;
+
   // Optional: known autopilot_run_id set for stricter pairing. Empty set means
   // accept any session with a non-empty autopilot_run_id field.
   const knownRunIds = new Set();
@@ -156,9 +198,14 @@ export function groupByMode(autopilotRuns, sessions) {
 
   // Bucket: mode → {manual: [], autopilot: []}
   const buckets = new Map();
-  for (const s of sessions) {
+  for (const s of realSessions) {
     if (!s || typeof s !== 'object') continue;
-    const mode = typeof s.mode === 'string' ? s.mode : null;
+    const mode =
+      typeof s.session_type === 'string'
+        ? s.session_type
+        : typeof s.mode === 'string'
+          ? s.mode
+          : null;
     if (!mode) continue;
     const apId = typeof s.autopilot_run_id === 'string' ? s.autopilot_run_id : null;
     const isAutopilot =

package/scripts/lib/harness-audit/categories/category4.mjs

@@ -11,6 +11,7 @@ import { existsSync } from 'node:fs';
 import { join, relative, sep } from 'node:path';
 
 import { parseFrontmatter, safeRead, parseJsonl, pass, fail } from './helpers.mjs';
+import { isRealSession } from '../../session-schema/filters.mjs';
 
 // Default session-lock TTL in hours — mirrors DEFAULT_TTL_HOURS in
 // scripts/lib/session-lock.mjs. Inlined to keep this category stdlib-only
@@ -93,9 +94,19 @@ export function runCategory4(root) {
           { latestCompletedAt: null, ageInDays: null },
           'sessions.jsonl is empty'));
       } else {
-        const lastLine = lines[lines.length - 1];
+        // Scan backward past any trailing `status: 'abandoned'` phantom stubs
+        // (#834, session-close-backfill) to the last REAL session — otherwise a
+        // recent phantom lets a dormant repo pass a check meant to certify
+        // recent REAL engagement.
         let lastObj = null;
-        try { lastObj = JSON.parse(lastLine); } catch { /* ignore */ }
+        for (let i = lines.length - 1; i >= 0; i -= 1) {
+          let parsed = null;
+          try { parsed = JSON.parse(lines[i]); } catch { /* ignore */ }
+          if (parsed && isRealSession(parsed)) {
+            lastObj = parsed;
+            break;
+          }
+        }
         const completedAt = lastObj ? lastObj.completed_at : null;
         if (!completedAt) {
           checks.push(fail('sessions-jsonl-recent', 3, relPath,