session-orchestrator 5.12.0 → 5.13.0

package/scripts/lib/events.mjs

@@ -2,7 +2,7 @@
  * events.mjs — JSONL event emission + optional webhook POST for session-orchestrator.
  *
  * Replaces scripts/lib/events.sh. Windows-safe (no hardcoded path separators).
- * Uses native fetch (Node 20+) and fs.promises — no external dependencies.
+ * Uses Stdlib HTTP and fs.promises — no external dependencies.
  *
  * Part of v3.0.0 migration (Epic #124, issue #133).
  * Issue #228: removed hardcoded personal-domain default URL. Clank Event Bus URL
@@ -39,6 +39,7 @@
  * 0.0961 ms/call.
  */
 
+import { httpRequest } from './http-request.mjs';
 import {
   promises as fs,
   closeSync,
@@ -54,7 +55,7 @@ import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { getProjectDir, resolveEventsLedgerRoot, SO_SHARED_DIR } from './platform.mjs';
 import { readLock } from './session-lock.mjs';
-import { resolveStateMdPath } from './state-md/frontmatter-mutators.mjs';
+import { resolveStateMdWriteTarget } from './state-md/frontmatter-mutators.mjs';
 import {
   classifyManifestSession,
   readProcessLocalSessionIds,
@@ -282,9 +283,12 @@ export function attributionForRecord(root = getProjectDir()) {
 /**
  * Absolute path of the active `wave-scope.json`, or `null` when none exists.
  *
- * The active platform's state dir is tried first (via `resolveStateMdPath()`,
- * the repo's existing resolver), then the remaining candidates — so a Codex or
- * Cursor run finds its own manifest rather than a stale `.claude/` one.
+ * This session's state dir is tried first (the dir of
+ * `resolveStateMdWriteTarget()`'s path), then the remaining candidates — so a
+ * Codex or Cursor run finds its own manifest rather than a stale `.claude/`
+ * one. #1636: the write target, not the first existing STATE.md, because a
+ * foreign STATE.md in another platform's dir would put that dir's (foreign)
+ * manifest first and hide this session's own wave.
  *
  * @param {string} root
  * @returns {string|null}
@@ -292,7 +296,7 @@ export function attributionForRecord(root = getProjectDir()) {
 function waveScopePath(root) {
   const dirs = [];
   try {
-    dirs.push(path.dirname(resolveStateMdPath(root)));
+    dirs.push(path.dirname(resolveStateMdWriteTarget(root).path));
   } catch {
     /* fall through to the fixed candidate list */
   }
@@ -364,8 +368,9 @@ function waveForRecord(root, attribution) {
  *
  * Writes `{timestamp, event, schema_version, ...payload}` as a single JSON line
  * to `.orchestrator/metrics/events.jsonl` (creates parent directory if needed).
- * If both `CLANK_EVENT_SECRET` and `CLANK_EVENT_URL` are set, fires an async
- * fire-and-forget POST to `CLANK_EVENT_URL` with a 3-second timeout. Network
+ * If both `CLANK_EVENT_SECRET` and `CLANK_EVENT_URL` are set, fires a
+ * fire-and-forget POST (not awaited, #1612 R1-03) to `CLANK_EVENT_URL` with a
+ * 3-second timeout over node:https (`scripts/lib/http-request.mjs`). Network
  * errors are swallowed. Write errors propagate to the caller. No personal-domain
  * default URL exists — both vars must be set explicitly (#228).
  *
@@ -476,7 +481,7 @@ export async function emitEvent(type, payload = {}, opts = {}) {
   await fs.mkdir(path.dirname(filePath), { recursive: true });
   await fs.appendFile(filePath, line, 'utf8');
 
-  // Fire-and-forget webhook POST — only when secret AND URL are configured.
+  // R1-03: do not await: multiple events per hook × 3s exceed the 5s hook budget.
   // No personal-domain default: CLANK_EVENT_URL must be set explicitly (#228).
   if (process.env.CLANK_EVENT_SECRET && process.env.CLANK_EVENT_URL) {
     const url = process.env.CLANK_EVENT_URL;
@@ -486,14 +491,15 @@ export async function emitEvent(type, payload = {}, opts = {}) {
       payload,
     });
 
-    fetch(`${url}/api/webhooks/events`, {
+    void httpRequest(`${url}/api/webhooks/events`, {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
         'Authorization': `Bearer ${process.env.CLANK_EVENT_SECRET}`,
       },
       body,
-      signal: AbortSignal.timeout(3000),
+      timeoutMs: 3000,
+      maxBytes: 64 * 1024,
     }).catch(() => {});
   }
 }
@@ -948,7 +954,12 @@ export function scanEventsBackwards(opts = {}) {
     let fd;
     try {
       fd = openSync(source.path, 'r');
-      let pos = fstatSync(fd).size;
+      const stats = fstatSync(fd);
+      // Windows opens a directory "as a file" with size 0; without this it
+      // would read as an EMPTY source instead of an unreadable one (POSIX
+      // throws EISDIR on the first read).
+      if (!stats.isFile()) throw new Error(`not a regular file: ${source.path}`);
+      let pos = stats.size;
       /** Partial line at the FRONT of everything read from THIS source so far. */
       let carry = Buffer.alloc(0);
       while (pos > 0) {

package/scripts/lib/evolve/autonomy-verdict.mjs

@@ -223,15 +223,10 @@ function readinessConfidence(autopilotSummary, judgmentSummary, score) {
 /**
  * Summarize autopilot run history plus type-8 mode effectiveness rollups.
  *
- * Abandoned-session filtering (#834) AND duplicate-identity collapse (#1167):
- * `sessions` is passed straight through to `groupByMode()`, which canonicalizes
- * the array (one record per physical session) and then filters phantom
- * `status: 'abandoned'` stubs before bucketing — this function inherits BOTH
- * guarantees transitively and duplicates neither. The abandoned filter alone
- * was not enough: a `supersedes` pair and an exact same-id duplicate are two
- * records of one session that both survive it. See
- * `autopilot-effectiveness.mjs` `groupByMode()` (which canonicalizes via
- * `canonicalizeSessions(sessions, { keepUnidentified: true })` before filtering).
+ * Measured-work filtering (#1584) AND duplicate-identity collapse (#1167):
+ * sessions pass through groupByMode(), which canonicalizes physical identities
+ * (keeping ID-less legacy rows) and applies isMeasuredSession before bucketing.
+ * This function inherits both guarantees without a separate population filter.
  *
  * @param {Array} autopilotRuns
  * @param {Array} sessions

package/scripts/lib/evolve/autopilot-effectiveness.mjs

@@ -26,8 +26,7 @@
 
 import { randomUUID } from 'node:crypto';
 
-import { filterRealSessions } from '../session-schema.mjs';
-import { canonicalizeSessions } from '../sessions-canonical.mjs';
+import { canonicalizeSessions, isMeasuredSession } from '../sessions-canonical.mjs';
 
 // ---------------------------------------------------------------------------
 // Constants
@@ -160,11 +159,9 @@ function mean(values) {
  * whose autopilot_run_id matches a known run). For the skeleton it is used
  * only as a non-emptiness signal.
  *
- * Abandoned-session filtering (#834): `sessions` is filtered to REAL (non-
- * phantom) records via `filterRealSessions()` before bucketing. Phantom
- * `status: 'abandoned'` stubs are legitimate DATA but not legitimate SIGNAL —
- * they must not inflate `n_manual`/`n_autopilot` or dilute the
- * completion/carryover averages. See `scripts/lib/session-schema/filters.mjs`.
+ * Measured-session filtering (#1584): canonical identities are counted only
+ * when the shared isMeasuredSession predicate sees real work. Synthetic rows
+ * cannot inflate counts or dilute averages, even with completed status.
  *
  * Mode resolution (#834): `mode` is a LEGACY alias for the canonical
  * `session_type` field (`session-schema/constants.mjs` `SESSION_KEY_ALIASES`).
@@ -182,23 +179,16 @@ export function groupByMode(autopilotRuns, sessions) {
   const out = new Map();
   if (!Array.isArray(sessions) || sessions.length === 0) return out;
 
-  // Canonicalize BEFORE filtering (#1167). `filterRealSessions` drops phantom
-  // `abandoned` stubs, but two records of ONE physical session that are both
-  // real survive it — a `supersedes` pair (the backfilled stub plus the
-  // authoritative `completed` record that refutes it) and an exact same-id
-  // duplicate line both do. Each of those inflates `n_manual` / `n_autopilot`
-  // and skews every mean computed from the bucket, so the identity collapse
-  // has to happen first; the two compose in exactly this order (see
-  // `sessions-canonical.mjs` § "What this module does not do").
+  // Canonicalize BEFORE the shared measured-work filter (#1167, #1584):
+  // duplicate identities and supersedes pairs must not inflate bucket counts.
   //
   // `keepUnidentified: true` because effectiveness analysis must not LOSE the
   // id-less rows `canonicalizeSessions` drops by default: legacy ledger rows
   // and every in-repo fixture of the pre-id era carry `session_type` and
   // metrics but no id, and dropping them would silently shrink `n_manual`
   // instead of de-duplicating it.
-  const realSessions = filterRealSessions(
-    canonicalizeSessions(sessions, { keepUnidentified: true }),
-  );
+  const realSessions = canonicalizeSessions(sessions, { keepUnidentified: true })
+    .filter(isMeasuredSession);
   if (realSessions.length === 0) return out;
 
   // Optional: known autopilot_run_id set for stricter pairing. Empty set means

package/scripts/lib/fleet-protocol.mjs

@@ -26,6 +26,10 @@
  * `repo` field resolves to the reader's own git common dir ({@link readAuflagen},
  * #1520 / navigator#190).
  *
+ * There is no Rücklauf claim marker (removed in #1611): the owner inbox only lets
+ * a session rest when it reported in the turn of its last answer, so every
+ * waiting turn reports and nothing here suppresses a repeat.
+ *
  * Session ids become file names, so {@link isSafeSessionId} is the security
  * boundary between check-in input and the filesystem: every path builder that
  * takes a session id refuses an unsafe one with a TypeError.

package/scripts/lib/gates/gate-full.mjs

@@ -5,11 +5,15 @@
 // Required env: TYPECHECK_CMD, TEST_CMD, LINT_CMD (each may equal "skip")
 // Optional env: SESSION_START_REF
 
+import { mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
 import {
   runCheck,
   extractCount,
   extractTestCounts,
   extractFailedTestFiles,
+  extractTimeoutOnlyFailedFiles,
   collectDebugArtifacts,
   commandKillFields,
 } from './gate-helpers.mjs';
@@ -109,6 +113,147 @@ const failedTestFiles = testResult.status === 'fail'
   ? extractFailedTestFiles(testResult.fullOutput ?? testResult.output ?? '')
   : [];
 
+// --- Serial re-run of timeout-only failures (#1673) ---
+// Under host load the parallel suite loses tests to their timeout while the
+// code is fine: four pre-push runs on m5 blocked with four different red sets,
+// none in the pushed diff, and the pushes then went out with the bypass. When
+// EVERY failure of the run is a timeout, exactly those files run once more,
+// one file at a time; only a clean re-run turns the test step green. An
+// assertion failure is never re-run (it may be a genuine flake), a killed
+// test command is never re-run (no complete error list), and the CI pipeline
+// keeps running without this re-run, so it stays the strict instance.
+// NAMED CEILING (BV-004): 10 files. More timeouts than that is not a few
+// load victims but a broken run, and a serial re-run would double the wait.
+// Revisit when a blocked push shows 11+ timeout-only files under load.
+const RERUN_MAX_FILES = 10;
+// The names go into a shell command line, so they must be plain paths.
+const RERUN_FILE_NAME = /^[\w.@+][\w./@+-]*$/;
+// Paths placed inside double quotes on that line: nothing the shell expands.
+// NAMED CEILING (BV-004): a backslash is refused, so on Windows (`C:\…`
+// temp paths) no re-run happens and the gate stays red as before. Revisit with
+// the Windows gate work (#1612); cmd.exe also expands `%VAR%` inside quotes.
+const QUOTABLE_PATH = /^[^"$`\\%\n\r]+$/;
+
+/**
+ * The re-run command: the gate's OWN test command with the file filters and
+ * reporters appended, so its config, project selection, environment prefix and
+ * binary are exactly those of the first run (Codex review round 2, H2). Only
+ * shapes whose trailing arguments demonstrably reach vitest are accepted;
+ * anything else returns null and nothing is re-run.
+ * NAMED CEILING (BV-004): `npm test` / `npm run test` and a direct
+ * `[VAR=… ][exec|npx ][path/]vitest run|--run` without shell operators. pnpm/yarn pass
+ * `--` differently by version and are not accepted until measured. Revisit
+ * when a consumer repo with another test command hits #1673.
+ */
+function rerunCommand(cmd, args) {
+  const trimmed = (cmd ?? '').trim();
+  if (/^npm (?:run )?test$/.test(trimmed)) return `${trimmed} -- ${args}`;
+  // `[VAR=value …] [exec|npx] [path/]vitest run` — vitest itself is the program,
+  // so the appended arguments are its own; no wrapper that might drop them.
+  const direct = /^(?:[A-Za-z_][A-Za-z0-9_]*=[^\s;&|`$()<>]*\s+)*(?:(?:exec|npx)\s+)?(?:[^\s;&|`$()<>]*\/)?vitest (?:run|--run)$/;
+  if (direct.test(trimmed)) return `${trimmed} ${args}`;
+  return null;
+}
+
+const realOrSelf = (p) => {
+  try {
+    return realpathSync(p);
+  } catch {
+    return p;
+  }
+};
+
+/**
+ * Judge the re-run from vitest's JSON report, never from its exit code or its
+ * summary counts alone: the default reporter does not name passing files, so
+ * only the report proves that EXACTLY the requested files ran and passed (a
+ * filter matching another file, or a config excluding the file, would
+ * otherwise read as green). Returns `{ reason, passed, failed }`, `reason`
+ * null when clean.
+ */
+function judgeRerunReport(reportPath, files, cwd) {
+  let report;
+  try {
+    report = JSON.parse(readFileSync(reportPath, 'utf8'));
+  } catch {
+    return { reason: 'report-unreadable', passed: null, failed: null };
+  }
+  const counts = { passed: report?.numPassedTests ?? null, failed: report?.numFailedTests ?? null };
+  if (report?.success !== true || report.numFailedTests !== 0 || !Array.isArray(report.testResults)) {
+    return { reason: 'report-not-clean', ...counts };
+  }
+  const root = realOrSelf(cwd);
+  const ran = report.testResults.map((t) => [
+    path.relative(root, realOrSelf(String(t?.name ?? ''))).split(path.sep).join('/'),
+    t?.status,
+  ]);
+  const wanted = new Set(files);
+  const exact =
+    ran.length === files.length && ran.every(([file, status]) => status === 'passed' && wanted.delete(file));
+  return { reason: exact && wanted.size === 0 ? null : 'file-set-mismatch', ...counts };
+}
+
+let rerun = null;
+let rerunResult = null;
+// Only a run that ENDED with exit 1 — not by a signal (`exitSignal`, which
+// `exitCode` alone reports as 1), not by the gate's own kill — and printed
+// vitest's closing `Test Files` summary (`testCounts.files`). A signal that hits
+// vitest under `npm test` surfaces as npm exit 143 (measured, npm 11.19.1), so
+// both routes of an externally ended run stay out (Codex review rounds 2+3, M3).
+if (
+  testResult.status === 'fail' &&
+  testResult.timedOut !== true &&
+  testResult.exitCode === 1 &&
+  !testResult.exitSignal &&
+  (testResult.killSignals ?? []).length === 0 &&
+  testCounts.files
+) {
+  const timeouts = extractTimeoutOnlyFailedFiles(testResult.fullOutput ?? testResult.output ?? '');
+  if (
+    timeouts.eligible &&
+    timeouts.files.length <= RERUN_MAX_FILES &&
+    timeouts.files.every((f) => RERUN_FILE_NAME.test(f))
+  ) {
+    let reportDir = null;
+    try {
+      reportDir = mkdtempSync(path.join(tmpdir(), 'gate-rerun-'));
+      const reportPath = path.join(reportDir, 'report.json');
+      const command = QUOTABLE_PATH.test(reportPath)
+        ? rerunCommand(
+            testCmd,
+            // `default` keeps a readable failure on stderr; `json` is the proof.
+            `--no-file-parallelism --reporter=default --reporter=json --outputFile.json="${reportPath}" ${timeouts.files.join(' ')}`,
+          )
+        : null;
+      if (command !== null) {
+        rerunResult = await runCheck(command, CHECK_OPTS);
+        const judged = judgeRerunReport(reportPath, timeouts.files, process.cwd());
+        const verdict = rerunResult.status === 'pass' ? judged : { ...judged, reason: 'exit-non-zero' };
+        rerun = {
+          files: timeouts.files,
+          status: verdict.reason === null ? 'pass' : 'fail',
+          ...(verdict.reason ? { reason: verdict.reason } : {}),
+          passed: verdict.passed,
+          failed: verdict.failed,
+          ...commandKillFields(rerunResult),
+        };
+      }
+    } catch (err) {
+      // Any failure around the re-run keeps the step red and still names the files.
+      rerun = { files: timeouts.files, status: 'fail', reason: `rerun-error: ${err?.code ?? err?.message ?? 'unknown'}`, passed: null, failed: null };
+    } finally {
+      if (reportDir !== null) {
+        try {
+          rmSync(reportDir, { recursive: true, force: true });
+        } catch {
+          /* a leftover temp dir must not crash the gate or flip its verdict */
+        }
+      }
+    }
+  }
+}
+const testStatus = rerun?.status === 'pass' ? 'pass' : testResult.status;
+
 const fileFields = testCounts.files
   ? {
       files_total: testCounts.files.total,
@@ -119,7 +264,9 @@ const fileFields = testCounts.files
       // test-case `failed: 0` that a file-level failure explains. Greppable —
       // a consumer no longer has to recompute the contradiction by hand.
       suite_died:
-        testResult.status === 'fail' && testCounts.failed === 0 && testCounts.files.failed > 0,
+        // `testStatus`: a suite whose only failure was a hook timeout and that
+        // re-ran clean (#1673) did not die.
+        testStatus === 'fail' && testCounts.failed === 0 && testCounts.files.failed > 0,
     }
   : {};
 
@@ -154,8 +301,10 @@ const output = {
   // re-derived as `total - passed` downstream: the derivation and the parse can
   // disagree, and only an explicit third number lets a consumer check
   // `passed + failed === total` as a real producer/consumer drift guard.
+  // `status` is the step's verdict after a re-run; the counts below stay the
+  // first, parallel run's measurement, and `rerun` says what turned it green.
   test: {
-    status: testResult.status,
+    status: testStatus,
     total: testCounts.total,
     passed: testCounts.passed,
     failed: testCounts.failed,
@@ -166,6 +315,7 @@ const output = {
     // document and the test-case triple keeps `total === passed + failed`.
     ...fileFields,
     ...commandKillFields(testResult),
+    ...(rerun ? { rerun } : {}),
   },
   lint: { status: lintResult.status, warnings: lintWarnings, ...commandKillFields(lintResult) },
   debug_artifacts: debugArtifacts,
@@ -175,10 +325,19 @@ const output = {
 process.stdout.write(JSON.stringify(output) + '\n');
 
 // Exit 2 if any check failed; skipped counts as pass.
-const failed = [tcResult, testResult, lintResult].some(
-  (r) => r.status === 'fail',
+const failed = [tcResult.status, testStatus, lintResult.status].some(
+  (status) => status === 'fail',
 );
 
+// Every re-run is named, green or not, so a load-sensitive file stays visible
+// even when the push goes through.
+if (rerun) {
+  process.stderr.write(
+    `\n──── test re-run: ${rerun.files.length} file(s) failed only by timeout, re-ran serially → ${rerun.status} ────\n` +
+      rerun.files.map((f) => `  ${f}\n`).join(''),
+  );
+}
+
 // --- Failure disclosure ---
 // A gate that BLOCKS and says nothing is not concise, it is unusable. Measured
 // 2026-08-23: a pre-push run emitted exactly
@@ -207,20 +366,22 @@ if (failed) {
   // Names FIRST, raw output after. Under the pre-push hook the raw block is
   // hundreds of lines; the operator reads the top of it, and the one fact he
   // needs to act (`npx vitest run <file>`) must not sit at the bottom.
-  if (failedTestFiles.length > 0) {
+  if (testStatus === 'fail' && failedTestFiles.length > 0) {
     process.stderr.write(
       `\n──── failing test files (${failedTestFiles.length}) ────\n` +
         failedTestFiles.map((f) => `  ${f}\n`).join('') +
         `  reproduce: npx vitest run ${failedTestFiles.join(' ')}\n`,
     );
   }
-  for (const [name, result] of [
-    ['typecheck', tcResult],
-    ['test', testResult],
-    ['lint', lintResult],
+  for (const [name, result, status] of [
+    ['typecheck', tcResult, tcResult.status],
+    ['test', testResult, testStatus],
+    ['test re-run', rerunResult, rerun?.status],
+    ['lint', lintResult, lintResult.status],
   ]) {
-    if (result.status !== 'fail') continue;
-    const raw = result.fullOutput ?? result.output ?? '';
+    if (status !== 'fail') continue;
+    // A re-run that never started (temp dir, spawn) has no result, only a reason.
+    const raw = result ? (result.fullOutput ?? result.output ?? '') : `(not run: ${rerun?.reason ?? 'unknown'})\n`;
     process.stderr.write(`\n──── ${name} FAILED — raw output ────\n`);
     process.stderr.write(raw.length > 0 ? raw : '(the gate captured no output)\n');
     process.stderr.write(`──── end ${name} ────\n`);