session-orchestrator 3.16.0 → 3.19.0

package/scripts/lib/skill-evolution/idempotency.mjs

@@ -12,7 +12,18 @@
  * full via tmp-file + atomic rename. It is NEVER appended to with `>>` semantics,
  * because supersession reconciliation must mutate existing lines (set
  * `superseded_by`) — an append would leave stale duplicates and corrupt the
- * supersession graph.
+ * supersession graph. The store is OWNED by this module: nothing else — no
+ * report, no analysis run, no agent — may append to it; a read-side shape guard
+ * ({@link isRepairCandidateShape}) drops any record that is not a
+ * RepairCandidate, COUNTS the drop, and WARNs on stderr.
+ *
+ * The warning lives in {@link readStore} — the ONE seam every public function
+ * reads through — rather than in each caller's return shape, because only
+ * `mergeCandidates` has a result object with room for a count: `markProcessed`
+ * returns `{ok, reason}` and `isProcessed` returns a boolean, yet both rewrite
+ * or gate on a store whose foreign lines the rewrite DELETES. Those two run far
+ * more often than a merge, so a count reported only by `mergeCandidates` would
+ * leave the most-travelled deletion path silent.
  *
  * Three responsibilities:
  *   1. Persistence — `loadCandidates` (read) + atomic full rewrite (internal).
@@ -63,12 +74,90 @@ function resolveStorePath(repoRoot, storePath) {
   return rel;
 }
 
+/**
+ * Minimal shape guard for a persisted store line. A record is accepted only when
+ * it carries the three fields THIS store's own consumers read:
+ *   - `id`          — THE idempotency key (`mergeCandidates` byId index,
+ *                     `markProcessed`, `isProcessed` all look records up by it).
+ *   - `source`      — one half of the supersession key (`mergeCandidates`
+ *                     compares `rec.source === cand.source`).
+ *   - `target_path` — the other half of the supersession key.
+ *
+ * Deliberately NOT a full schema check, and NOT a copy of the reconcile store's
+ * guard: the two record types share only 5 fields, and the reconcile guard keys
+ * on `learning_key`, which a {@link RepairCandidate} does not have at all —
+ * applying it here would reject 100% of legitimate repair candidates. The fields
+ * above were chosen by what consumers actually READ, not by the typedef: a grep
+ * of `scripts/lib/skill-evolution/` (2026-07-31) found ZERO consumer reads of
+ * `created_at` — only the producer `candidate-intake.mjs` sets it — so requiring
+ * it would be schema validation, not a load-bearing check. The store is a
+ * mutable work-queue whose records may gain fields across schema versions, so
+ * over-strict validation would silently drop legitimate future records.
+ *
+ * What it DOES reject is a record no writer in this repo produces — the concrete
+ * incident class being a hand-written report artefact landing in the sibling
+ * reconcile store using `candidate_id` / `generated_at` / `status:"candidate"`
+ * (2026-07-31, GitLab #955; this store is the near-twin that kept the old
+ * permissive `typeof parsed === 'object'` check).
+ * @param {unknown} rec
+ * @returns {boolean}
+ */
+function isRepairCandidateShape(rec) {
+  if (!rec || typeof rec !== 'object' || Array.isArray(rec)) return false;
+  const r = /** @type {Record<string, unknown>} */ (rec);
+  if (typeof r.id !== 'string' || r.id.length === 0) return false;
+  if (typeof r.source !== 'string' || r.source.length === 0) return false;
+  if (typeof r.target_path !== 'string' || r.target_path.length === 0) return false;
+  return true;
+}
+
+/**
+ * Per-store memo of the skip count already WARNed about, keyed by resolved
+ * absolute store path. Without it a caller that reads the same store in a loop
+ * (the engine calls {@link isProcessed} once per candidate) would emit one
+ * identical line per read. A read that comes back CLEAN clears the entry, so a
+ * store contaminated again later warns again instead of being muted forever.
+ * @type {Map<string, number>}
+ */
+const warnedSkips = new Map();
+
+/**
+ * Emit ONE stderr WARN per (store, skip-count) transition. Called from every
+ * {@link readStore} exit — including the ENOENT path, whose `skipped: 0` clears
+ * the memo. Never throws: a stderr write failure must not break a store read.
+ * @param {string} absPath
+ * @param {number} skipped
+ * @returns {void}
+ */
+function warnOnSkipped(absPath, skipped) {
+  if (skipped <= 0) {
+    warnedSkips.delete(absPath);
+    return;
+  }
+  if (warnedSkips.get(absPath) === skipped) return;
+  warnedSkips.set(absPath, skipped);
+  try {
+    process.stderr.write(
+      `WARN repair-candidate store: ${skipped} unreadable record(s) skipped in ${absPath} — ` +
+        'not RepairCandidate shape (need id/source/target_path) or unparseable JSON. ' +
+        'The next full rewrite (mergeCandidates/markProcessed) DELETES them from disk.\n',
+    );
+  } catch {
+    // stderr unavailable (closed pipe) — the count is still returned to callers.
+  }
+}
+
 /**
  * Read + defensively parse the store's JSONL lines into RepairCandidate records.
- * Malformed lines (bad JSON, non-object) are skipped silently. A missing file
- * yields `[]`. Never throws.
+ * Malformed lines (bad JSON) and shape-foreign records (see
+ * {@link isRepairCandidateShape}) are skipped — and COUNTED, because every
+ * writer in this module rewrites the store in FULL, so a skipped line is dropped
+ * from disk on the next merge or stamp; a silent drop would be unattributable
+ * data loss. Every skip is ALSO reported on stderr via {@link warnOnSkipped},
+ * which is what covers the three callers whose return contract has no room for
+ * the count. A missing file yields `{ records: [], skipped: 0 }`. Never throws.
  * @param {string} absPath
- * @returns {RepairCandidate[]}
+ * @returns {{ records: RepairCandidate[], skipped: number }}
  */
 function readStore(absPath) {
   let raw;
@@ -76,11 +165,13 @@ function readStore(absPath) {
     raw = readFileSync(absPath, 'utf8');
   } catch {
     // ENOENT or any read error → empty store.
-    return [];
+    warnOnSkipped(absPath, 0);
+    return { records: [], skipped: 0 };
   }
 
   /** @type {RepairCandidate[]} */
   const records = [];
+  let skipped = 0;
   for (const line of raw.split('\n')) {
     const trimmed = line.trim();
     if (trimmed.length === 0) continue;
@@ -88,13 +179,17 @@ function readStore(absPath) {
     try {
       parsed = JSON.parse(trimmed);
     } catch {
+      skipped += 1;
       continue; // skip malformed line
     }
-    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+    if (isRepairCandidateShape(parsed)) {
       records.push(/** @type {RepairCandidate} */ (parsed));
+    } else {
+      skipped += 1;
     }
   }
-  return records;
+  warnOnSkipped(absPath, skipped);
+  return { records, skipped };
 }
 
 /**
@@ -127,8 +222,15 @@ function writeStore(absPath, records) {
 
 /**
  * Load every RepairCandidate currently persisted in the store. Reads JSONL,
- * skips malformed lines, returns `[]` for a missing file. Does NOT create the
- * runtime dir (mkdir -p happens only on write). Never throws.
+ * skips malformed lines AND shape-foreign records (missing `id` / `source` /
+ * `target_path`), returns `[]` for a missing file. Does NOT create the runtime
+ * dir (mkdir -p happens only on write). Never throws.
+ *
+ * Read-only: the skipped lines are still on disk after this call. The count is
+ * not in the return value — this function's plain-array contract is unchanged —
+ * but it is never silent: {@link readStore} WARNs it on stderr, and a caller
+ * that needs the number programmatically gets it from {@link mergeCandidates}
+ * as `dropped_on_read`.
  * @param {Object} params
  * @param {string} [params.repoRoot] - repo root; relative `storePath` is resolved against it.
  * @param {string} [params.storePath] - store path (relative ⇒ joined to repoRoot). Defaults to {@link DEFAULT_STORE_PATH}.
@@ -136,7 +238,7 @@ function writeStore(absPath, records) {
  */
 export function loadCandidates({ repoRoot, storePath } = {}) {
   const absPath = resolveStorePath(repoRoot, storePath);
-  return readStore(absPath);
+  return readStore(absPath).records;
 }
 
 /**
@@ -157,15 +259,25 @@ export function loadCandidates({ repoRoot, storePath } = {}) {
  * The store is rewritten in full (read-all → merge → atomic tmp+rename), never
  * appended. Never throws; a write failure still returns counts with
  * `total` reflecting the in-memory final line count.
+ *
+ * `dropped_on_read` reports how many PERSISTED lines the read-side guard
+ * rejected (malformed JSON, or a record missing `id`/`source`/`target_path`).
+ * Because the store is rewritten in full, those lines are DROPPED from disk by
+ * this call — the count is what makes that loss attributable instead of silent.
+ * It is deliberately NOT named `skipped`: `skipped_processed` above already
+ * means "an INCOMING candidate was skipped because its id is already processed",
+ * a different population (incoming, not persisted) with a different cause
+ * (terminal verdict, not contamination). Confusing the two would read a
+ * contaminated store as a healthy idempotent no-op.
  * @param {Object} params
  * @param {RepairCandidate[]} params.candidates - newly minted candidates to merge.
  * @param {string} [params.repoRoot]
  * @param {string} [params.storePath]
- * @returns {{ written: number, superseded: number, skipped_processed: number, total: number }}
+ * @returns {{ written: number, superseded: number, skipped_processed: number, dropped_on_read: number, total: number }}
  */
 export function mergeCandidates({ candidates, repoRoot, storePath } = {}) {
   const absPath = resolveStorePath(repoRoot, storePath);
-  const store = readStore(absPath);
+  const { records: store, skipped: droppedOnRead } = readStore(absPath);
 
   // Index existing records by id for O(1) lookup.
   /** @type {Map<string, RepairCandidate>} */
@@ -219,6 +331,7 @@ export function mergeCandidates({ candidates, repoRoot, storePath } = {}) {
     written,
     superseded,
     skipped_processed: skippedProcessed,
+    dropped_on_read: droppedOnRead,
     total: store.length,
   };
 }
@@ -230,7 +343,11 @@ export function mergeCandidates({ candidates, repoRoot, storePath } = {}) {
  *   - id not found → `{ ok: false, reason: 'not-found' }`
  *   - already processed → `{ ok: true }` (idempotent no-op; store untouched)
  *   - filesystem write failure → `{ ok: false, reason: 'fs-error' }`
- * Never throws.
+ * Never throws. Like {@link mergeCandidates} this rewrites the store in full, so
+ * shape-foreign lines are DELETED from disk here too. This API's result shape
+ * has no room to report the count (it is `{ ok, reason }` by contract), so the
+ * deletion is surfaced by {@link readStore}'s stderr WARN instead — this is the
+ * most-travelled of the three rewrite paths, so it must not be the silent one.
  * @param {Object} params
  * @param {string} params.id - candidate id to stamp.
  * @param {string} [params.repoRoot]
@@ -244,7 +361,7 @@ export function markProcessed({ id, repoRoot, storePath, now } = {}) {
   }
 
   const absPath = resolveStorePath(repoRoot, storePath);
-  const store = readStore(absPath);
+  const { records: store } = readStore(absPath);
 
   const target = store.find((rec) => rec && rec.id === id);
   if (!target) return { ok: false, reason: 'not-found' };
@@ -263,7 +380,9 @@ export function markProcessed({ id, repoRoot, storePath, now } = {}) {
 
 /**
  * True iff a candidate with the given id exists in the store AND has been
- * processed (`processed_at !== null`). Never throws.
+ * processed (`processed_at !== null`). Read-only — it does not itself rewrite
+ * the store — but it reads through the same shape guard, so a contaminated
+ * store is reported here too via {@link readStore}'s stderr WARN. Never throws.
  * @param {Object} params
  * @param {string} params.id
  * @param {string} [params.repoRoot]
@@ -273,7 +392,7 @@ export function markProcessed({ id, repoRoot, storePath, now } = {}) {
 export function isProcessed({ id, repoRoot, storePath } = {}) {
   if (typeof id !== 'string' || id.length === 0) return false;
   const absPath = resolveStorePath(repoRoot, storePath);
-  const store = readStore(absPath);
+  const { records: store } = readStore(absPath);
   const rec = store.find((r) => r && r.id === id);
   return !!rec && rec.processed_at !== null && rec.processed_at !== undefined;
 }

package/scripts/lib/skill-evolution/mr-opener.mjs

@@ -30,6 +30,9 @@
  *   - opts.vcsDetect  — ({ config, projectRoot }) => { bin } VCS detector
  *   - opts.createMr   — async (loop, draftOpts) => { created, mrUrl } MR opener
  *   - opts.log        — (level, msg) => void diagnostic logger
+ *   - opts.resolveRepoSpecFn — optional passthrough to the createMr seam's own
+ *     -R/--repo host-pinning resolver (#872); undefined lets createMr fall
+ *     back to its own real-resolveRepoSpec default.
  */
 
 import { execFile as execFileCb, spawnSync } from 'node:child_process';
@@ -468,6 +471,11 @@ export async function openRepairMr(
   }
 
   // Delegate MR creation to the mr-draft seam (collision check + draft create).
+  // repoRoot is passed through so maybeCreateDraftMR's -R/--repo host-pinning
+  // resolution (#872) resolves against THIS repo, not process.cwd() (which
+  // may differ inside a dispatched agent/worktree context); resolveRepoSpecFn
+  // is an optional passthrough seam for tests — undefined here simply means
+  // maybeCreateDraftMR falls back to its own real-resolveRepoSpec default.
   let mrResult;
   try {
     mrResult = await createMr(
@@ -480,7 +488,7 @@ export async function openRepairMr(
         parentRunId: `repair/${candidate.id}`,
         worktreePath: candidate.target_path ?? '(unknown)',
       },
-      { execFile: opts.execFile, log },
+      { execFile: opts.execFile, log, repoRoot, resolveRepoSpecFn: opts.resolveRepoSpecFn },
     );
   } catch (err) {
     const msg = err instanceof MrDraftError ? err.message : String(err?.message ?? err);

package/scripts/lib/skill-health/join.mjs

@@ -15,6 +15,18 @@
  * the session is still in progress, or the record was never written), the session
  * contributes `unknown` to the outcome tally — it is never silently dropped.
  *
+ * ABANDONED-SESSION HANDLING (#834): a session_id CAN be found in sessions.jsonl
+ * yet be a phantom stub (`status: 'abandoned'`, written by session-close-backfill
+ * for a session that ended without a real close — 0 waves, all-zero agent_summary).
+ * Counting such a join as `sessionsJoined` would inflate the skill's join
+ * denominator with zero real contribution — the join "succeeds" but carries no
+ * signal. Per this module's own "never silently dropped" contract, an abandoned
+ * join is routed to a DISTINCT `abandoned` outcome bucket rather than either (a)
+ * silently folding into `sessionsJoined`/the numeric outcome fields with zero
+ * contribution (inflates the denominator invisibly), or (b) folding into
+ * `unknown` (which means "not found in the ledger at all" — a different failure
+ * mode a caller may want to distinguish from "found but phantom").
+ *
  * Part of Epic #645 — Skill Self-Evolution Foundation, Layer 2.
  */
 
@@ -22,6 +34,8 @@ import { promises as fs } from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
+import { isRealSession } from '../session-schema/filters.mjs';
+
 const DEFAULT_INVOCATIONS_PATH = path.resolve(
   fileURLToPath(import.meta.url),
   '../../../../.orchestrator/metrics/skill-invocations.jsonl',
@@ -61,18 +75,21 @@ async function readJsonl(filePath) {
 }
 
 /**
- * Builds a Map<session_id, agent_summary> from sessions.jsonl records.
+ * Builds a Map<session_id, { agentSummary, real }> from sessions.jsonl records.
  * Records without a session_id or with a non-object agent_summary are skipped.
+ * `real` is false for phantom `status: 'abandoned'` stubs (#834) — callers use
+ * it to route the join to the `abandoned` outcome bucket instead of counting
+ * a zero-signal join as `sessionsJoined`.
  *
  * @param {object[]} sessionRecords
- * @returns {Map<string, { complete: number, partial: number, failed: number, spiral: number }>}
+ * @returns {Map<string, { agentSummary: { complete: number, partial: number, failed: number, spiral: number }, real: boolean }>}
  */
 function buildSessionMap(sessionRecords) {
   const map = new Map();
   for (const rec of sessionRecords) {
     if (typeof rec.session_id !== 'string' || !rec.session_id) continue;
     if (rec.agent_summary && typeof rec.agent_summary === 'object') {
-      map.set(rec.session_id, rec.agent_summary);
+      map.set(rec.session_id, { agentSummary: rec.agent_summary, real: isRealSession(rec) });
     }
   }
   return map;
@@ -91,12 +108,13 @@ function buildSessionMap(sessionRecords) {
  *       skill: string,
  *       selections: number,
  *       sessions: string[],
- *       outcomes: { complete: number, partial: number, failed: number, spiral: number, unknown: number }
+ *       outcomes: { complete: number, partial: number, failed: number, spiral: number, unknown: number, abandoned: number }
  *     }
  *   },
  *   totalSelections: number,
  *   sessionsJoined: number,
- *   sessionsUnknown: number
+ *   sessionsUnknown: number,
+ *   sessionsAbandoned: number
  * }>}
  */
 export async function joinSkillOutcomes({
@@ -116,6 +134,7 @@ export async function joinSkillOutcomes({
   let totalSelections = 0;
   let sessionsJoined = 0;
   let sessionsUnknown = 0;
+  let sessionsAbandoned = 0;
 
   for (const inv of invocations) {
     // Only process skill-selection events with a valid skill field
@@ -132,7 +151,7 @@ export async function joinSkillOutcomes({
         skill,
         selectionCount: 0,
         sessions: new Set(),
-        outcomes: { complete: 0, partial: 0, failed: 0, spiral: 0, unknown: 0 },
+        outcomes: { complete: 0, partial: 0, failed: 0, spiral: 0, unknown: 0, abandoned: 0 },
       });
     }
 
@@ -144,14 +163,21 @@ export async function joinSkillOutcomes({
       record.sessions.add(sessionId);
 
       if (isNew) {
-        const summary = sessionMap.get(sessionId);
-        if (summary) {
+        const entry = sessionMap.get(sessionId);
+        if (entry && entry.real) {
+          const summary = entry.agentSummary;
           // Sum session-level aggregate outcomes into this skill's buckets
           record.outcomes.complete += typeof summary.complete === 'number' ? summary.complete : 0;
           record.outcomes.partial += typeof summary.partial === 'number' ? summary.partial : 0;
           record.outcomes.failed += typeof summary.failed === 'number' ? summary.failed : 0;
           record.outcomes.spiral += typeof summary.spiral === 'number' ? summary.spiral : 0;
           sessionsJoined += 1;
+        } else if (entry && !entry.real) {
+          // Found in sessions.jsonl but a phantom abandoned stub (#834) — a
+          // zero-signal join. Route to a distinct bucket instead of inflating
+          // sessionsJoined or conflating with "not found at all" (unknown).
+          record.outcomes.abandoned += 1;
+          sessionsAbandoned += 1;
         } else {
           // Session id not found in sessions.jsonl — count as unknown, never drop
           record.outcomes.unknown += 1;
@@ -177,5 +203,5 @@ export async function joinSkillOutcomes({
     };
   }
 
-  return { bySkill: bySkillObj, totalSelections, sessionsJoined, sessionsUnknown };
+  return { bySkill: bySkillObj, totalSelections, sessionsJoined, sessionsUnknown, sessionsAbandoned };
 }