session-orchestrator 3.17.0 → 3.20.0

package/scripts/lib/reconcile/engine.mjs

@@ -62,7 +62,24 @@
  * @property {number} eligible
  * @property {number} proposed
  * @property {number} rejected
+ * @property {number} capped   - count of eligible learnings that were NOT proposed
+ *        this run purely because of the `maxProposalsPerRun` volume brake (issue
+ *        #900 D — confidence-sorted, lowest-confidence entries cut first). Each
+ *        capped learning is ALSO counted inside `rejected` (with a `capped — ...`
+ *        reason) — `totalLearnings === proposed + rejected` still holds unchanged;
+ *        `capped` is a diagnostic sub-count that lets a report distinguish
+ *        "genuinely ineligible" rejections from "eligible but cut by the volume
+ *        brake" ones at a glance.
  * @property {boolean} written
+ * @property {number} [skipped] - how many persisted sidecar lines the store's
+ *        read-side shape guard rejected and this run therefore DROPPED from disk
+ *        (`mergeCandidates` rewrites the store in full, never appends — see
+ *        `idempotency.mjs`). PRESENT-vs-ABSENT is load-bearing and must not be
+ *        collapsed: `0` means "the store was inspected and nothing was dropped",
+ *        while ABSENCE means "the store was never inspected this run" — the case
+ *        under `dryRun` (merge skipped entirely), on the empty short-circuit, on
+ *        the top-level error path, and when the merge seam reports no count.
+ *        Defaulting the absent case to `0` would be a false all-clear.
  *
  * @typedef {Object} ReconcileResult
  * @property {ReconcileProposal[]} proposals
@@ -74,15 +91,29 @@
 import { readFileSync } from 'node:fs';
 import { isAbsolute, join } from 'node:path';
 
+import { learningKeyOf } from '../learnings/kebab.mjs';
 import { migrateLegacyLearning, normalizeLearning } from '../learnings/schema.mjs';
 import { filterEligible } from './eligibility.mjs';
 import { toActivationMetadata } from './emitter.mjs';
 import { renderRule } from './renderer.mjs';
-import { makeCandidateId, mergeCandidates as realMergeCandidates } from './idempotency.mjs';
+import {
+  DEFAULT_STORE_PATH,
+  buildCandidate,
+  makeCandidateId,
+  mergeCandidates as realMergeCandidates,
+} from './idempotency.mjs';
 
 /** Default repo-relative location of the learnings corpus. */
 const DEFAULT_LEARNINGS_PATH = '.orchestrator/metrics/learnings.jsonl';
 
+/**
+ * Default volume brake (issue #900 D) — mirrors the `reconcile.max-proposals-
+ * per-run` Session Config default in `scripts/lib/config/reconcile.mjs`. Applied
+ * even when a caller omits `maxProposalsPerRun` entirely, so the engine never
+ * silently proposes an unbounded number of rules in one run.
+ */
+const DEFAULT_MAX_PROPOSALS_PER_RUN = 10;
+
 /**
  * Build a fully-zeroed result (the empty / error shape). Touches no disk.
  * @param {string} [error]
@@ -98,6 +129,7 @@ function zeroedResult(error) {
       eligible: 0,
       proposed: 0,
       rejected: 0,
+      capped: 0,
       written: false,
     },
   };
@@ -149,26 +181,19 @@ function defaultLoadLearnings(repoRoot) {
 
 /**
  * Best-effort logical key for a REJECTED learning (rejections never run the
- * emitter, so there is no metadata.learningKey). Mirrors the emitter's key
- * shape `${type}/${kebab(subject||title)}` when both halves are present; falls
- * back to `null` when the type or subject/title is unusable. Never throws.
+ * emitter, so there is no `metadata.learningKey`). Delegates to the shared
+ * `learningKeyOf` — it must NOT merely "mirror" the emitter's shape, it has to
+ * BE it: this key is written to `reconcile-candidates.jsonl` and folded into
+ * `makeCandidateId`, so a learning that succeeds one run and is rejected the
+ * next would otherwise appear in the sidecar under two identities and defeat
+ * `isProcessed`/`mergeCandidates` dedupe. Returns `null` for an unkeyable
+ * record. Never throws.
  *
  * @param {unknown} learning
  * @returns {string|null}
  */
 function rejectedLearningKey(learning) {
-  if (learning === null || typeof learning !== 'object' || Array.isArray(learning)) return null;
-  const rec = /** @type {Record<string, unknown>} */ (learning);
-  const type = typeof rec.type === 'string' && rec.type !== '' ? rec.type : '';
-  const subjectOrTitle =
-    (typeof rec.title === 'string' && rec.title !== '' ? rec.title : '') ||
-    (typeof rec.subject === 'string' && rec.subject !== '' ? rec.subject : '');
-  if (type === '' || subjectOrTitle === '') return null;
-  const kebab = subjectOrTitle
-    .toLowerCase()
-    .replace(/[^a-z0-9]+/g, '-')
-    .replace(/^-+|-+$/g, '');
-  return `${type}/${kebab}`;
+  return learningKeyOf(learning);
 }
 
 /**
@@ -186,33 +211,31 @@ function learningType(learning) {
 }
 
 /**
- * Build a sidecar ReconcileCandidate line-record (idempotency.mjs schema) for a
- * proposed or rejected learning. `created_at` is stamped from the injectable
- * clock so output stays deterministic under test.
+ * Surface a shape-guard drop on stderr. The sidecar is a mutable work-queue that
+ * `mergeCandidates` rewrites in FULL, so a record the read-side shape guard
+ * rejects is not merely ignored — it is gone from disk after this run. The count
+ * alone makes that loss attributable; this WARN is what makes it VISIBLE, since
+ * the summary field only helps a caller that thinks to read it.
  *
- * @param {Object} params
- * @param {string} params.id
- * @param {string|null} params.learningKey
- * @param {string} params.slug
- * @param {'proposed'|'rejected'} params.status
- * @param {string} params.reason
- * @param {number} params.confidence
- * @param {string} params.createdAt - ISO timestamp.
- * @returns {import('./idempotency.mjs').ReconcileCandidate}
+ * Never throws: a failing diagnostic must not break the never-throws contract of
+ * {@link runReconcile} (a broken stderr pipe would otherwise zero the result).
+ *
+ * @param {number} skipped - drop count (> 0 by the time this is called).
+ * @returns {void}
  */
-function buildCandidate({ id, learningKey, slug, status, reason, confidence, createdAt }) {
-  return {
-    id,
-    schema_version: 1,
-    learning_key: typeof learningKey === 'string' ? learningKey : '',
-    slug,
-    status,
-    reason,
-    confidence,
-    created_at: createdAt,
-    processed_at: null,
-    superseded_by: null,
-  };
+function warnDroppedStoreRecords(skipped) {
+  try {
+    console.warn(
+      `⚠️  reconcile: ${skipped} record(s) in ${DEFAULT_STORE_PATH} failed the ` +
+        `candidate shape guard and were DROPPED by this merge — the store is ` +
+        `rewritten in full, so they are no longer on disk. Expected shape: a ` +
+        `ReconcileCandidate with \`learning_key\` + \`created_at\` (see ` +
+        `scripts/lib/reconcile/idempotency.mjs). Only \`mergeCandidates\` may ` +
+        `write this store; hand-written or report records do not belong in it.`,
+    );
+  } catch {
+    // A diagnostic must never become the failure it reports on.
+  }
 }
 
 /**
@@ -236,6 +259,11 @@ function buildCandidate({ id, learningKey, slug, status, reason, confidence, cre
  *        eligibility placeholder-insight check (forwarded to `filterEligible`). Inert
  *        (no additional rejections) when omitted.
  * @param {number|Date} [params.now]      - injectable clock (emitter fallback + candidate `created_at`).
+ * @param {number} [params.maxProposalsPerRun] - volume brake (issue #900 D): after
+ *        sorting eligible learnings by confidence DESC, only the top N are proposed;
+ *        the rest are recorded as `capped` rejections. Defaults to
+ *        {@link DEFAULT_MAX_PROPOSALS_PER_RUN} (10) when omitted, non-finite, or < 1
+ *        — the brake is ALWAYS active, matching the Session Config default.
  * @param {boolean} [params.dryRun]       - when true, compute proposals but SKIP the merge entirely
  *        (also accepted as `opts.dryRun`; either location sets it).
  * @param {Object} [opts]                 - DI seams (all default to real behaviour).
@@ -249,7 +277,15 @@ function buildCandidate({ id, learningKey, slug, status, reason, confidence, cre
  * @returns {Promise<ReconcileResult>}
  */
 export async function runReconcile(
-  { repoRoot, ruleExpiryDays, minRuleDays, minInsightChars, now, dryRun: dryRunParam } = {},
+  {
+    repoRoot,
+    ruleExpiryDays,
+    minRuleDays,
+    minInsightChars,
+    now,
+    maxProposalsPerRun: maxProposalsPerRunParam,
+    dryRun: dryRunParam,
+  } = {},
   opts = {},
 ) {
   try {
@@ -258,6 +294,12 @@ export async function runReconcile(
     // (the documented DI seam) — either location flips it on.
     const dryRun = dryRunParam === true || opts.dryRun === true;
     const merge = typeof opts.merge === 'function' ? opts.merge : realMergeCandidates;
+    // Volume brake (#900 D) — always active; a missing/invalid override falls
+    // back to the same default the Session Config parser uses.
+    const maxProposalsPerRun =
+      Number.isFinite(maxProposalsPerRunParam) && maxProposalsPerRunParam >= 1
+        ? Math.floor(maxProposalsPerRunParam)
+        : DEFAULT_MAX_PROPOSALS_PER_RUN;
 
     // --- Pipeline step 1 — load learnings ----------------------------------
     /** @type {Array<Record<string, unknown>>} */
@@ -289,6 +331,21 @@ export async function runReconcile(
       minInsightChars,
     });
 
+    // --- Pipeline step 3b — volume brake (#900 D) ---------------------------
+    // Sort eligible learnings by confidence DESC (ties keep their original,
+    // stable relative order) and keep only the top `maxProposalsPerRun`. The
+    // rest are cut BEFORE they ever reach the emitter — never proposed this
+    // run — and recorded as `capped` rejections in step 4b below so a report
+    // stays honest about the cut instead of silently dropping them.
+    const confidenceOf = (l) =>
+      l && typeof l === 'object' && typeof l.confidence === 'number' ? l.confidence : 0;
+    const sortedEligible = eligible
+      .map((learning, index) => ({ learning, index }))
+      .sort((a, b) => confidenceOf(b.learning) - confidenceOf(a.learning) || a.index - b.index)
+      .map(({ learning }) => learning);
+    const keptEligible = sortedEligible.slice(0, maxProposalsPerRun);
+    const cappedEligible = sortedEligible.slice(maxProposalsPerRun);
+
     /** @type {ReconcileProposal[]} */
     const proposals = [];
     /** @type {ReconcileRejection[]} */
@@ -297,7 +354,7 @@ export async function runReconcile(
     const candidates = [];
 
     // --- Pipeline step 4 — per eligible learning (wrapped per-item) ---------
-    for (const learning of eligible) {
+    for (const learning of keptEligible) {
       try {
         const metadata = toActivationMetadata(learning, { ruleExpiryDays, now, minRuleDays });
         const { slug, path, content } = renderRule(learning, metadata);
@@ -352,6 +409,34 @@ export async function runReconcile(
       }
     }
 
+    // --- Pipeline step 4b — capped-eligible learnings (#900 D) --------------
+    // Learnings that passed eligibility but were cut by the volume brake are
+    // NEVER passed to the emitter — recorded directly as rejections (with a
+    // `capped — ...` reason) so a report distinguishes this from a genuine
+    // ineligibility rejection.
+    for (const learning of cappedEligible) {
+      const learningKey = rejectedLearningKey(learning);
+      const type = learningType(learning);
+      const reason = `capped — max-proposals-per-run (${maxProposalsPerRun}) reached; ${cappedEligible.length} lower-confidence eligible learning(s) not proposed this run`;
+      rejected.push({
+        learningKey,
+        type,
+        reason,
+        status: 'rejected',
+      });
+      candidates.push(
+        buildCandidate({
+          id: makeCandidateId(learningKey ?? '', `rejected-${type}`),
+          learningKey,
+          slug: '',
+          status: 'rejected',
+          reason,
+          confidence: confidenceOf(learning),
+          createdAt,
+        }),
+      );
+    }
+
     // --- Pipeline step 5 — per rejected learning (eligibility rejects) ------
     for (const { learning, reason } of rejectedLearnings) {
       const learningKey = rejectedLearningKey(learning);
@@ -382,28 +467,45 @@ export async function runReconcile(
     // The engine's ONLY disk write — and it is skipped entirely under dryRun.
     // It never writes `.claude/rules/` (FA3 / #696 owns that, post-approval).
     let written = false;
+    // `undefined` (NOT 0) until the merge actually inspects the store — absence
+    // means "not checked", 0 means "checked, nothing dropped". See the
+    // ReconcileSummary `skipped` typedef.
+    /** @type {number|undefined} */
+    let skipped;
     if (!dryRun) {
       try {
         const mergeResult = merge({ candidates, repoRoot });
         written = !!(mergeResult && mergeResult.written === true);
+        // Only a finite count from the seam counts as "inspected". A merge seam
+        // that reports nothing leaves `skipped` absent rather than fabricating 0.
+        if (mergeResult && Number.isFinite(mergeResult.skipped)) {
+          skipped = Number(mergeResult.skipped);
+        }
       } catch {
-        // Merge failure is non-fatal; proposals still returned, written stays false.
+        // Merge failure is non-fatal; proposals still returned, written stays
+        // false and `skipped` stays absent (the store was never inspected).
         written = false;
       }
     }
 
+    // Make an attributable drop VISIBLE, not merely recorded (WARN, never throw).
+    if (typeof skipped === 'number' && skipped > 0) warnDroppedStoreRecords(skipped);
+
     // --- Pipeline step 7 — summary -----------------------------------------
-    return {
-      proposals,
-      rejected,
-      summary: {
-        totalLearnings,
-        eligible: eligible.length,
-        proposed: proposals.length,
-        rejected: rejected.length,
-        written,
-      },
+    /** @type {ReconcileSummary} */
+    const summary = {
+      totalLearnings,
+      eligible: eligible.length,
+      proposed: proposals.length,
+      rejected: rejected.length,
+      capped: cappedEligible.length,
+      written,
     };
+    // Additive + absence-preserving: the key exists ONLY when the store was
+    // actually inspected, so no consumer can read a false `skipped: 0`.
+    if (typeof skipped === 'number') summary.skipped = skipped;
+
+    return { proposals, rejected, summary };
   } catch (err) {
     // never-throws top-level guard.
     const msg = err && err.message ? err.message : String(err);

package/scripts/lib/reconcile/idempotency.mjs

@@ -10,7 +10,13 @@
  * lines — an append would leave stale duplicates.
  *
  * Store: `.orchestrator/runtime/reconcile-candidates.jsonl` — a mutable
- * work-queue in JSON-Lines format (one ReconcileCandidate per line).
+ * work-queue in JSON-Lines format (one ReconcileCandidate per line). The store
+ * is OWNED by `mergeCandidates`: it is the only sanctioned writer. Nothing else
+ * — no report, no analysis run, no agent — may append to it; a read-side shape
+ * guard drops any record that is not a ReconcileCandidate and COUNTS the drop.
+ * The count reaches readers through the ONE reader ({@link loadCandidates}
+ * returns `{records, skipped}`) and writers through `mergeCandidates`'s
+ * `skipped` — there is deliberately no lossy array-only variant beside them.
  *
  * Two responsibilities differ from the repair store:
  *   1. The IDEMPOTENCY KEY is the LOGICAL `learning_key` (issue #695), not the
@@ -66,12 +72,40 @@ function resolveStorePath(repoRoot, storePath) {
   return join(root, rel);
 }
 
+/**
+ * Minimal shape guard for a persisted store line. A record is accepted only
+ * when it carries the two fields every consumer of this store depends on:
+ *   - `learning_key` — THE logical dedupe key (`mergeCandidates`, `isProcessed`).
+ *   - `created_at`   — the recency axis (`reconcile-nudge-banner.mjs` `_lastRunAt`).
+ *
+ * This is deliberately NOT a full schema check: the store is a mutable
+ * work-queue whose records may gain fields across schema versions, so
+ * over-strict validation would silently drop legitimate future records. It
+ * rejects only records that no writer in this repo produces — the concrete
+ * incident being a hand-written report artefact using `candidate_id` /
+ * `generated_at` / `status:"candidate"` (2026-07-31, see
+ * `docs/reconcile/2026-07-31-reconcile-candidates.md`).
+ * @param {unknown} rec
+ * @returns {boolean}
+ */
+function isCandidateShape(rec) {
+  if (!rec || typeof rec !== 'object' || Array.isArray(rec)) return false;
+  const r = /** @type {Record<string, unknown>} */ (rec);
+  if (typeof r.learning_key !== 'string' || r.learning_key.length === 0) return false;
+  if (typeof r.created_at !== 'string') return false;
+  return true;
+}
+
 /**
  * Read + defensively parse the store's JSONL lines into ReconcileCandidate
- * records. Malformed lines (bad JSON, non-object) are skipped silently. A
- * missing file yields `[]`. Never throws.
+ * records. Malformed lines (bad JSON, non-object) and shape-foreign records
+ * (see {@link isCandidateShape}) are skipped — the latter are COUNTED, because
+ * the store is a mutable work-queue that `mergeCandidates` rewrites in full, so
+ * a skipped line is dropped from disk on the next merge and a silent drop would
+ * be unattributable data loss. A missing file yields `{ records: [], skipped: 0 }`.
+ * Never throws.
  * @param {string} absPath
- * @returns {ReconcileCandidate[]}
+ * @returns {{ records: ReconcileCandidate[], skipped: number }}
  */
 function readStore(absPath) {
   let raw;
@@ -79,11 +113,12 @@ function readStore(absPath) {
     raw = readFileSync(absPath, 'utf8');
   } catch {
     // ENOENT or any read error → empty store.
-    return [];
+    return { records: [], skipped: 0 };
   }
 
   /** @type {ReconcileCandidate[]} */
   const records = [];
+  let skipped = 0;
   for (const line of raw.split('\n')) {
     const trimmed = line.trim();
     if (trimmed.length === 0) continue;
@@ -91,13 +126,16 @@ function readStore(absPath) {
     try {
       parsed = JSON.parse(trimmed);
     } catch {
+      skipped += 1;
       continue; // skip malformed line
     }
-    if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+    if (isCandidateShape(parsed)) {
       records.push(/** @type {ReconcileCandidate} */ (parsed));
+    } else {
+      skipped += 1;
     }
   }
-  return records;
+  return { records, skipped };
 }
 
 /**
@@ -153,13 +191,70 @@ export function makeCandidateId(learningKey, slug) {
 }
 
 /**
- * Load every ReconcileCandidate currently persisted in the store. Reads JSONL,
- * skips malformed lines, returns `[]` for a missing file. Does NOT create the
+ * Build a ReconcileCandidate line-record for a proposed or rejected learning.
+ *
+ * Lives here — beside the {@link ReconcileCandidate} typedef it instantiates and
+ * the read-side `isCandidateShape` guard that judges it — so ONE file decides
+ * which fields a persisted record carries. It deliberately does NOT mint the
+ * `id` (see {@link makeCandidateId}): rejections currently derive their id from
+ * `(learningKey, 'rejected-<type>')` while storing `slug: ''`, so folding the
+ * mint in here using the record's own slug would change every rejection
+ * candidate's id and orphan every rejection row already on disk. Caller-supplied
+ * `id` keeps that decision at the call site.
+ *
+ * `created_at` is caller-supplied (from the engine's injectable clock) so output
+ * stays deterministic under test. `processed_at`/`superseded_by` always start
+ * null — only the merge/approval path stamps them. Never throws.
+ *
+ * @param {Object} params
+ * @param {string} params.id - deterministic physical id, see {@link makeCandidateId}.
+ * @param {string|null} params.learningKey - logical dedupe key; a non-string coerces to `''`.
+ * @param {string} params.slug - `.claude/rules/<slug>.md` slug (`''` for rejections).
+ * @param {'proposed'|'rejected'} params.status
+ * @param {string} params.reason
+ * @param {number} params.confidence
+ * @param {string} params.createdAt - ISO timestamp.
+ * @returns {ReconcileCandidate}
+ */
+export function buildCandidate({ id, learningKey, slug, status, reason, confidence, createdAt }) {
+  return {
+    id,
+    schema_version: 1,
+    learning_key: typeof learningKey === 'string' ? learningKey : '',
+    slug,
+    status,
+    reason,
+    confidence,
+    created_at: createdAt,
+    processed_at: null,
+    superseded_by: null,
+  };
+}
+
+/**
+ * Load the persisted store: every ReconcileCandidate that survives the read-side
+ * shape guard, PLUS the count of lines it rejected. Reads JSONL, skips malformed
+ * lines and shape-foreign records (missing `learning_key` / `created_at`),
+ * yields `{ records: [], skipped: 0 }` for a missing file. Does NOT create the
  * runtime dir (mkdir -p happens only on write). Never throws.
+ *
+ * `skipped` is part of the return value rather than a second, "diagnostics"
+ * reader beside this one, because `records.length === 0` is AMBIGUOUS on its
+ * own: a missing store and a store whose every line is shape-foreign both yield
+ * `[]`, and a consumer that sees only the array reports "no reconcile run on
+ * record" for a store that in fact holds quarantined evidence of one (the
+ * concrete defect in `scripts/lib/reconcile-nudge-banner.mjs`, GitLab #955
+ * finding 2). Splitting the honest reader off under a longer name left the
+ * OBVIOUS name as the lossy one — the next consumer would reach for
+ * `loadCandidates`, get `[]`, and re-derive the same wrong conclusion. One
+ * reader, one answer.
+ *
+ * Read-only: unlike {@link mergeCandidates} this does NOT rewrite the store, so
+ * the skipped lines are still on disk after this call.
  * @param {Object} [params]
  * @param {string} [params.repoRoot] - repo root; relative `storePath` is resolved against it (defaults to `process.cwd()`).
  * @param {string} [params.storePath] - store path (relative ⇒ joined to repoRoot). Defaults to {@link DEFAULT_STORE_PATH}.
- * @returns {ReconcileCandidate[]}
+ * @returns {{ records: ReconcileCandidate[], skipped: number }}
  */
 export function loadCandidates({ repoRoot, storePath } = {}) {
   const absPath = resolveStorePath(repoRoot, storePath);
@@ -198,16 +293,21 @@ export function isProcessed(candidate, existing) {
  * The runtime dir is created with mkdir -p semantics. The store is rewritten in
  * full (read-all → merge → atomic tmp+rename), never appended. Output lines are
  * sorted by `learning_key` for deterministic output. Never throws; on write
- * failure returns `{ merged, written: false }`.
+ * failure returns `written: false`.
+ *
+ * `skipped` reports how many persisted lines the read-side shape guard rejected
+ * (malformed JSON, or a record missing `learning_key`/`created_at`). Because the
+ * store is rewritten in full, those lines are DROPPED from disk by this call —
+ * the count is what makes that loss attributable instead of silent.
  * @param {Object} [params]
  * @param {ReconcileCandidate[]} [params.candidates] - newly minted candidates to merge.
  * @param {string} [params.repoRoot]
  * @param {string} [params.storePath]
- * @returns {{ merged: ReconcileCandidate[], written: boolean }}
+ * @returns {{ merged: ReconcileCandidate[], written: boolean, skipped: number }}
  */
 export function mergeCandidates({ candidates, repoRoot, storePath } = {}) {
   const absPath = resolveStorePath(repoRoot, storePath);
-  const store = readStore(absPath);
+  const { records: store, skipped } = readStore(absPath);
 
   // Index existing records by learning_key for O(1) lookup. Last write wins for
   // any pre-existing duplicates in the file (defensive — store should be unique).
@@ -235,5 +335,5 @@ export function mergeCandidates({ candidates, repoRoot, storePath } = {}) {
   );
 
   const result = writeStore(absPath, merged);
-  return { merged, written: result.ok === true };
+  return { merged, written: result.ok === true, skipped };
 }