session-orchestrator 3.22.0 → 3.23.0

package/scripts/lib/reconcile/engine.mjs

@@ -70,6 +70,16 @@
  *        `capped` is a diagnostic sub-count that lets a report distinguish
  *        "genuinely ineligible" rejections from "eligible but cut by the volume
  *        brake" ones at a glance.
+ * @property {number} alreadyMaterialized - count of eligible learnings that were
+ *        NOT proposed this run because they are already terminal — either the
+ *        idempotency sidecar already carries a `processed_at` stamp for their
+ *        `learning_key` (`isProcessed`), or a `.claude/rules/*.md` file already
+ *        carries a matching `learning-key`/`learning-id` provenance marker
+ *        (issue #484: 9 of 10 proposals in one run were exactly this). Runs
+ *        BEFORE the `maxProposalsPerRun` volume brake so an already-materialized
+ *        learning never consumes a new learning's quota. Same accounting pattern
+ *        as `capped`: each one is ALSO counted inside `rejected`, and
+ *        `totalLearnings === proposed + rejected` still holds unchanged.
  * @property {boolean} written
  * @property {number} [skipped] - how many persisted sidecar lines the store's
  *        read-side shape guard rejected and this run therefore DROPPED from disk
@@ -88,9 +98,10 @@
  * @property {string} [error]  - present only when the never-throws top-level guard fired.
  */
 
-import { readFileSync } from 'node:fs';
+import { readFileSync, readdirSync } from 'node:fs';
 import { isAbsolute, join } from 'node:path';
 
+import { expandTilde } from '../common.mjs';
 import { learningKeyOf } from '../learnings/kebab.mjs';
 import { migrateLegacyLearning, normalizeLearning } from '../learnings/schema.mjs';
 import { filterEligible } from './eligibility.mjs';
@@ -99,7 +110,9 @@ import { renderRule } from './renderer.mjs';
 import {
   DEFAULT_STORE_PATH,
   buildCandidate,
+  isProcessed,
   makeCandidateId,
+  loadCandidates as realLoadCandidates,
   mergeCandidates as realMergeCandidates,
 } from './idempotency.mjs';
 
@@ -130,6 +143,7 @@ function zeroedResult(error) {
       proposed: 0,
       rejected: 0,
       capped: 0,
+      alreadyMaterialized: 0,
       written: false,
     },
   };
@@ -238,6 +252,172 @@ function warnDroppedStoreRecords(skipped) {
   }
 }
 
+/**
+ * Session Config placeholder convention: a path key whose committed value is a
+ * marker that MUST be overridden host-locally (this repo ships
+ * `plan-baseline-path: OVERRIDE-IN-owner.yaml` in CLAUDE.md — or AGENTS.md on
+ * Codex CLI — § Session Config).
+ * Treating the marker as a real path would create `./OVERRIDE-IN-owner.yaml/`.
+ */
+const PLACEHOLDER_PATH_RE = /^OVERRIDE-IN-/;
+
+/**
+ * Report that the `baseline` write-target was dropped for this run.
+ *
+ * stderr, never stdout — `scripts/lib/config.mjs` consumers parse stdout as JSON.
+ * Same posture as {@link warnDroppedStoreRecords}: an attributable drop must be
+ * VISIBLE, and a diagnostic must never become the failure it reports on.
+ *
+ * @param {string} reason
+ */
+function warnBaselineTargetDropped(reason) {
+  try {
+    console.warn(
+      `⚠️  reconcile: target "baseline" DROPPED for this run — ${reason}. ` +
+        `Resolution order is SO_BASELINE_PATH env > owner.yaml paths.baseline-path > ` +
+        `committed plan-baseline-path. No baseline proposal is surfaced in the approval ` +
+        `AUQ and nothing is written outside this repo; every other target is unaffected.`,
+    );
+  } catch {
+    // A diagnostic must never become the failure it reports on.
+  }
+}
+
+/**
+ * Decide which reconcile targets can ACTUALLY be written this run (issue #1099).
+ *
+ * Two of the three no-op checks live here, upstream of both the approval AUQ and
+ * the writer, because the important half is not the refused write — it is that
+ * **the operator must never be asked to approve a write to a destination that
+ * cannot exist.** A `baseline` target whose root is unresolvable, is still the
+ * committed placeholder, or is not absolute is therefore dropped from the
+ * effective list BEFORE proposals are surfaced. The third check (the root does
+ * not exist on disk) belongs to `writer.mjs`, the only layer holding the
+ * filesystem at write time.
+ *
+ * Pure and never-throws: it resolves nothing from disk and reads no env — the
+ * caller passes the already-resolved `baselineRoot` (`config['plan-baseline-path']`,
+ * which `scripts/lib/config.mjs` has already run through the full 3-tier chain).
+ *
+ * @param {{targets?: unknown, baselineRoot?: unknown}} [opts]
+ * @returns {{targets: string[], baselineRoot: string|null, dropped: string[], reason: string|null}}
+ *   `targets` — the effective list; `baselineRoot` — the ~-expanded root, or null
+ *   when `baseline` is not in play; `dropped`/`reason` — the audit trail.
+ */
+export function resolveEffectiveTargets({ targets, baselineRoot } = {}) {
+  const declared = Array.isArray(targets) ? targets.filter((t) => typeof t === 'string' && t.length > 0) : [];
+  const list = [...new Set(declared.length > 0 ? declared : ['repo-local'])];
+
+  if (!list.includes('baseline')) {
+    return { targets: list, baselineRoot: null, dropped: [], reason: null };
+  }
+
+  const raw = typeof baselineRoot === 'string' ? baselineRoot.trim() : '';
+  const expanded = raw === '' ? '' : expandTilde(raw);
+
+  let reason = null;
+  if (raw === '') {
+    reason = 'no baseline path is configured on any tier';
+  } else if (PLACEHOLDER_PATH_RE.test(raw)) {
+    reason = `the committed placeholder "${raw}" was never overridden host-locally`;
+  } else if (!isAbsolute(expanded)) {
+    reason = `"${raw}" is not an absolute path after ~-expansion`;
+  }
+
+  if (reason === null) {
+    return { targets: list, baselineRoot: expanded, dropped: [], reason: null };
+  }
+
+  warnBaselineTargetDropped(reason);
+  return {
+    targets: list.filter((t) => t !== 'baseline'),
+    baselineRoot: null,
+    dropped: ['baseline'],
+    reason,
+  };
+}
+
+/**
+ * Default sidecar-candidate loader for the issue #484 idempotency dedupe
+ * check (below, step 3a). Deliberately gated on `repoRoot` being a
+ * caller-supplied, non-empty string — UNLIKE `defaultLoadLearnings` and
+ * `realMergeCandidates`, this does NOT fall back to `process.cwd()` when
+ * `repoRoot` is absent. Every existing engine test exercises this module via
+ * `opts.learnings` with no `repoRoot`, precisely to avoid touching this repo's
+ * OWN `.orchestrator/runtime/reconcile-candidates.jsonl`; a cwd fallback here
+ * would silently read it. The disk-touching default is reserved for callers
+ * that always pass an explicit `repoRoot` (the `/reconcile` skill resolves it
+ * via `git rev-parse --show-toplevel`).
+ * @param {string|undefined} repoRoot
+ * @returns {{ records: import('./idempotency.mjs').ReconcileCandidate[] }}
+ */
+function defaultLoadCandidatesForDedupe(repoRoot) {
+  if (typeof repoRoot !== 'string' || repoRoot.length === 0) return { records: [] };
+  const { records } = realLoadCandidates({ repoRoot });
+  return { records };
+}
+
+/** Frontmatter form emitted by renderer.mjs: `learning-key: <value>` (no backticks, no leading dash). */
+const FRONTMATTER_LEARNING_KEY_RE = /^learning-key:\s*(.+)$/gm;
+/** Provenance-body form emitted by renderer.mjs: `` - learning-key: `<value>` ``. */
+const BODY_LEARNING_KEY_RE = /-\s*learning-key:\s*`([^`]+)`/g;
+/** Provenance-body form emitted by renderer.mjs: `` - learning-id: `<value>` ``. */
+const BODY_LEARNING_ID_RE = /-\s*learning-id:\s*`([^`]+)`/g;
+
+/**
+ * Scan `<repoRoot>/.claude/rules/*.md` for the provenance markers the
+ * renderer stamps on every machine-generated rule — the frontmatter
+ * `learning-key:` line and the body `## Provenance` block's `learning-key`/
+ * `learning-id` bullets (`renderer.mjs`) — and return the two identity sets a
+ * learning can already be materialized under. A learning whose derived
+ * `learning_key` OR raw `.id` appears in either set already has a rule file
+ * on disk: re-proposing it is the issue #484 defect (9 of 10 proposals in one
+ * run were learnings a `.claude/rules/` file already covered).
+ *
+ * Gated the same way as {@link defaultLoadCandidatesForDedupe}: an absent
+ * `repoRoot` yields empty sets rather than falling back to `process.cwd()`.
+ * Never throws — a missing `.claude/rules/` dir or an unreadable file
+ * degrades to "nothing materialized" for that source, never a crash.
+ * @param {string|undefined} repoRoot
+ * @returns {{ keys: Set<string>, ids: Set<string> }}
+ */
+function defaultReadMaterializedProvenance(repoRoot) {
+  const keys = new Set();
+  const ids = new Set();
+  if (typeof repoRoot !== 'string' || repoRoot.length === 0) return { keys, ids };
+
+  const rulesDir = join(repoRoot, '.claude', 'rules');
+  let entries;
+  try {
+    entries = readdirSync(rulesDir);
+  } catch {
+    return { keys, ids }; // no rules dir yet → nothing materialized
+  }
+
+  for (const entry of entries) {
+    if (!entry.endsWith('.md')) continue;
+    let content;
+    try {
+      content = readFileSync(join(rulesDir, entry), 'utf8');
+    } catch {
+      continue; // unreadable file — skip it, do not fail the whole scan
+    }
+    for (const m of content.matchAll(FRONTMATTER_LEARNING_KEY_RE)) {
+      const v = m[1].trim();
+      if (v) keys.add(v);
+    }
+    for (const m of content.matchAll(BODY_LEARNING_KEY_RE)) {
+      const v = m[1].trim();
+      if (v) keys.add(v);
+    }
+    for (const m of content.matchAll(BODY_LEARNING_ID_RE)) {
+      const v = m[1].trim();
+      if (v && v !== 'n/a') ids.add(v);
+    }
+  }
+  return { keys, ids };
+}
+
 /**
  * Run the reconciliation engine.
  *
@@ -273,6 +453,13 @@ function warnDroppedStoreRecords(skipped) {
  *        direct learnings injection (takes precedence over `loadLearnings`).
  * @param {typeof realMergeCandidates} [opts.merge]
  *        override the sidecar merge (so tests never touch real `.orchestrator/runtime/`).
+ * @param {(repoRoot?: string) => { records: import('./idempotency.mjs').ReconcileCandidate[] }} [opts.loadCandidates]
+ *        override the idempotency-sidecar read used by the issue #484 dedupe step (below,
+ *        step 3a) — defaults to {@link defaultLoadCandidatesForDedupe} (repoRoot-gated, no
+ *        cwd fallback; see that function's doc for why).
+ * @param {(repoRoot?: string) => { keys: Set<string>, ids: Set<string> }} [opts.readMaterializedProvenance]
+ *        override the `.claude/rules/` provenance scan used by the same dedupe step —
+ *        defaults to {@link defaultReadMaterializedProvenance} (same repoRoot gate).
  * @param {boolean} [opts.dryRun]         - when true, compute proposals but SKIP the merge entirely.
  * @returns {Promise<ReconcileResult>}
  */
@@ -294,6 +481,12 @@ export async function runReconcile(
     // (the documented DI seam) — either location flips it on.
     const dryRun = dryRunParam === true || opts.dryRun === true;
     const merge = typeof opts.merge === 'function' ? opts.merge : realMergeCandidates;
+    const loadCandidatesForDedupe =
+      typeof opts.loadCandidates === 'function' ? opts.loadCandidates : defaultLoadCandidatesForDedupe;
+    const readMaterializedProvenance =
+      typeof opts.readMaterializedProvenance === 'function'
+        ? opts.readMaterializedProvenance
+        : defaultReadMaterializedProvenance;
     // Volume brake (#900 D) — always active; a missing/invalid override falls
     // back to the same default the Session Config parser uses.
     const maxProposalsPerRun =
@@ -331,20 +524,8 @@ export async function runReconcile(
       minInsightChars,
     });
 
-    // --- Pipeline step 3b — volume brake (#900 D) ---------------------------
-    // Sort eligible learnings by confidence DESC (ties keep their original,
-    // stable relative order) and keep only the top `maxProposalsPerRun`. The
-    // rest are cut BEFORE they ever reach the emitter — never proposed this
-    // run — and recorded as `capped` rejections in step 4b below so a report
-    // stays honest about the cut instead of silently dropping them.
     const confidenceOf = (l) =>
       l && typeof l === 'object' && typeof l.confidence === 'number' ? l.confidence : 0;
-    const sortedEligible = eligible
-      .map((learning, index) => ({ learning, index }))
-      .sort((a, b) => confidenceOf(b.learning) - confidenceOf(a.learning) || a.index - b.index)
-      .map(({ learning }) => learning);
-    const keptEligible = sortedEligible.slice(0, maxProposalsPerRun);
-    const cappedEligible = sortedEligible.slice(maxProposalsPerRun);
 
     /** @type {ReconcileProposal[]} */
     const proposals = [];
@@ -353,6 +534,92 @@ export async function runReconcile(
     /** @type {import('./idempotency.mjs').ReconcileCandidate[]} */
     const candidates = [];
 
+    // --- Pipeline step 3a — idempotency + on-disk dedupe (issue #484) ------
+    // Runs BEFORE the volume brake (step 3b) so an already-materialized
+    // learning does not consume `maxProposalsPerRun` quota that a genuinely
+    // new learning could use — the #484 defect measured on a real repo was
+    // exactly this: 9 of 10 proposals in one run were learnings that already
+    // had a `.claude/rules/` file on disk, crowding out the tenth new one.
+    // Two independent sources both count as terminal, either is sufficient:
+    //   - the idempotency sidecar already carries a `processed_at` stamp for
+    //     this `learning_key` (`isProcessed`, previously computed but NEVER
+    //     called from this module — the other half of #484);
+    //   - a `.claude/rules/*.md` file already carries a matching
+    //     `learning-key`/`learning-id` provenance marker, discovered by
+    //     scanning disk directly (covers the case where a rule was written
+    //     without ever going through this sidecar, e.g. hand-authored).
+    const { records: existingCandidates } = loadCandidatesForDedupe(repoRoot) ?? { records: [] };
+    const materialized = readMaterializedProvenance(repoRoot) ?? { keys: new Set(), ids: new Set() };
+
+    /** @type {Array<Record<string, unknown>>} */
+    const stillEligible = [];
+    let alreadyMaterialized = 0;
+
+    for (const learning of eligible) {
+      const learningKey = rejectedLearningKey(learning);
+      const learningId =
+        learning &&
+        typeof learning === 'object' &&
+        typeof learning.id === 'string' &&
+        learning.id.length > 0
+          ? learning.id
+          : null;
+
+      const sidecarTerminal =
+        learningKey !== null && isProcessed({ learning_key: learningKey }, existingCandidates);
+      const onDisk =
+        (learningKey !== null && materialized.keys.has(learningKey)) ||
+        (learningId !== null && materialized.ids.has(learningId));
+
+      if (!sidecarTerminal && !onDisk) {
+        stillEligible.push(learning);
+        continue;
+      }
+
+      alreadyMaterialized += 1;
+      const type = learningType(learning);
+      const reason = sidecarTerminal
+        ? 'already processed — the idempotency sidecar already carries a terminal verdict for this learning-key'
+        : 'already materialized — a .claude/rules/ file already carries this learning-key/learning-id';
+      rejected.push({ learningKey, type, reason, status: 'rejected' });
+
+      // Freshly discovered on-disk materialization (not yet reflected in the
+      // sidecar): stamp it terminal now so a FUTURE run's `isProcessed()`
+      // check catches it without re-scanning `.claude/rules/` every time.
+      // Routed through the SAME `candidates` array + `merge()` call as every
+      // other record produced this run (step 6 below), so — like everything
+      // else here — it is skipped entirely under `dryRun`; this is not a
+      // second write path.
+      if (onDisk && !sidecarTerminal && learningKey !== null) {
+        const stamped = buildCandidate({
+          id: makeCandidateId(learningKey, `materialized-${type}`),
+          learningKey,
+          slug: '',
+          status: 'proposed',
+          reason,
+          confidence: confidenceOf(learning),
+          createdAt,
+        });
+        stamped.processed_at = createdAt;
+        stamped.outcome = 'already-on-disk';
+        candidates.push(stamped);
+      }
+    }
+
+    // --- Pipeline step 3b — volume brake (#900 D) ---------------------------
+    // Sort STILL-eligible learnings (post-dedupe) by confidence DESC (ties
+    // keep their original, stable relative order) and keep only the top
+    // `maxProposalsPerRun`. The rest are cut BEFORE they ever reach the
+    // emitter — never proposed this run — and recorded as `capped`
+    // rejections in step 4b below so a report stays honest about the cut
+    // instead of silently dropping them.
+    const sortedEligible = stillEligible
+      .map((learning, index) => ({ learning, index }))
+      .sort((a, b) => confidenceOf(b.learning) - confidenceOf(a.learning) || a.index - b.index)
+      .map(({ learning }) => learning);
+    const keptEligible = sortedEligible.slice(0, maxProposalsPerRun);
+    const cappedEligible = sortedEligible.slice(maxProposalsPerRun);
+
     // --- Pipeline step 4 — per eligible learning (wrapped per-item) ---------
     for (const learning of keptEligible) {
       try {
@@ -499,6 +766,7 @@ export async function runReconcile(
       proposed: proposals.length,
       rejected: rejected.length,
       capped: cappedEligible.length,
+      alreadyMaterialized,
       written,
     };
     // Additive + absence-preserving: the key exists ONLY when the store was

package/scripts/lib/reconcile/idempotency.mjs

@@ -44,6 +44,17 @@
  * @property {string} created_at      - ISO timestamp.
  * @property {string|null} processed_at  - terminal stamp (mirrors repair store).
  * @property {string|null} superseded_by
+ * @property {string|null} [outcome]  - terminal disposition, one of:
+ *        `'written'` (writer.mjs persisted the rule file), `'already-on-disk'`
+ *        (engine.mjs found a matching `.claude/rules/` provenance block before
+ *        proposing — issue #484), or `'rejected'` (the operator declined the
+ *        proposal in the approval AUQ and writer.mjs archived it — issue #1042).
+ *        Optional and additive: `isCandidateShape` only requires `learning_key`
+ *        + `created_at`, so an older record without this field still round-trips
+ *        through the store unchanged, and nothing BRANCHES on the value —
+ *        terminality is `processed_at` alone (see {@link isProcessed}), so a
+ *        pre-#1042 record with no `outcome` keeps behaving exactly as before
+ *        (absence reads as `'written'`, the only outcome that existed then).
  */
 
 import { mkdirSync, readFileSync, writeFileSync, renameSync } from 'node:fs';
@@ -265,7 +276,15 @@ export function loadCandidates({ repoRoot, storePath } = {}) {
  * True iff `existing` already holds a candidate that shares `candidate`'s
  * `learning_key` AND has a terminal `processed_at` stamp. The reconcile engine
  * uses this to idempotently SKIP re-proposing a learning whose verdict is
- * already terminal. Never throws.
+ * already terminal.
+ *
+ * Deliberately OUTCOME-AGNOSTIC: `processed_at` alone decides. A rule the
+ * operator DECLINED (`outcome: 'rejected'`, issue #1042) is therefore just as
+ * terminal as one that was written — an explicit "no" is a verdict, and
+ * re-asking every run is how it gets forgotten. By the same token a pre-#1042
+ * record carrying no `outcome` at all keeps its old meaning unchanged.
+ *
+ * Never throws.
  * @param {ReconcileCandidate} candidate - the candidate under consideration.
  * @param {ReconcileCandidate[]} existing - the currently-persisted candidates.
  * @returns {boolean}
@@ -337,3 +356,85 @@ export function mergeCandidates({ candidates, repoRoot, storePath } = {}) {
   const result = writeStore(absPath, merged);
   return { merged, written: result.ok === true, skipped };
 }
+
+/**
+ * Stamp the persisted candidate for `learningKey` as terminally processed and
+ * write it back through {@link mergeCandidates} — the ONLY writer of this
+ * store, so this function opens no second write path and no new persisted
+ * format: `outcome` is an ordinary field alongside `processed_at`, and the
+ * read-side shape guard (`isCandidateShape`) only requires `learning_key` +
+ * `created_at`, so it accepts the stamped record unchanged.
+ *
+ * Looks up the existing record by `learningKey` first, so the write preserves
+ * everything the run that proposed it originally recorded (`slug`, `reason`,
+ * `confidence`, `created_at`) and only ADDS the terminal stamp. When no record
+ * exists yet — e.g. the proposing run used `dryRun` and never merged into the
+ * store, or `engine.mjs` discovered the on-disk `.claude/rules/` file directly
+ * without ever having proposed it in a prior run — a fresh terminal record is
+ * minted from the caller-supplied `fallback*` fields so a LATER run can still
+ * dedupe against it.
+ *
+ * `mergeCandidates`'s own dedupe rule still applies underneath this call: if
+ * the existing record is ALREADY terminal (a prior `processed_at`), the merge
+ * keeps the existing verdict and this call is a no-op on disk — a terminal
+ * verdict is never regressed, even by a second stamp attempt.
+ *
+ * Never throws (delegates to the never-throwing `loadCandidates`/
+ * `mergeCandidates`); on write failure returns `{ written: false, ... }`.
+ *
+ * @param {Object} [params]
+ * @param {string} [params.learningKey] - required; a missing/empty key is a no-op (`{written:false, stamped:null}`).
+ * @param {'written'|'already-on-disk'|'rejected'|string} [params.outcome]
+ * @param {string} [params.processedAt] - ISO timestamp; defaults to `new Date().toISOString()`.
+ * @param {string} [params.fallbackSlug] - used only when no existing record is found.
+ * @param {string} [params.fallbackCandidateId] - used only when no existing record is found; defaults to `makeCandidateId(learningKey, fallbackSlug)`.
+ * @param {number} [params.fallbackConfidence] - used only when no existing record is found.
+ * @param {string} [params.repoRoot]
+ * @param {string} [params.storePath]
+ * @returns {{ written: boolean, stamped: ReconcileCandidate|null }}
+ */
+export function markCandidateProcessed({
+  learningKey,
+  outcome,
+  processedAt,
+  fallbackSlug,
+  fallbackCandidateId,
+  fallbackConfidence,
+  repoRoot,
+  storePath,
+} = {}) {
+  if (typeof learningKey !== 'string' || learningKey.length === 0) {
+    return { written: false, stamped: null };
+  }
+
+  const stampAt =
+    typeof processedAt === 'string' && processedAt.length > 0 ? processedAt : new Date().toISOString();
+
+  const { records: existing } = loadCandidates({ repoRoot, storePath });
+  const found = existing.find((rec) => rec && rec.learning_key === learningKey);
+
+  /** @type {ReconcileCandidate} */
+  let stamped;
+  if (found) {
+    stamped = { ...found, processed_at: stampAt, outcome: typeof outcome === 'string' ? outcome : (found.outcome ?? null) };
+  } else {
+    const slug = typeof fallbackSlug === 'string' ? fallbackSlug : '';
+    stamped = buildCandidate({
+      id:
+        typeof fallbackCandidateId === 'string' && fallbackCandidateId.length > 0
+          ? fallbackCandidateId
+          : makeCandidateId(learningKey, slug),
+      learningKey,
+      slug,
+      status: 'proposed',
+      reason: 'stamped without a prior sidecar record',
+      confidence: typeof fallbackConfidence === 'number' ? fallbackConfidence : 0,
+      createdAt: stampAt,
+    });
+    stamped.processed_at = stampAt;
+    stamped.outcome = typeof outcome === 'string' ? outcome : null;
+  }
+
+  const result = mergeCandidates({ candidates: [stamped], repoRoot, storePath });
+  return { written: result.written === true, stamped };
+}