session-orchestrator 3.22.0 → 3.24.0

package/scripts/lib/reconcile/idempotency.mjs

@@ -44,6 +44,17 @@
  * @property {string} created_at      - ISO timestamp.
  * @property {string|null} processed_at  - terminal stamp (mirrors repair store).
  * @property {string|null} superseded_by
+ * @property {string|null} [outcome]  - terminal disposition, one of:
+ *        `'written'` (writer.mjs persisted the rule file), `'already-on-disk'`
+ *        (engine.mjs found a matching `.claude/rules/` provenance block before
+ *        proposing — issue #484), or `'rejected'` (the operator declined the
+ *        proposal in the approval AUQ and writer.mjs archived it — issue #1042).
+ *        Optional and additive: `isCandidateShape` only requires `learning_key`
+ *        + `created_at`, so an older record without this field still round-trips
+ *        through the store unchanged, and nothing BRANCHES on the value —
+ *        terminality is `processed_at` alone (see {@link isProcessed}), so a
+ *        pre-#1042 record with no `outcome` keeps behaving exactly as before
+ *        (absence reads as `'written'`, the only outcome that existed then).
  */
 
 import { mkdirSync, readFileSync, writeFileSync, renameSync } from 'node:fs';
@@ -265,7 +276,15 @@ export function loadCandidates({ repoRoot, storePath } = {}) {
  * True iff `existing` already holds a candidate that shares `candidate`'s
  * `learning_key` AND has a terminal `processed_at` stamp. The reconcile engine
  * uses this to idempotently SKIP re-proposing a learning whose verdict is
- * already terminal. Never throws.
+ * already terminal.
+ *
+ * Deliberately OUTCOME-AGNOSTIC: `processed_at` alone decides. A rule the
+ * operator DECLINED (`outcome: 'rejected'`, issue #1042) is therefore just as
+ * terminal as one that was written — an explicit "no" is a verdict, and
+ * re-asking every run is how it gets forgotten. By the same token a pre-#1042
+ * record carrying no `outcome` at all keeps its old meaning unchanged.
+ *
+ * Never throws.
  * @param {ReconcileCandidate} candidate - the candidate under consideration.
  * @param {ReconcileCandidate[]} existing - the currently-persisted candidates.
  * @returns {boolean}
@@ -337,3 +356,85 @@ export function mergeCandidates({ candidates, repoRoot, storePath } = {}) {
   const result = writeStore(absPath, merged);
   return { merged, written: result.ok === true, skipped };
 }
+
+/**
+ * Stamp the persisted candidate for `learningKey` as terminally processed and
+ * write it back through {@link mergeCandidates} — the ONLY writer of this
+ * store, so this function opens no second write path and no new persisted
+ * format: `outcome` is an ordinary field alongside `processed_at`, and the
+ * read-side shape guard (`isCandidateShape`) only requires `learning_key` +
+ * `created_at`, so it accepts the stamped record unchanged.
+ *
+ * Looks up the existing record by `learningKey` first, so the write preserves
+ * everything the run that proposed it originally recorded (`slug`, `reason`,
+ * `confidence`, `created_at`) and only ADDS the terminal stamp. When no record
+ * exists yet — e.g. the proposing run used `dryRun` and never merged into the
+ * store, or `engine.mjs` discovered the on-disk `.claude/rules/` file directly
+ * without ever having proposed it in a prior run — a fresh terminal record is
+ * minted from the caller-supplied `fallback*` fields so a LATER run can still
+ * dedupe against it.
+ *
+ * `mergeCandidates`'s own dedupe rule still applies underneath this call: if
+ * the existing record is ALREADY terminal (a prior `processed_at`), the merge
+ * keeps the existing verdict and this call is a no-op on disk — a terminal
+ * verdict is never regressed, even by a second stamp attempt.
+ *
+ * Never throws (delegates to the never-throwing `loadCandidates`/
+ * `mergeCandidates`); on write failure returns `{ written: false, ... }`.
+ *
+ * @param {Object} [params]
+ * @param {string} [params.learningKey] - required; a missing/empty key is a no-op (`{written:false, stamped:null}`).
+ * @param {'written'|'already-on-disk'|'rejected'|string} [params.outcome]
+ * @param {string} [params.processedAt] - ISO timestamp; defaults to `new Date().toISOString()`.
+ * @param {string} [params.fallbackSlug] - used only when no existing record is found.
+ * @param {string} [params.fallbackCandidateId] - used only when no existing record is found; defaults to `makeCandidateId(learningKey, fallbackSlug)`.
+ * @param {number} [params.fallbackConfidence] - used only when no existing record is found.
+ * @param {string} [params.repoRoot]
+ * @param {string} [params.storePath]
+ * @returns {{ written: boolean, stamped: ReconcileCandidate|null }}
+ */
+export function markCandidateProcessed({
+  learningKey,
+  outcome,
+  processedAt,
+  fallbackSlug,
+  fallbackCandidateId,
+  fallbackConfidence,
+  repoRoot,
+  storePath,
+} = {}) {
+  if (typeof learningKey !== 'string' || learningKey.length === 0) {
+    return { written: false, stamped: null };
+  }
+
+  const stampAt =
+    typeof processedAt === 'string' && processedAt.length > 0 ? processedAt : new Date().toISOString();
+
+  const { records: existing } = loadCandidates({ repoRoot, storePath });
+  const found = existing.find((rec) => rec && rec.learning_key === learningKey);
+
+  /** @type {ReconcileCandidate} */
+  let stamped;
+  if (found) {
+    stamped = { ...found, processed_at: stampAt, outcome: typeof outcome === 'string' ? outcome : (found.outcome ?? null) };
+  } else {
+    const slug = typeof fallbackSlug === 'string' ? fallbackSlug : '';
+    stamped = buildCandidate({
+      id:
+        typeof fallbackCandidateId === 'string' && fallbackCandidateId.length > 0
+          ? fallbackCandidateId
+          : makeCandidateId(learningKey, slug),
+      learningKey,
+      slug,
+      status: 'proposed',
+      reason: 'stamped without a prior sidecar record',
+      confidence: typeof fallbackConfidence === 'number' ? fallbackConfidence : 0,
+      createdAt: stampAt,
+    });
+    stamped.processed_at = stampAt;
+    stamped.outcome = typeof outcome === 'string' ? outcome : null;
+  }
+
+  const result = mergeCandidates({ candidates: [stamped], repoRoot, storePath });
+  return { written: result.written === true, stamped };
+}

package/scripts/lib/reconcile/renderer.mjs

@@ -43,6 +43,8 @@
 
 import { createHash } from 'node:crypto';
 
+import yaml from 'js-yaml';
+
 import { kebab } from '../learnings/kebab.mjs';
 import {
   EVIDENCE_ITEM_MAX_BYTES,
@@ -64,6 +66,10 @@ import {
 /**
  * Short, stable SHA-1 prefix (first 7 hex chars) of an input string.
  *
+ * Slug-collision helper ONLY. Deliberately NOT reused for the evidence digest
+ * below (#1101): 7 hex chars of SHA-1 is a namespacing device, not a tamper
+ * seal, and conflating the two would let a slug change read as a seal break.
+ *
  * @param {string} input
  * @returns {string}
  */
@@ -71,6 +77,117 @@ function shortHash(input) {
   return createHash('sha1').update(String(input)).digest('hex').slice(0, 7);
 }
 
+/**
+ * Shape of the `evidence-digest` frontmatter scalar (#1101).
+ *
+ * `sha256-v1:` names BOTH the algorithm and the canonical-input version, so a
+ * future change to the input recipe bumps the prefix rather than silently
+ * invalidating every seal on disk.
+ *
+ * A LITERAL COPY of this regex lives in `skills/claude-md-drift-check/checker.mjs`
+ * (Check 8). The checker is a standalone skill script; importing across the
+ * skill/script boundary would be new coupling for one 30-character literal.
+ * `tests/lib/reconcile/renderer.test.mjs` pins the two copies equal.
+ */
+export const EVIDENCE_DIGEST_RE = /^sha256-v1:[0-9a-f]{64}$/;
+
+/**
+ * Shape of the `evidence-recorded-at` provenance value — an ISO-8601 instant.
+ *
+ * A learning's `created_at` is machine-written by the learnings store, but it
+ * reaches this renderer on an agent-authored record, and it lands on a body
+ * line OUTSIDE an inline-code span. A value carrying a newline could forge a
+ * sibling provenance line, so a non-conforming value is treated as ABSENT (the
+ * empty-string digest component) rather than interpolated. Declining to record
+ * an unusable timestamp is the same posture as the three backfill cases whose
+ * `created_at` is genuinely unavailable — an invented date would make the
+ * digest a reference masquerading as evidence.
+ */
+const EVIDENCE_RECORDED_AT_RE = /^\d{4}-\d{2}-\d{2}T[0-9:.]+(?:Z|[+-]\d{2}:\d{2})$/;
+
+/**
+ * Compute the `sha256-v1:` evidence digest from its six canonical components.
+ *
+ * ── Why this is EVIDENCE and not a REFERENCE (#1101) ─────────────────────────
+ * `.orchestrator/metrics/learnings.jsonl` is gitignored, so in a fresh clone a
+ * rule's `learning-key` resolves to nothing and its provenance is unverifiable.
+ * The last component here is the RENDERED `## Evidence` block itself — so a
+ * reader holding only the `.md` file re-reads its own `## Evidence` section,
+ * re-joins the five header fields from its own `## Provenance` block, hashes,
+ * and compares. `learnings.jsonl` is never consulted.
+ *
+ * The input is the six components joined with `\n`, UTF-8, **no trailing
+ * newline**. Order is fixed and versioned by the `sha256-v1:` prefix.
+ *
+ * @param {object} parts
+ * @param {string} parts.learningKey
+ * @param {string} parts.learningId - the same `'n/a'` fallback the body emits.
+ * @param {string} parts.sourceSession - ditto.
+ * @param {string} parts.evidenceRecordedAt - the learning's `created_at`, or
+ *   `''` when absent/unusable. NEVER a substituted or invented date.
+ * @param {string} parts.confidence - `String(metadata.confidence)`.
+ * @param {string} parts.evidenceBlock - the EXACT string written to
+ *   `## Evidence`, verbatim.
+ * @returns {string} `sha256-v1:<64 lowercase hex>`
+ */
+export function computeEvidenceDigest({
+  learningKey,
+  learningId,
+  sourceSession,
+  evidenceRecordedAt,
+  confidence,
+  evidenceBlock,
+}) {
+  const canonical = [
+    String(learningKey ?? ''),
+    String(learningId ?? ''),
+    String(sourceSession ?? ''),
+    String(evidenceRecordedAt ?? ''),
+    String(confidence ?? ''),
+    String(evidenceBlock ?? ''),
+  ].join('\n');
+  return `sha256-v1:${createHash('sha256').update(canonical, 'utf8').digest('hex')}`;
+}
+
+/**
+ * Serialize a single-line frontmatter scalar the way real YAML requires it
+ * (#1041), while staying parseable by `rule-loader.mjs`'s hand-rolled parser.
+ *
+ * `description` is the one frontmatter field {@link assertSafeDescription}
+ * permits to contain a colon (14 live rules carry a second `:` in their
+ * description, and this repo's own loader tolerates it — see `sanitize.mjs`'s
+ * module doc). This repo's loader is not the only reader: Claude Code's OWN
+ * native frontmatter loader parses `.claude/rules/*.md` as real YAML, and
+ * `description: fixes X: this breaks Y` emitted unquoted is a YAML mapping
+ * with an extra, unindented `this breaks Y` value — js-yaml throws "bad
+ * indentation of a mapping entry" on exactly that shape.
+ *
+ * `yaml.dump` emits a bare plain scalar when no quoting is needed (byte-
+ * identical to the un-quoted output this renderer shipped before #1041, so
+ * every colon-free description stays unchanged) and falls back to a quoted
+ * scalar only when required. js-yaml's DEFAULT quoting style (no
+ * `quotingType` option passed) is SINGLE-quote, not double — confirmed via
+ * `node_modules/js-yaml/lib/dumper.js` `chooseScalarStyle`, which only forces
+ * DOUBLE for a string containing a non-printable code point. A single-quoted
+ * scalar is what `rule-loader.mjs`'s hand-rolled parser round-trips
+ * correctly: it strips exactly one leading/trailing quote character
+ * (`/^["']|["']$/g`) with NO escape processing, so a js-yaml DOUBLE-quoted
+ * `\"` escape would survive the strip verbatim and corrupt the parsed value —
+ * single-quote style has no such escape to survive (its only escape, an
+ * interior `'` doubled to `''`, is a narrower edge case than the colon defect
+ * this fix targets and is out of scope here). `description` can never contain
+ * a raw newline by the time this runs — {@link assertSafeDescription} /
+ * {@link assertNoControlChars} reject one earlier in the same call — so the
+ * literal/folded block-scalar branches `yaml.dump` has for multi-line input
+ * are unreachable on this call site.
+ *
+ * @param {string} value
+ * @returns {string} a single YAML scalar line, no trailing newline
+ */
+function dumpYamlScalar(value) {
+  return yaml.dump(value, { lineWidth: -1 }).trimEnd();
+}
+
 /**
  * Derive a kebab-case, stable, idempotent slug for a learning's rule file.
  *
@@ -253,7 +370,9 @@ export function renderRule(learning, metadata) {
   fm.push('---');
   fm.push('auto-generated: true');
   fm.push('alwaysApply: false');
-  fm.push(`description: ${metadata.description}`);
+  // #1041: js-yaml-safe serialization — see `dumpYamlScalar` above. Byte-
+  // identical to the prior unquoted output for every colon-free description.
+  fm.push(`description: ${dumpYamlScalar(metadata.description)}`);
   // OMIT the `globs:` key entirely when there are no globs (a host-class-only
   // activation axis). An empty `globs:` line parses in the loader as
   // `globs: []`, which short-circuits BEFORE the host-class gate runs
@@ -287,7 +406,6 @@ export function renderRule(learning, metadata) {
   fm.push(`learning-key: ${metadata.learningKey}`);
   fm.push(`confidence: ${metadata.confidence}`);
   fm.push(`expires-at: ${metadata.expiresAt}`);
-  fm.push('---');
 
   // ── Body (free markdown; does not affect frontmatter parsing) ────────────
   // Both provenance values are rendered INSIDE an inline-code span below, which
@@ -306,6 +424,28 @@ export function renderRule(learning, metadata) {
       })
     : 'n/a';
 
+  // ── Evidence digest (#1101) ──────────────────────────────────────────────
+  // Rendered ONCE and reused, so the hashed bytes and the written bytes are the
+  // same string by construction rather than by two calls agreeing.
+  const evidenceBlock = renderEvidence(learning.evidence);
+  const evidenceRecordedAt =
+    typeof learning.created_at === 'string' && EVIDENCE_RECORDED_AT_RE.test(learning.created_at)
+      ? learning.created_at
+      : '';
+  const evidenceDigest = computeEvidenceDigest({
+    learningKey: metadata.learningKey,
+    learningId,
+    sourceSession,
+    evidenceRecordedAt,
+    confidence: String(metadata.confidence),
+    evidenceBlock,
+  });
+  // Appended AFTER `expires-at` on purpose: every key BEFORE it keeps its byte
+  // position, so the frontmatter of an existing rule stays byte-stable and the
+  // digest is purely additive.
+  fm.push(`evidence-digest: ${evidenceDigest}`);
+  fm.push('---');
+
   // The untrusted region (H1 + insight + Evidence) is FRAMED: the envelope
   // states, in machine-generated text the record cannot forge, that everything
   // inside is agent-authored data rather than an instruction to the reading
@@ -319,7 +459,7 @@ export function renderRule(learning, metadata) {
     insight,
     '',
     '## Evidence',
-    renderEvidence(learning.evidence),
+    evidenceBlock,
     '',
     UNTRUSTED_END,
     '',
@@ -328,6 +468,11 @@ export function renderRule(learning, metadata) {
     `- learning-key: \`${metadata.learningKey}\``,
     `- learning-id: \`${learningId}\``,
     `- source-session: \`${sourceSession}\``,
+    // #1101: the seal, its one un-derivable input, and the recipe to re-check
+    // both — everything an offline reader needs, in the file itself.
+    `- evidence-digest: \`${evidenceDigest}\``,
+    `- evidence-recorded-at: ${evidenceRecordedAt}`,
+    '- evidence-digest-input: learning-key \\n learning-id \\n source-session \\n evidence-recorded-at \\n confidence \\n <the `## Evidence` block above, verbatim UTF-8, LF-joined, no trailing newline>',
     `- confidence: ${metadata.confidence}`,
     '- generated-by: reconciliation-engine (Epic #693 FA2 / #695)',
     `- expires-at: ${metadata.expiresAt}`,

package/scripts/lib/reconcile/sanitize.mjs

@@ -43,21 +43,40 @@
  *     throw to an audited `emit/render error: …` rejection, never a crash.
  *
  * -- What this module deliberately does NOT touch -----------------------------
- * Colons (14 live `description:` lines carry a second one, and the loader's
- * first-colon split at rule-loader.mjs handles them), non-ASCII punctuation
- * (`—` is content; the trailing `…` is the emitter's own truncation marker),
- * `*`/`**` inside globs (already double-quoted by the renderer), body `---`
- * horizontal rules (a WRAPPER-side concern in print-applicable-rules.mjs, not
- * ours), fenced code blocks, markdown headings, inline backticks and HTML
- * comments in the body — those ARE the reconciler's own output shape.
- *
- * The frontmatter parser this defends against is HAND-ROLLED, not a YAML
- * library (`rule-loader.mjs` imports only node:fs/path/module): a `:` in a
- * scalar is harmless, a mid-line `#` is harmless, quotes survive — the ONE
- * escape is a newline, which starts a new top-level key. That is why the
- * machine-value asserts below are shaped as "no control characters / exact
- * token shape" and NOT as YAML quoting or colon escaping, which would be both
- * wrong and destructive here.
+ * Colons in general (a mid-line `#`), non-ASCII punctuation (`—` is content;
+ * the trailing `…` is the emitter's own truncation marker), `*`/`**` inside
+ * globs (already double-quoted by the renderer), body `---` horizontal rules
+ * (a WRAPPER-side concern in print-applicable-rules.mjs, not ours), fenced
+ * code blocks, markdown headings, inline backticks and HTML comments in the
+ * body — those ARE the reconciler's own output shape, and this module's SHAPE
+ * asserts below are unaffected either way: they reject on control characters,
+ * dangerous invisibles, wrapper-forgery literals and length, never on colons.
+ *
+ * The frontmatter parser THIS module's asserts are shaped for is HAND-ROLLED,
+ * not a YAML library (`rule-loader.mjs` imports only node:fs/path/module): a
+ * `:` in a scalar is harmless there, a mid-line `#` is harmless, quotes
+ * survive — the ONE escape is a newline, which starts a new top-level key.
+ * That is why the machine-value asserts below are shaped as "no control
+ * characters / exact token shape" and NOT as YAML quoting or colon escaping,
+ * which would be both wrong and destructive for THIS parser.
+ *
+ * That reasoning stopped being the whole story once #1041 landed: Claude
+ * Code's OWN native frontmatter loader — the thing that actually delivers a
+ * rendered rule into every future agent's context — parses the SAME file as
+ * real YAML, and a colon-bearing `description:` value emitted unquoted
+ * (`description: fixes X: this breaks Y`) is a syntax error there ("bad
+ * indentation of a mapping entry"), not merely a shape this module chose not
+ * to touch. The fix for that is NOT a new assert in this module — 14 live
+ * `description:` lines legitimately carry a second colon, and this module's
+ * job stays "reject an unsafe SHAPE", never "reformat a safe one". Instead
+ * `renderer.mjs`'s `dumpYamlScalar` (built on the already-installed `js-yaml`
+ * dependency) serializes the `description` value returned by
+ * {@link assertSafeDescription} as a real YAML scalar (plain when safe,
+ * single-quoted only when a colon or similar forces it) at the render POINT,
+ * after this module's shape gate has already run. The two layers stay
+ * distinct on purpose: this module still decides whether the value may be
+ * emitted at all; `renderer.mjs` decides how to spell the value both parsers
+ * — the hand-rolled one AND js-yaml — can read back unchanged.
  *
  * Pure functions — no file I/O, no process state. Part of issue #1015.
  *
@@ -190,8 +209,12 @@ export const PROVENANCE_TOKEN_RE = /^[A-Za-z0-9._:-]+$/;
 // byte gate. The property class keeps the source pure ASCII by construction.
 //
 // CONTROL_RE is the REJECT test for single-line machine values, where a TAB or
-// LF is an escape rather than formatting.
-const CONTROL_RE = /\p{Cc}/u;
+// LF is an escape rather than formatting. U+2028/U+2029 (Zl/Zp) are NOT \p{Cc}
+// but YAML 1.1 and js-yaml's loader treat them as line breaks — measured
+// 2026-08-23 (W4-R2): `"safe\u2028alwaysApply: true"` passed this gate and was
+// neutralised only by js-yaml's escaping in the renderer. The guarantee
+// belongs in the gate, not in a downstream serializer's style choice.
+const CONTROL_RE = /[\p{Cc}\u2028\u2029]/u;
 
 // The same category, global, for the prose strip. TAB/LF are re-admitted by the
 // replacement callback in `sanitizeProse` (multi-line prose is legitimate in a