session-orchestrator 5.5.0 → 5.7.0

package/scripts/lib/vault-mirror/prune.mjs

@@ -0,0 +1,413 @@
+/**
+ * prune.mjs — plan and apply the retention rules to the EXISTING mirror zones
+ * (`40-learnings/`, `50-sessions/`) of a vault. Issue #1513.
+ *
+ * The mirror (`process.mjs`) stops producing ballast from now on; this module
+ * cleans up what it produced before. Same rules, same module for them:
+ * `./retention.mjs`.
+ *
+ * ## What it decides, per mirror-owned note (one reason per note)
+ *
+ * | reason                 | zone      | action  | rule |
+ * |------------------------|-----------|---------|------|
+ * | `duplicate`            | both      | archive | same normalised insight (learnings), or the same note id already at the relocation target; the canonical copy is kept |
+ * | `superseded`           | learnings | archive | same slug modulo hyphens (pre-#725 / post-#725 slug of one subject) with a different insight — the newest `updated` is kept; or the mirror already marked it `archived-reason: superseded` |
+ * | `expired`              | learnings | archive | frontmatter `expires:` before today, or already marked `archived-reason: expired` |
+ * | `metrics-only-session` | sessions  | archive | `## Notes` free text below `min-narrative-chars`; its row is written to the month rollup FIRST |
+ * | `namespace-alias`      | both      | move    | the namespace folder is an alias of another (explicit `--alias`, or the same name modulo hyphens where only the other one has hyphens) |
+ * | `flat-relocate`        | both      | move    | a flat legacy note (no namespace folder) whose `source-repo` names its namespace |
+ *
+ * Expiry is decided FIRST: an expired or already archived note is never the
+ * kept copy of a group (#1513 review H1). Among the live copies the canonical
+ * one is, in this order: in its own namespace folder > in any namespace
+ * folder > `status: verified` > later `expires` > carries `source-record` >
+ * more hyphens in the file name (the post-#725 slug) > newer `updated` > path
+ * order. Placeholder and very short insights never form a group (M4).
+ *
+ * ## What it never does
+ *
+ * - Touch a note without `_generator: session-orchestrator-vault-mirror@1`
+ *   (hand-written notes), a rollup, or anything outside the two zones.
+ * - Delete. `archive` moves the note to `90-archive/mirror/<zone>/<same path>`
+ *   with `status: archived` + `archived-reason`; `move` renames inside the zone.
+ *   Git sees renames; history stays.
+ * - Write anything in plan mode. {@link applyPrune} is the only write path.
+ *
+ * Idempotent: after an apply, the archived notes are out of the zones, the
+ * moved notes sit where the plan computes them, and the rollups already hold
+ * their rows — a second plan is empty.
+ *
+ * @module scripts/lib/vault-mirror/prune
+ */
+
+import {
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  realpathSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { createHash } from 'node:crypto';
+import { basename, dirname, join, resolve, sep } from 'node:path';
+
+import { parseFrontmatter, subjectToSlug } from './utils.mjs';
+import {
+  GENERATOR_MARKER,
+  ROLLUP_PREFIX,
+  archivePathFor,
+  canonicalAmong,
+  markArchived,
+  noteInsightKey,
+  noteNarrativeChars,
+  renderRollup,
+  rollupRelPath,
+  rollupRowFromNote,
+} from './retention.mjs';
+
+export const ZONES = Object.freeze(['40-learnings', '50-sessions']);
+export const MANIFEST_SCHEMA = 'vault-mirror-prune/1';
+
+const dehyphen = (s) => String(s).replace(/-/g, '');
+const hyphens = (s) => (String(s).match(/-/g) || []).length;
+
+/**
+ * Read every mirror-owned note of both zones. Depth: `<zone>/<file>` (flat) or
+ * `<zone>/<ns>/<file>`; anything deeper is not a mirror layout and is ignored.
+ */
+function readZoneNotes(vaultDir) {
+  const notes = [];
+  let handwritten = 0;
+  for (const zone of ZONES) {
+    const zoneAbs = join(vaultDir, zone);
+    let top;
+    try {
+      top = readdirSync(zoneAbs, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    const files = [];
+    for (const ent of top) {
+      if (ent.isFile() && ent.name.endsWith('.md')) files.push({ dirNs: '', name: ent.name });
+      else if (ent.isDirectory()) {
+        let inner;
+        try {
+          inner = readdirSync(join(zoneAbs, ent.name), { withFileTypes: true });
+        } catch {
+          continue;
+        }
+        for (const f of inner) if (f.isFile() && f.name.endsWith('.md')) files.push({ dirNs: ent.name, name: f.name });
+      }
+    }
+    for (const { dirNs, name } of files) {
+      const rel = dirNs ? `${zone}/${dirNs}/${name}` : `${zone}/${name}`;
+      if (name.startsWith(ROLLUP_PREFIX)) continue;
+      let content;
+      try {
+        content = readFileSync(join(vaultDir, rel), 'utf8');
+      } catch {
+        continue;
+      }
+      const fm = parseFrontmatter(content);
+      if (!fm || fm['_generator'] !== GENERATOR_MARKER) {
+        handwritten += 1;
+        continue;
+      }
+      notes.push({ rel, zone, dirNs, name, base: name.slice(0, -3), fm, content });
+    }
+  }
+  return { notes, handwritten };
+}
+
+/**
+ * Namespace alias resolver: explicit map first, then the hyphen rule — `ns`
+ * maps to a sibling folder (in either zone) with the same name modulo hyphens
+ * when `ns` has fewer hyphens (the directory-name fallback of the namespace
+ * resolver drops them; the remote-derived name keeps them).
+ */
+function makeAliasResolver(notes, explicit) {
+  const present = new Set(notes.map((n) => n.dirNs).filter(Boolean));
+  for (const to of Object.values(explicit)) present.add(to);
+  // The hyphen rule is the SAME function the mirror applies at write time
+  // (`canonicalAmong`, #1513 review M2), so both agree on the folder.
+  const resolveNs = (ns) => {
+    if (!ns) return ns;
+    if (Object.hasOwn(explicit, ns)) return explicit[ns];
+    return canonicalAmong(present, ns);
+  };
+  const used = {};
+  for (const ns of present) if (resolveNs(ns) !== ns) used[ns] = resolveNs(ns);
+  return { resolveNs, aliases: used };
+}
+
+/**
+ * Which copy of a group stays. Callers pass LIVE notes only (not expired, not
+ * archived — #1513 review H1), so rank never has to weigh a dead copy.
+ */
+function canonicalRank(a, b) {
+  const own = (n) => (n.dirNs !== '' && n.dirNs === n.effNs ? 1 : 0);
+  const inNs = (n) => (n.dirNs !== '' ? 1 : 0);
+  const verified = (n) => (n.fm.status === 'verified' ? 1 : 0);
+  const rec = (n) => (n.fm['source-record'] ? 1 : 0);
+  return (
+    own(b) - own(a) ||
+    inNs(b) - inNs(a) ||
+    verified(b) - verified(a) ||
+    String(b.fm.expires ?? '9999').localeCompare(String(a.fm.expires ?? '9999')) ||
+    rec(b) - rec(a) ||
+    hyphens(b.base) - hyphens(a.base) ||
+    String(b.fm.updated ?? '').localeCompare(String(a.fm.updated ?? '')) ||
+    a.rel.localeCompare(b.rel)
+  );
+}
+
+/**
+ * Plan the prune. Reads only.
+ *
+ * @param {object} opts
+ * @param {string} opts.vaultDir absolute vault root
+ * @param {string} opts.today `YYYY-MM-DD`
+ * @param {number} [opts.minNarrativeChars=400]
+ * @param {Record<string,string>} [opts.aliases] explicit namespace aliases
+ * @returns {{actions: object[], counts: Record<string, number>, aliases: Record<string,string>, handwritten: number, rollups: Record<string, object[]>}}
+ */
+export function planPrune({ vaultDir, today, minNarrativeChars = 400, aliases = {} }) {
+  if (typeof vaultDir !== 'string' || vaultDir.length === 0) throw new TypeError('planPrune: vaultDir is required');
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(String(today))) throw new TypeError(`planPrune: today must be YYYY-MM-DD, got ${today}`);
+  const root = resolve(vaultDir);
+  const { notes, handwritten } = readZoneNotes(root);
+  const { resolveNs, aliases: usedAliases } = makeAliasResolver(notes, aliases);
+
+  for (const n of notes) {
+    const sourceRepo = n.fm['source-repo'] ? subjectToSlug(n.fm['source-repo']) : '';
+    n.effNs = n.dirNs ? resolveNs(n.dirNs) : resolveNs(sourceRepo);
+  }
+
+  /** @type {Map<string, {reason: string, keep?: string}>} */
+  const decided = new Map();
+  const decide = (n, reason, keep) => {
+    if (!decided.has(n.rel)) decided.set(n.rel, keep ? { reason, keep } : { reason });
+  };
+
+  // ── learnings: expired FIRST, then duplicate / superseded among the living ─
+  // #1513 review H1: deciding duplicates first let a group keep its EXPIRED
+  // copy and archive the valid one as its duplicate — the expiry pass then
+  // archived the keeper too, leaving no active copy at all (350 groups in the
+  // Meta-Vault, 34 of them with a live copy). A dead copy is never a keeper.
+  const learnings = notes.filter((n) => n.zone === '40-learnings' && n.fm.type === 'learning');
+  const live = [];
+  for (const n of learnings) {
+    const marked = n.fm.status === 'archived' ? n.fm['archived-reason'] : '';
+    if (marked === 'superseded' || marked === 'expired') decide(n, marked);
+    else if (n.fm.status === 'archived') decide(n, 'superseded');
+    else if (/^\d{4}-\d{2}-\d{2}/.test(String(n.fm.expires ?? '')) && String(n.fm.expires).slice(0, 10) < today) {
+      decide(n, 'expired');
+    } else live.push(n);
+  }
+
+  const byInsight = new Map();
+  for (const n of live) {
+    n.insightKey = noteInsightKey(n.content);
+    if (!n.insightKey) continue;
+    if (!byInsight.has(n.insightKey)) byInsight.set(n.insightKey, []);
+    byInsight.get(n.insightKey).push(n);
+  }
+  for (const group of byInsight.values()) {
+    if (group.length < 2) continue;
+    const buckets = new Map();
+    for (const n of group) {
+      if (!buckets.has(n.effNs)) buckets.set(n.effNs, []);
+      buckets.get(n.effNs).push(n);
+    }
+    const named = [...buckets.keys()].filter(Boolean).sort();
+    for (const ns of named) {
+      const sorted = buckets.get(ns).sort(canonicalRank);
+      for (const n of sorted.slice(1)) decide(n, 'duplicate', sorted[0].rel);
+    }
+    const unnamed = (buckets.get('') ?? []).sort(canonicalRank);
+    if (named.length > 0) {
+      const keeper = buckets.get(named[0]).sort(canonicalRank)[0];
+      for (const n of unnamed) decide(n, 'duplicate', keeper.rel);
+    } else {
+      for (const n of unnamed.slice(1)) decide(n, 'duplicate', unnamed[0].rel);
+    }
+  }
+
+  const bySlug = new Map();
+  for (const n of live) {
+    if (decided.has(n.rel)) continue;
+    const k = `${n.effNs}\0${dehyphen(n.base)}`;
+    if (!bySlug.has(k)) bySlug.set(k, []);
+    bySlug.get(k).push(n);
+  }
+  for (const group of bySlug.values()) {
+    if (group.length < 2) continue;
+    const sorted = group.sort(
+      (a, b) => String(b.fm.updated ?? '').localeCompare(String(a.fm.updated ?? '')) || canonicalRank(a, b),
+    );
+    for (const n of sorted.slice(1)) decide(n, 'superseded', sorted[0].rel);
+  }
+
+  // ── sessions: metrics-only → rollup + archive ──────────────────────────────
+  /** @type {Record<string, object[]>} rollup rel path → rows */
+  const rollups = {};
+  const sessions = notes.filter((n) => n.zone === '50-sessions' && n.fm.type === 'session');
+  const unparseable = new Set();
+  for (const n of sessions) {
+    if (noteNarrativeChars(n.content) >= minNarrativeChars) continue;
+    const row = rollupRowFromNote(n.content, { id: n.base });
+    if (row === null) {
+      unparseable.add(n.rel);
+      continue;
+    }
+    const rollup = rollupRelPath(n.effNs, row.date.slice(0, 7));
+    (rollups[rollup] ??= []).push(row);
+    decided.set(n.rel, { reason: 'metrics-only-session', rollup });
+  }
+
+  // ── relocation of the survivors: alias folder / flat legacy layout ─────────
+  const existingRel = new Set(notes.map((n) => n.rel));
+  const claimed = new Set();
+  const actions = [];
+  for (const n of notes) {
+    const d = decided.get(n.rel);
+    if (d) {
+      actions.push({ path: n.rel, reason: d.reason, action: 'archive', target: archivePathFor(n.rel), ...(d.keep ? { keep: d.keep } : {}), ...(d.rollup ? { rollup: d.rollup } : {}) });
+      continue;
+    }
+    if (unparseable.has(n.rel)) {
+      actions.push({ path: n.rel, reason: 'metrics-only-session', action: 'keep', note: 'no date or id to roll up' });
+      continue;
+    }
+    if (!n.effNs || n.effNs === n.dirNs) continue;
+    const reason = n.dirNs ? 'namespace-alias' : 'flat-relocate';
+    const target = `${n.zone}/${n.effNs}/${n.name}`;
+    if (claimed.has(target) || existingRel.has(target)) {
+      const twin = notes.find((m) => m.rel === target);
+      if (twin && twin.fm.id === n.fm.id && !decided.has(twin.rel)) {
+        actions.push({ path: n.rel, reason: 'duplicate', action: 'archive', target: archivePathFor(n.rel), keep: target });
+      } else {
+        actions.push({ path: n.rel, reason, action: 'keep', note: `relocation target taken: ${target}` });
+      }
+      continue;
+    }
+    claimed.add(target);
+    actions.push({ path: n.rel, reason, action: 'move', target });
+  }
+
+  actions.sort((a, b) => a.path.localeCompare(b.path));
+  const counts = {};
+  for (const a of actions) {
+    const k = `${a.reason}:${a.action}`;
+    counts[k] = (counts[k] ?? 0) + 1;
+  }
+  return { actions, counts, aliases: usedAliases, handwritten, rollups };
+}
+
+/**
+ * Refuse a path that leaves the vault — lexically, or through a symlinked
+ * directory on the way (#1513 review N1: `90-archive` as a symlink would have
+ * turned the archive move into a write anywhere on the host). The nearest
+ * EXISTING ancestor is realpath'd and must sit inside the realpath'd root.
+ */
+function assertInside(root, rel) {
+  const abs = resolve(root, rel);
+  if (!abs.startsWith(`${root}${sep}`)) throw new Error(`refusing ${rel}: outside the vault`);
+  const realRoot = realpathSync(root);
+  let probe = dirname(abs);
+  while (!existsSync(probe) && probe.startsWith(`${root}${sep}`)) probe = dirname(probe);
+  const realProbe = realpathSync(probe);
+  if (realProbe !== realRoot && !realProbe.startsWith(`${realRoot}${sep}`)) {
+    throw new Error(`refusing ${rel}: resolves outside the vault through a symlink`);
+  }
+  return abs;
+}
+
+function assertRegularFile(abs, rel) {
+  const st = lstatSync(abs);
+  if (st.isSymbolicLink() || !st.isFile()) throw new Error(`refusing ${rel}: not a regular file`);
+}
+
+/**
+ * Apply a plan. THE ONLY WRITE PATH. Rollups first, so a metrics-only session
+ * is archived only after its row exists. Re-checks the generator marker of
+ * every note right before touching it (the vault may have moved since the
+ * plan). Per-path failures are collected, never thrown.
+ *
+ * @param {ReturnType<typeof planPrune>} plan
+ * @param {{vaultDir: string}} ctx
+ * @returns {{archived: number, moved: number, rollupsWritten: number, suffixed: Array<{path: string, target: string, written: string}>, errors: Array<{path: string, error: string}>}}
+ */
+export function applyPrune(plan, { vaultDir }) {
+  const root = resolve(vaultDir);
+  const errors = [];
+  let archived = 0;
+  let moved = 0;
+  let rollupsWritten = 0;
+  /** @type {Array<{path: string, target: string, written: string}>} */
+  const suffixed = [];
+  const failedRollups = new Set();
+
+  for (const [rel, rows] of Object.entries(plan.rollups ?? {})) {
+    try {
+      const abs = assertInside(root, rel);
+      let existing = null;
+      if (existsSync(abs)) {
+        assertRegularFile(abs, rel);
+        existing = readFileSync(abs, 'utf8');
+        if (parseFrontmatter(existing)?.['_generator'] !== GENERATOR_MARKER) throw new Error('rollup exists and is not mirror-owned');
+      }
+      const ns = rel.split('/').length === 3 ? rel.split('/')[1] : '';
+      const month = rel.match(/_rollup-(\d{4}-\d{2})\.md$/)[1];
+      const next = renderRollup(existing, rows, { ns, month });
+      if (next !== existing) {
+        mkdirSync(dirname(abs), { recursive: true });
+        writeFileSync(abs, next, 'utf8');
+        rollupsWritten += 1;
+      }
+    } catch (err) {
+      failedRollups.add(rel);
+      errors.push({ path: rel, error: err?.message ?? String(err) });
+    }
+  }
+
+  for (const a of plan.actions ?? []) {
+    if (a.action !== 'archive' && a.action !== 'move') continue;
+    try {
+      if (a.rollup && failedRollups.has(a.rollup)) throw new Error(`rollup ${a.rollup} was not written`);
+      const src = assertInside(root, a.path);
+      const dst = assertInside(root, a.target);
+      assertRegularFile(src, a.path);
+      const content = readFileSync(src, 'utf8');
+      if (parseFrontmatter(content)?.['_generator'] !== GENERATOR_MARKER) throw new Error('no longer mirror-owned');
+      mkdirSync(dirname(dst), { recursive: true });
+      if (a.action === 'move') {
+        if (existsSync(dst)) throw new Error(`target exists: ${a.target}`);
+        renameSync(src, dst);
+        moved += 1;
+        continue;
+      }
+      const next = markArchived(content, a.reason);
+      let finalDst = dst;
+      if (existsSync(dst) && readFileSync(dst, 'utf8') !== next) {
+        // #1513 review N4: an occupied target with OTHER content (an earlier
+        // archive of a namesake) is never overwritten and never a permanent
+        // error: the note goes beside it, suffixed by its content hash, and
+        // the apply result names it. Same content → same suffix → idempotent.
+        const hash8 = createHash('sha256').update(next).digest('hex').slice(0, 8);
+        finalDst = join(dirname(dst), `${basename(dst, '.md')}.dup-${hash8}.md`);
+        suffixed.push({ path: a.path, target: a.target, written: `${dirname(a.target)}/${basename(finalDst)}` });
+      }
+      if (!existsSync(finalDst)) writeFileSync(finalDst, next, { encoding: 'utf8', flag: 'wx' });
+      else if (readFileSync(finalDst, 'utf8') !== next) throw new Error(`archive target exists with other content: ${finalDst}`);
+      unlinkSync(src);
+      archived += 1;
+    } catch (err) {
+      errors.push({ path: a.path, error: err?.message ?? String(err) });
+    }
+  }
+  return { archived, moved, rollupsWritten, suffixed, errors };
+}

package/scripts/lib/vault-mirror/render-sessions.mjs

@@ -17,6 +17,7 @@
 
 import { toDate, buildTag, slugifyIdSafe } from './utils.mjs';
 import { sourceRecordValue, sourceRecordLine } from './render-learnings.mjs';
+import { sessionNarrativeText } from './retention.mjs';
 
 const GENERATOR_MARKER = 'session-orchestrator-vault-mirror@1';
 
@@ -425,6 +426,9 @@ export function generateSessionNote(entry, options = {}) {
   const created = toDate(started_at);
   const updated = toDate(completed_at);
   const durationLabel = renderDuration(duration_seconds);
+  // #1513 review H2: v1 printed no free text at all; same block as v2/v3.
+  const v1Narrative = sessionNarrativeText(entry);
+  const v1NotesBlock = v1Narrative ? `\n## Notes\n\n${v1Narrative}\n` : '';
   // #M1: every `effectiveness` sub-field is OPTIONAL (validator.mjs
   // `_validateOptionalFields` shape-checks the object, never its members), so a
   // raw interpolation writes the literal string `undefined` into a note a human
@@ -564,7 +568,7 @@ ${waveRows}
 ## Agent summary
 
 - Complete: ${complete} · Partial: ${partial} · Failed: ${failed} · Spiral: ${spiral}
-`;
+${v1NotesBlock}`;
 }
 
 export function generateSessionNoteV2(entry, options = {}) {
@@ -580,7 +584,7 @@ export function generateSessionNoteV2(entry, options = {}) {
     throw new Error(`vault-mirror: session entry missing nested field 'effectiveness' (session_id=${entry.session_id})`);
   }
 
-  const { session_id, session_type, started_at, completed_at, duration_seconds, branch, planned_issues, waves, files_changed, issues_closed, issues_created, effectiveness, notes } = entry;
+  const { session_id, session_type, started_at, completed_at, duration_seconds, branch, planned_issues, waves, files_changed, issues_closed, issues_created, effectiveness } = entry;
 
   const created = toDate(started_at);
   const updated = toDate(completed_at);
@@ -616,7 +620,9 @@ export function generateSessionNoteV2(entry, options = {}) {
   const closedList = Array.isArray(issues_closed) && issues_closed.length ? issues_closed.join(', ') : '—';
   const createdList = Array.isArray(issues_created) && issues_created.length ? issues_created.join(', ') : '—';
   const branchLine = branch ? ` · **Branch:** ${branch}` : '';
-  const notesBlock = notes ? `\n## Notes\n\n${notes}\n` : '';
+  // #1513 review H2: notes + narrative + summary, the fields the gate counts.
+  const narrativeText = sessionNarrativeText(entry);
+  const notesBlock = narrativeText ? `\n## Notes\n\n${narrativeText}\n` : '';
 
   // #732: emit `source-repo` (leak-guarded/pseudonym-mapped) instead of the
   // legacy raw `repo` field — see generateSessionNote for the full rationale.
@@ -680,7 +686,7 @@ export function generateSessionNoteV3(entry, options = {}) {
     session_id, session_type, platform, branch, started_at, completed_at,
     duration_minutes, duration_seconds, waves, agents_dispatched, agent_summary,
     planned_issues, effectiveness, commits, issues_closed, issues_created,
-    follow_ups_filed, tests_total_pre, tests_total_post, tests_added, notes,
+    follow_ups_filed, tests_total_pre, tests_total_post, tests_added,
   } = entry;
 
   const created = toDate(started_at);
@@ -741,7 +747,9 @@ export function generateSessionNoteV3(entry, options = {}) {
 
   const platformBullet = platform === null || platform === undefined || platform === '' ? '' : ` · **Platform:** ${platform}`;
   const branchLine = branch ? ` · **Branch:** ${branch}` : '';
-  const notesBlock = notes ? `\n## Notes\n\n${notes}\n` : '';
+  // #1513 review H2: notes + narrative + summary, the fields the gate counts.
+  const narrativeText = sessionNarrativeText(entry);
+  const notesBlock = narrativeText ? `\n## Notes\n\n${narrativeText}\n` : '';
 
   // #732: emit `source-repo` (leak-guarded/pseudonym-mapped) instead of the
   // legacy raw `repo` field — see generateSessionNote for the full rationale.