session-orchestrator 5.4.0 → 5.5.0

package/scripts/lib/learnings/expiry-sweep.mjs

@@ -30,6 +30,11 @@
  *     ONLY destructive step, and it reuses `rewriteLearnings()` from io.mjs
  *     (#721), which itself snapshots a `.bak-<ISO>` backup before the atomic
  *     rename — the same safety net that protects every other bulk rewrite.
+ *     A store line that does not parse is carried through the rewrite
+ *     verbatim (#1489): neither partition can judge it, and the rewrite is
+ *     built from parsed records, so leaving it out deleted it with no archive
+ *     record. Refusing the rewrite instead would stall the session-end sweep
+ *     on every close until someone repaired the line by hand.
  *   - Grace period (default 14 days): an entry that JUST expired stays in the
  *     active store for `graceDays` more days. This absorbs two things: (a)
  *     TTL edge-noise near the boundary, and (b) a window for /evolve's
@@ -54,15 +59,82 @@
  *     (the incident that destroyed 107 live learnings via a bespoke writer
  *     with no backup/dry-run safety net).
  *
+ * #1500 — {@link dropMalformedLines}, the one sanctioned removal of an
+ * unparseable store line: the lines the two writers above keep verbatim.
+ *
  * Sibling-module import convention (learnings.mjs barrel doc): import
  * directly from `./io.mjs`, never from `../learnings.mjs`, to preserve the
  * acyclic dependency graph.
  */
 
-import { existsSync } from 'node:fs';
-import { mkdir, appendFile } from 'node:fs/promises';
+import { constants as fsConstants, existsSync } from 'node:fs';
+import { mkdir, appendFile, copyFile } from 'node:fs/promises';
 import path from 'node:path';
-import { readLearnings, rewriteLearnings, withLearningsLock } from './io.mjs';
+import { readLearnings, readLearningsSnapshot, rewriteLearnings, withLearningsLock } from './io.mjs';
+import { redactHomeDir } from '../common.mjs';
+
+/**
+ * Thrown by {@link pruneLearnings} when `expectedGeneration` no longer matches
+ * the store (#1486): the caller's next generation was derived from a store
+ * state that has since changed, so every record added after that derivation
+ * would read as caller-dropped and be archived `pruned`. Nothing is written.
+ */
+export class StoreGenerationMismatchError extends Error {
+  /**
+   * @param {string} filePath
+   * @param {string} expected - token the caller derived its generation from
+   * @param {string} actual - token of the store as read under the lock
+   */
+  constructor(filePath, expected, actual) {
+    // The tokens live on the object, never in the message: callers print the
+    // message, and the CURRENT token copied onto the stale sidecar's line 1 is
+    // exactly what gets that sidecar past this check — measured, the copy
+    // archived the peer's record `pruned`, exit 0.
+    super(`store-generation-mismatch: ${filePath} changed since the next generation was derived — nothing written`);
+    this.name = 'StoreGenerationMismatchError';
+    this.code = 'store-generation-mismatch';
+    this.filePath = filePath;
+    this.expected = expected;
+    this.actual = actual;
+  }
+}
+
+/**
+ * Thrown by {@link dropMalformedLines} when a requested line may not be
+ * dropped. Nothing is written. Two reasons:
+ *   - `not-unparseable`: a record, a blank line, or past the end. The message
+ *     names where the unparseable lines are NOW — every rewrite moves them to
+ *     the end, so a number from an earlier WARN is often stale (#1500 F3).
+ *   - `holds-a-record`: the line still carries a readable record (behind a
+ *     BOM, or fused after a torn one) and `acceptEmbedded` was not given.
+ */
+export class MalformedLineRefusedError extends Error {
+  /**
+   * @param {string} filePath
+   * @param {number[]} lines - the requested lines that may not be dropped
+   * @param {{ reason?: 'not-unparseable'|'holds-a-record', unparseable?: number[] }} [opts]
+   *   `unparseable`: the store's unparseable lines as read (for `not-unparseable`)
+   */
+  constructor(filePath, lines, { reason = 'not-unparseable', unparseable = [] } = {}) {
+    const where = `line ${lines.join(', ')} of ${redactHomeDir(filePath)}`;
+    const now =
+      unparseable.length === 0
+        ? 'none'
+        : unparseable.slice(0, 10).join(', ') + (unparseable.length > 10 ? `, +${unparseable.length - 10} more` : '');
+    super(
+      reason === 'holds-a-record'
+        ? `drop-malformed refused: ${where} still holds a readable record (see bom / embedded_record ` +
+            `in the dry run) — dropping it loses that record; nothing written. Pass --accept-embedded ` +
+            `to drop it anyway (the pre-drop snapshot keeps the line)`
+        : `drop-malformed refused: ${where} is not an unparseable line (a record, a blank line, or ` +
+            `past the end) — nothing written; unparseable lines now: ${now}`
+    );
+    this.name = 'MalformedLineRefusedError';
+    this.code = 'malformed-line-refused';
+    this.reason = reason;
+    this.lines = lines;
+  }
+}
 
 /**
  * Composite-key separator for the in-memory consolidation Map.
@@ -202,6 +274,65 @@ function countByReason(batch) {
   return out;
 }
 
+/**
+ * Result field for the store's unparseable lines: `{malformed: n}` when the
+ * read held `n > 0`, else `{}` — so the pinned zero-case result shape (the
+ * CLI's `--json` contract) is unchanged, and an absent field always means a
+ * read that found none. A non-zero count also goes to stderr, because a
+ * caller that forwards only its own fields (the session-end tail runner, the
+ * CLI's human-readable line) would otherwise drop it on the floor.
+ *
+ * @param {string} filePath
+ * @param {string[]} malformed - raw text of the lines that did not parse
+ * @param {number[]} lineNumbers - their 1-based store lines (`malformedLineNumbers`)
+ * @returns {{malformed?: number}}
+ */
+function malformedField(filePath, malformed, lineNumbers) {
+  if (malformed.length === 0) return {};
+  warnUnparseableLines(filePath, malformed, lineNumbers);
+  return { malformed: malformed.length };
+}
+
+/**
+ * Store + line set already WARNed about in this process. One `/close` reads the
+ * same store twice in one process — the planner's dry run, then the apply — and
+ * printed the identical WARN twice. Grows by one key per distinct store+line
+ * set, so it stays tiny; revisit if a long-lived process ever sweeps many stores.
+ * @type {Set<string>}
+ */
+const warnedUnparseable = new Set();
+
+/**
+ * Print the one stderr WARN for a store's unparseable lines — at most once per
+ * store and identical line set per process. The policy it states is the store
+ * policy since #1489: such lines are kept verbatim at the end of every rewrite,
+ * never archived, and never a reason to refuse a write.
+ *
+ * @param {string} filePath
+ * @param {string[]} malformed - raw text of the lines that did not parse
+ * @param {number[]} [lineNumbers] - their 1-based store lines; at most ten listed
+ *
+ * The text names no dry-run state: one WARN per process is shared by the
+ * planner's dry run and the apply that follows it in a /close, so a dry-run
+ * qualifier would be printed for — and contradict — the applied rewrite.
+ */
+export function warnUnparseableLines(filePath, malformed, lineNumbers = []) {
+  if (malformed.length === 0) return;
+  const key = `${path.resolve(filePath)}\0${malformed.join('\n')}`;
+  if (warnedUnparseable.has(key)) return;
+  warnedUnparseable.add(key);
+  const more = lineNumbers.length > 10 ? `, +${lineNumbers.length - 10} more` : '';
+  const where = lineNumbers.length > 0 ? ` (line(s) ${lineNumbers.slice(0, 10).join(', ')}${more})` : '';
+  console.error(
+    // The store path is absolute and lands in every /close transcript (#1490).
+    `[learnings] WARN: ${malformed.length} unparseable line(s) in ${redactHomeDir(filePath)}${where} kept verbatim, ` +
+      `never archived; report them to the operator — ` +
+      `never hand-edit the store. Once the operator decides, remove one with ` +
+      `\`node scripts/sweep-expired-learnings.mjs --drop-malformed --line N --file ${redactHomeDir(filePath)}\` ` +
+      `(dry run; it prints the --generation its --apply needs)`
+  );
+}
+
 /** Resolve the `now` parameter (Date | epoch ms | undefined) to epoch ms. */
 function resolveNowMs(now) {
   if (now instanceof Date) return now.getTime();
@@ -223,15 +354,32 @@ function resolveNowMs(now) {
  * @param {string} opts.archivePath
  * @param {object[]} opts.keep — the next store generation
  * @param {{entry: object, verdict: {reason: string, tombstone: object|null}}[]} opts.archiveBatch
+ * @param {string[]} opts.malformed — unparseable store lines, written back
+ *   verbatim after `keep` (see the module header)
+ * @param {number[]} opts.malformedLineNumbers — their 1-based store lines (WARN only)
  * @param {number} opts.nowMs
  * @param {boolean} opts.dryRun
- * @returns {Promise<{kept: number, archived: number, byReason: Record<string, number>}>}
+ * @returns {Promise<{kept: number, archived: number, byReason: Record<string, number>,
+ *   malformedField: {malformed?: number}}>}
  */
-async function archiveThenRewrite({ filePath, archivePath, keep, archiveBatch, nowMs, dryRun }) {
+async function archiveThenRewrite({
+  filePath,
+  archivePath,
+  keep,
+  archiveBatch,
+  malformed,
+  malformedLineNumbers,
+  nowMs,
+  dryRun,
+}) {
   const byReason = countByReason(archiveBatch);
-  if (dryRun) {
-    return { kept: keep.length, archived: archiveBatch.length, byReason };
-  }
+  const result = {
+    kept: keep.length,
+    archived: archiveBatch.length,
+    byReason,
+    malformedField: malformedField(filePath, malformed, malformedLineNumbers),
+  };
+  if (dryRun) return result;
 
   // Validate the KEEP batch BEFORE the archive append (dry-run rewrite: throws
   // on a bad record, writes nothing). Without this, an invalid-but-parseable
@@ -240,7 +388,7 @@ async function archiveThenRewrite({ filePath, archivePath, keep, archiveBatch, n
   // duplicates the archive append every time (never de-duplicated on read)
   // while the store is never actually pruned. Validating first means a bad
   // record throws here, before anything on disk has been touched.
-  await rewriteLearnings(filePath, keep, { dryRun: true });
+  await rewriteLearnings(filePath, keep, { dryRun: true, malformedLines: malformed });
 
   // Crash-safe ordering: archive append FIRST. A duplicate re-append after a
   // crash is harmless (append-only, never de-duplicated on read); the reverse
@@ -259,9 +407,9 @@ async function archiveThenRewrite({ filePath, archivePath, keep, archiveBatch, n
   // Re-validates (cheap, idempotent) and snapshots a `.bak-<ISO>` backup of
   // the current store before the atomic rename (io.mjs #721). The KEEP batch
   // already passed the dry-run probe above, so this call cannot throw here.
-  await rewriteLearnings(filePath, keep, { dryRun: false });
+  await rewriteLearnings(filePath, keep, { dryRun: false, malformedLines: malformed });
 
-  return { kept: keep.length, archived: archiveBatch.length, byReason };
+  return result;
 }
 
 /**
@@ -279,7 +427,10 @@ async function archiveThenRewrite({ filePath, archivePath, keep, archiveBatch, n
  * @param {Date|number} [opts.now] - injectable clock; defaults to `Date.now()`
  * @param {boolean} [opts.dryRun=true] - when true, computes counts but writes nothing
  * @param {number} [opts.graceDays=14] - days past expiry before an entry is archived
- * @returns {Promise<{scanned: number, kept: number, archived: number, dryRun: boolean, archivePath: string}>}
+ * @returns {Promise<{scanned: number, kept: number, archived: number, dryRun: boolean, archivePath: string,
+ *   malformed?: number}>} `malformed` is present only when the store held
+ *   unparseable lines: their count. They are kept verbatim at the end of the
+ *   rewritten store, never archived, and not counted in `scanned`/`kept`.
  *   Never throws on a missing store — returns the zeroed shape instead. Both
  *   `filePath` and `archivePath` are required; a missing/invalid `filePath`
  *   throws a plain `Error` (programmer error, not a runtime data condition).
@@ -321,7 +472,7 @@ async function sweepExpiredLearningsUnlocked({
   const graceMs =
     (Number.isFinite(graceDays) && graceDays >= 0 ? graceDays : DEFAULT_GRACE_DAYS) * MS_PER_DAY;
 
-  const { entries } = await readLearnings(filePath);
+  const { entries, malformed, malformedLineNumbers } = await readLearnings(filePath);
 
   const keep = [];
   const archiveBatch = [];
@@ -342,11 +493,13 @@ async function sweepExpiredLearningsUnlocked({
     }
   }
 
-  const { kept, archived } = await archiveThenRewrite({
+  const { kept, archived, malformedField: malformedCount } = await archiveThenRewrite({
     filePath,
     archivePath,
     keep,
     archiveBatch,
+    malformed,
+    malformedLineNumbers,
     nowMs,
     dryRun,
   });
@@ -354,7 +507,7 @@ async function sweepExpiredLearningsUnlocked({
   // Result shape is pinned by the CLI (`scripts/sweep-expired-learnings.mjs`)
   // and its JSON contract — deliberately WITHOUT `byReason`, which only the
   // multi-reason prune path below can populate meaningfully.
-  return { scanned: entries.length, kept, archived, dryRun, archivePath };
+  return { scanned: entries.length, kept, archived, dryRun, archivePath, ...malformedCount };
 }
 
 /**
@@ -411,8 +564,19 @@ async function sweepExpiredLearningsUnlocked({
  * @param {string|Function} [opts.dropReason='pruned'] - verdict for case 3
  *   above: a reason string, or `(entry) => reason|{reason, supersededBy, mergedInto}`
  *   for per-record routing (this is the seam a later `merged` producer uses).
+ * @param {string} [opts.expectedGeneration] - generation token
+ *   (`readLearningsSnapshot().generation`) of the store state `entries` was
+ *   derived from (#1486). Compared against the store as read here — under the
+ *   lock on apply — and a mismatch throws {@link StoreGenerationMismatchError}
+ *   before anything is written. Required whenever `entries` was derived in an
+ *   earlier read, outside the lock (the `/evolve` sidecar); a caller that
+ *   reads and prunes inside one `withLearningsLock` section may omit it.
  * @returns {Promise<{scanned: number, kept: number, archived: number,
- *   byReason: Record<string, number>, dryRun: boolean, archivePath: string}>}
+ *   byReason: Record<string, number>, dryRun: boolean, archivePath: string,
+ *   malformed?: number}>} `malformed` as in {@link sweepExpiredLearnings}: the
+ *   store's unparseable lines, kept verbatim — no caller generation can carry
+ *   them, so they are never read as caller drops.
+ * @throws {StoreGenerationMismatchError} when `expectedGeneration` is stale
  */
 export async function pruneLearnings(opts = {}) {
   const { filePath, dryRun = true } = opts;
@@ -437,6 +601,7 @@ async function pruneLearningsUnlocked({
   now,
   dryRun = true,
   dropReason = 'pruned',
+  expectedGeneration,
 } = {}) {
   if (typeof filePath !== 'string' || filePath.length === 0) {
     throw new Error('pruneLearnings: filePath is required');
@@ -447,13 +612,33 @@ async function pruneLearningsUnlocked({
   if (entries !== undefined && !Array.isArray(entries)) {
     throw new Error('pruneLearnings: entries must be an array when provided');
   }
+  if (
+    expectedGeneration !== undefined &&
+    (typeof expectedGeneration !== 'string' || expectedGeneration.length === 0)
+  ) {
+    throw new Error('pruneLearnings: expectedGeneration must be a non-empty string when provided');
+  }
 
   const nowMs = resolveNowMs(now);
-  const { entries: current } = await readLearnings(filePath);
+  // Token and records come from ONE read, so the comparison judges exactly the
+  // records loop (3) reconciles against.
+  const { entries: current, malformed, malformedLineNumbers, generation } = await readLearningsSnapshot(filePath);
+  if (expectedGeneration !== undefined && generation !== expectedGeneration) {
+    throw new StoreGenerationMismatchError(filePath, expectedGeneration, generation);
+  }
   const next = entries ?? current;
 
   if (next.length === 0 && current.length === 0) {
-    return { scanned: 0, kept: 0, archived: 0, byReason: {}, dryRun, archivePath };
+    // Nothing to write — the malformed lines, if any, stay where they are.
+    return {
+      scanned: 0,
+      kept: 0,
+      archived: 0,
+      byReason: {},
+      dryRun,
+      archivePath,
+      ...malformedField(filePath, malformed, malformedLineNumbers),
+    };
   }
 
   const resolveDrop = (entry) =>
@@ -537,14 +722,196 @@ async function pruneLearningsUnlocked({
     dropped += 1;
   }
 
-  const { kept, archived, byReason } = await archiveThenRewrite({
+  const { kept, archived, byReason, malformedField: malformedCount } = await archiveThenRewrite({
     filePath,
     archivePath,
     keep,
     archiveBatch,
+    malformed,
+    malformedLineNumbers,
     nowMs,
     dryRun,
   });
 
-  return { scanned: next.length + dropped, kept, archived, byReason, dryRun, archivePath };
+  return {
+    scanned: next.length + dropped,
+    kept,
+    archived,
+    byReason,
+    dryRun,
+    archivePath,
+    ...malformedCount,
+  };
+}
+
+/** Characters of a dropped line shown in the dry-run preview. */
+const DROP_PREVIEW_CHARS = 80;
+
+/** U+FEFF — built, not typed: a literal BOM in source fails validate-plugin's unicode-safety check. */
+const BOM = String.fromCharCode(0xfeff);
+
+/**
+ * The readable record an unparseable line may still carry (#1500 F1) — the
+ * line counts as malformed, yet dropping it would delete a learning:
+ *   - `bom`: the line is a JSON object behind a UTF-8 BOM (an editor's save);
+ *   - `embedded_record`: a `{` after the line's start (after the BOM, if any)
+ *     begins a suffix that parses as a JSON object — the #1489 fusion shape, a
+ *     torn record with the next complete record appended to it. The smallest
+ *     such offset wins; `id` is the record's id, or null.
+ *
+ * Ceiling (BV-004): one JSON.parse of the suffix per `{`, so quadratic in the
+ * line length — a few ms for the KB-sized lines a store holds; revisit if an
+ * unparseable line ever passes ~1 MB.
+ *
+ * @param {string} text - one unparseable store line
+ * @returns {{bom: boolean, embedded_record: {offset: number, id: string|null}|null}}
+ */
+function inspectMalformedLine(text) {
+  const asObject = (s) => {
+    try {
+      const v = JSON.parse(s);
+      return v !== null && typeof v === 'object' && !Array.isArray(v) ? v : null;
+    } catch {
+      return null;
+    }
+  };
+  const hasBom = text.startsWith(BOM);
+  const bom = hasBom && asObject(text.slice(1)) !== null;
+  let embedded = null;
+  for (let i = text.indexOf('{', hasBom ? 2 : 1); i !== -1; i = text.indexOf('{', i + 1)) {
+    const record = asObject(text.slice(i));
+    if (record !== null) {
+      embedded = { offset: i, id: typeof record.id === 'string' ? record.id : null };
+      break;
+    }
+  }
+  return { bom, embedded_record: embedded };
+}
+
+/**
+ * Where {@link dropMalformedLines} snapshots the store before its rewrite:
+ * `<stem>.pre-drop-malformed.jsonl.bak-<ISO>` beside it. The name is chosen
+ * twice over: `.orchestrator/metrics/*.jsonl.bak-*` gitignores it (here; in a
+ * consumer repo once bootstrap's `#store-lock-ignore` snippet has run), and it
+ * does NOT start with `<store>.bak`, so the keep-3 rotation of
+ * `rewriteLearnings()` never prunes it — the line's text outlives any number
+ * of later closes.
+ *
+ * @param {string} filePath
+ * @param {number} nowMs
+ * @returns {string}
+ */
+function dropSnapshotPath(filePath, nowMs) {
+  const base = path.basename(filePath);
+  const stem = base.endsWith('.jsonl') ? base.slice(0, -'.jsonl'.length) : base;
+  const ts = new Date(nowMs).toISOString().replace(/[:.]/g, '-');
+  return path.join(path.dirname(filePath), `${stem}.pre-drop-malformed.jsonl.bak-${ts}`);
+}
+
+/**
+ * Remove unparseable lines from a learnings store — the ONE sanctioned way
+ * (#1500). Every rewrite keeps such lines verbatim (#1489) and the WARN forbids
+ * hand edits, so without this the only removal was a library call.
+ *
+ * `lines` are 1-based store lines as the WARN or a dry run of this function
+ * reported them. Line numbers move: every rewrite puts the unparseable lines at
+ * the END of the store, so after the next /close sweep a number can name a
+ * different unparseable line. Hence an apply needs `expectedGeneration` — the
+ * `generation` a dry run returned — and the store read, the generation check,
+ * the line check, the snapshot and the rewrite all run under ONE store lock;
+ * read outside it, a record a peer appended meanwhile is lost to the rewrite.
+ *
+ * The rewrite goes through `rewriteLearnings()` (validation, `.bak-<ISO>`,
+ * atomic rename) after a full copy of the store to {@link dropSnapshotPath}.
+ *
+ * An unparseable line can still carry a readable record — fused after a torn
+ * one, or behind a BOM ({@link inspectMalformedLine}). The dry run flags it;
+ * an apply refuses such a line unless `acceptEmbedded` is set (#1500 F1): an
+ * 80-character preview hid a record at offset 250, and the apply deleted it.
+ *
+ * @param {object} opts
+ * @param {string} opts.filePath - the learnings store
+ * @param {number[]} opts.lines - 1-based lines to drop; each must be unparseable in the store as read
+ * @param {string} [opts.expectedGeneration] - generation the line numbers were read from;
+ *   REQUIRED to apply, checked on a dry run when given
+ * @param {boolean} [opts.acceptEmbedded=false] - apply even to a line that still holds a readable record
+ * @param {boolean} [opts.dryRun=true] - check and validate, write nothing
+ * @param {Date|number} [opts.now] - injectable clock (snapshot name)
+ * @returns {Promise<{generation: string, dropped: {line: number, chars: number, preview: string,
+ *   bom: boolean, embedded_record: {offset: number, id: string|null}|null}[], records: number,
+ *   remainingMalformed: number, recordBearing: number, dryRun: boolean, snapshot: string|null}>}
+ *   `generation` is the store's as read; `chars` the line's full length, `preview` its first
+ *   {@link DROP_PREVIEW_CHARS} characters with the home dir as `~`; `recordBearing` counts the
+ *   dropped lines that held a readable record
+ * @throws {StoreGenerationMismatchError} the store is not at `expectedGeneration` — nothing written
+ * @throws {MalformedLineRefusedError} a line is not unparseable, or (apply without
+ *   `acceptEmbedded`) still holds a readable record — nothing written
+ */
+export async function dropMalformedLines({
+  filePath,
+  lines,
+  expectedGeneration,
+  acceptEmbedded = false,
+  dryRun = true,
+  now,
+} = {}) {
+  if (typeof filePath !== 'string' || filePath.length === 0) {
+    throw new Error('dropMalformedLines: filePath is required');
+  }
+  if (!Array.isArray(lines) || lines.length === 0 || !lines.every((n) => Number.isInteger(n) && n > 0)) {
+    throw new Error('dropMalformedLines: lines must be a non-empty array of positive integers');
+  }
+  if (!dryRun && (typeof expectedGeneration !== 'string' || expectedGeneration.length === 0)) {
+    throw new Error('dropMalformedLines: expectedGeneration is required to apply');
+  }
+  const run = () => dropMalformedLinesUnlocked({ filePath, lines, expectedGeneration, acceptEmbedded, dryRun, now });
+  // An absent store has no line to drop — the body refuses without a lock.
+  return dryRun || !existsSync(filePath) ? run() : withLearningsLock(filePath, run);
+}
+
+/**
+ * Body of {@link dropMalformedLines}; the caller holds the store lock or runs
+ * a pass that cannot write.
+ */
+async function dropMalformedLinesUnlocked({ filePath, lines, expectedGeneration, acceptEmbedded, dryRun, now }) {
+  const snap = await readLearningsSnapshot(filePath);
+  if (expectedGeneration !== undefined && snap.generation !== expectedGeneration) {
+    throw new StoreGenerationMismatchError(filePath, expectedGeneration, snap.generation);
+  }
+  const wanted = [...new Set(lines)].sort((a, b) => a - b);
+  const notMalformed = wanted.filter((n) => !snap.malformedLineNumbers.includes(n));
+  if (notMalformed.length > 0) {
+    throw new MalformedLineRefusedError(filePath, notMalformed, { unparseable: snap.malformedLineNumbers });
+  }
+
+  const drop = new Set(wanted);
+  const keepMalformed = snap.malformed.filter((_, i) => !drop.has(snap.malformedLineNumbers[i]));
+  const dropped = wanted.map((line) => {
+    const text = snap.malformed[snap.malformedLineNumbers.indexOf(line)];
+    const cut = text.length > DROP_PREVIEW_CHARS ? `${text.slice(0, DROP_PREVIEW_CHARS)}…` : text;
+    return { line, chars: text.length, preview: redactHomeDir(cut), ...inspectMalformedLine(text) };
+  });
+  const recordBearing = dropped.filter((d) => d.bom || d.embedded_record !== null);
+  if (!dryRun && !acceptEmbedded && recordBearing.length > 0) {
+    throw new MalformedLineRefusedError(filePath, recordBearing.map((d) => d.line), { reason: 'holds-a-record' });
+  }
+  // Validate the next generation before anything touches disk, dry run too:
+  // a record the tolerant gate rejects throws here, before the snapshot.
+  await rewriteLearnings(filePath, snap.entries, { dryRun: true, malformedLines: keepMalformed });
+  const result = {
+    generation: snap.generation,
+    dropped,
+    records: snap.entries.length,
+    remainingMalformed: keepMalformed.length,
+    recordBearing: recordBearing.length,
+    dryRun,
+    snapshot: null,
+  };
+  if (dryRun) return result;
+
+  const snapshot = dropSnapshotPath(filePath, resolveNowMs(now));
+  // COPYFILE_EXCL: never overwrite an earlier snapshot.
+  await copyFile(filePath, snapshot, fsConstants.COPYFILE_EXCL);
+  await rewriteLearnings(filePath, snap.entries, { malformedLines: keepMalformed });
+  return { ...result, snapshot };
 }