session-orchestrator 5.3.0 → 5.5.0

package/scripts/lib/learnings/io.mjs

@@ -8,14 +8,20 @@
 import {
   readFile,
   writeFile,
+  appendFile,
   mkdir,
   rename,
   copyFile,
   readdir,
   unlink,
+  open,
 } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
+import { existsSync, realpathSync } from 'node:fs';
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { randomUUID } from 'node:crypto';
 import path from 'node:path';
+import { withFileLock } from '../file-lock.mjs';
+import { digestSha256 } from '../crypto-digest-utils.mjs';
 import {
   validateLearning,
   normalizeLearning,
@@ -24,6 +30,117 @@ import {
   ValidationError,
 } from './schema.mjs';
 
+// ---------------------------------------------------------------------------
+// Store lock (GitLab #1447 point 8)
+// ---------------------------------------------------------------------------
+
+/**
+ * How long a writer waits for the store lock before giving up — the
+ * `withFileLock` default. Every critical section here is one read plus one
+ * append or rewrite of a JSONL file (milliseconds), so ten seconds of
+ * contention means a stuck holder, not a slow one.
+ */
+export const LEARNINGS_LOCK_TIMEOUT_MS = 10000;
+
+/**
+ * Thrown when the learnings store lock cannot be acquired. Every writer takes
+ * the lock BEFORE it touches the store, so this error always means nothing was
+ * written. The message is one line and names the lock file.
+ */
+export class LearningsLockError extends Error {
+  /**
+   * @param {string} lockPath
+   * @param {'timeout'|'fs-error'} reason
+   * @param {number} timeoutMs
+   * @param {string} [detail]
+   */
+  constructor(lockPath, reason, timeoutMs, detail) {
+    const why =
+      reason === 'timeout'
+        ? `held by another writer, waited ${timeoutMs} ms`
+        : `${reason}${detail ? `: ${detail}` : ''}`;
+    super(`learnings store lock ${lockPath} not acquired (${why}) — nothing written`);
+    this.name = 'LearningsLockError';
+    this.lockPath = lockPath;
+    this.reason = reason;
+  }
+}
+
+/**
+ * Store locks held by the CURRENT async call chain, so a nested writer inside a
+ * critical section (apply → prune → rewrite, sweep → rewrite, promote →
+ * rewrite) runs straight through instead of waiting on its own lock until the
+ * timeout. Keyed per async context, deliberately NOT per process: a concurrent
+ * call chain in the same process does not inherit the entry and waits like a
+ * second process would — which is exactly the interleaving the lock exists for.
+ * @type {AsyncLocalStorage<Map<string, {live: boolean}>>}
+ */
+const heldStoreLocks = new AsyncLocalStorage();
+
+/**
+ * `<store>.lock` beside the store. The directory is canonicalised so two
+ * spellings of one store (`/var` vs `/private/var`, a realpath'd repo root as
+ * `memory-proposals/sink.mjs` passes it) contend on ONE lock file. Creates the
+ * directory: the lock file needs it, and every caller is about to write there.
+ *
+ * @param {string} filePath
+ * @returns {Promise<string>}
+ */
+async function resolveStoreLockPath(filePath) {
+  const dir = path.dirname(path.resolve(filePath));
+  await mkdir(dir, { recursive: true });
+  return path.join(realpathSync(dir), `${path.basename(filePath)}.lock`);
+}
+
+/**
+ * Run `fn` while holding the exclusive lock of the learnings store at
+ * `filePath`. Every read-modify-write of the store must run its WHOLE
+ * read → write sequence inside one call: a write outside the lock is silently
+ * lost when a rewriter that read before it renames its next generation over
+ * the file. `appendLearning()` and `rewriteLearnings()` take the lock
+ * themselves.
+ *
+ * Reentrant within one async call chain (see `heldStoreLocks`). Reentrancy
+ * prevents a deadlock, not a lost write: an `appendLearning()` issued INSIDE a
+ * read → rewrite section lands in the file that section is about to replace —
+ * fold such a record into the rewrite batch instead.
+ *
+ * Crash safety comes from `withFileLock`'s default `staleCheck: 'pid'`: a lock
+ * whose holder process is gone is overridden (WARN on stderr); a live holder or
+ * one on another host is waited for until `timeoutMs`.
+ *
+ * @template T
+ * @param {string} filePath — the learnings store (the lock file sits beside it)
+ * @param {() => (T | Promise<T>)} fn
+ * @param {{ timeoutMs?: number }} [opts]
+ * @returns {Promise<T>}
+ * @throws {LearningsLockError} when the lock is not acquired — `fn` never ran
+ */
+export async function withLearningsLock(filePath, fn, { timeoutMs = LEARNINGS_LOCK_TIMEOUT_MS } = {}) {
+  const lockPath = await resolveStoreLockPath(filePath);
+  const held = heldStoreLocks.getStore();
+  if (held?.get(lockPath)?.live === true) return fn();
+
+  const token = { live: true };
+  const scope = new Map(held ?? []);
+  scope.set(lockPath, token);
+  let result;
+  try {
+    result = await withFileLock(lockPath, () => heldStoreLocks.run(scope, fn), {
+      timeoutMs,
+      // A per-acquisition holder makes the owner-guarded release delete only
+      // THIS acquisition's lock, never a successor's (#1285).
+      holder: `learnings-store:${process.pid}:${randomUUID()}`,
+    });
+  } finally {
+    // A continuation `fn` left running past its own return must not keep
+    // treating the released lock as held.
+    token.live = false;
+  }
+  if (!result.ok) throw new LearningsLockError(lockPath, result.reason, timeoutMs, result.error);
+  return result.value;
+}
+
 // ---------------------------------------------------------------------------
 // Pre-write self-validation seam (issue #662)
 // ---------------------------------------------------------------------------
@@ -42,7 +159,7 @@ import {
  * append touches disk, so a bad write can never reach the file.
  *
  * @param {object} validated — already validated+normalized learning entry
- * @param {{ legacyTolerant?: boolean }} [opts] — GitLab #386. When `true`,
+ * @param {{ legacyTolerant?: boolean }} [opts] — EventDrop #386. When `true`,
  *   a field that was ALREADY ABSENT on `validated` (e.g. a legacy record with
  *   no `source_session`, tolerated by `readLearnings()`) stays tolerated after
  *   the round-trip too — the re-validation call below runs in the same
@@ -80,8 +197,9 @@ function serializeLearningLineChecked(validated, { legacyTolerant = false } = {}
     // A key that existed on `validated` (present, even as `undefined`) but
     // vanished from `reparsed` was DROPPED by JSON.stringify — the exact
     // undefined/NaN/etc. corruption #662 exists to catch. A key that was
-    // never on `validated` in the first place (the #386 legacy-field case)
-    // cannot appear here, because we only iterate `validated`'s own keys.
+    // never on `validated` in the first place (the EventDrop #386
+    // legacy-field case) cannot appear here, because we only iterate
+    // `validated`'s own keys.
     const droppedKeys = Object.keys(validated).filter((k) => !(k in reparsed));
     if (droppedKeys.length > 0) {
       throw new ValidationError(
@@ -103,26 +221,69 @@ function serializeLearningLineChecked(validated, { legacyTolerant = false } = {}
 /**
  * Read all learnings from the given JSONL path. Returns normalized entries
  * (missing extended fields are defaulted). Malformed lines are skipped with
- * their raw text preserved in the result's `malformed` array.
+ * their raw text preserved in the result's `malformed` array, and their
+ * 1-based line numbers in `malformedLineNumbers` (same order).
  *
  * @param {string} filePath — absolute or project-relative path to learnings.jsonl
- * @returns {Promise<{entries: object[], malformed: string[]}>}
+ * @returns {Promise<{entries: object[], malformed: string[], malformedLineNumbers: number[]}>}
  */
 export async function readLearnings(filePath) {
-  if (!existsSync(filePath)) return { entries: [], malformed: [] };
-  const raw = await readFile(filePath, 'utf8');
-  const lines = raw.split('\n').filter((l) => l.length > 0);
+  if (!existsSync(filePath)) return { entries: [], malformed: [], malformedLineNumbers: [] };
+  return parseLearningsText(await readFile(filePath, 'utf8'));
+}
+
+/**
+ * Read a learnings store together with its generation token (#1486), both
+ * from ONE read, so the token describes exactly the records returned.
+ *
+ * The token is `sha256:<hex>` over the file's full text: any append, rewrite
+ * or sweep changes it. An absent store reads as empty text — zero records
+ * either way, so the two states share a token.
+ *
+ * @param {string} filePath
+ * @returns {Promise<{entries: object[], malformed: string[], malformedLineNumbers: number[],
+ *   generation: string}>}
+ */
+export async function readLearningsSnapshot(filePath) {
+  const raw = existsSync(filePath) ? await readFile(filePath, 'utf8') : '';
+  return { ...parseLearningsText(raw), generation: `sha256:${digestSha256(raw)}` };
+}
+
+/**
+ * Parse learnings JSONL text — the line parser behind {@link readLearnings},
+ * exported for callers that must split a header off the text first.
+ *
+ * A blank line — empty, whitespace-only, or a lone `\r` from a CRLF blank line
+ * — carries no record and is skipped, never counted as malformed: malformed
+ * lines are kept verbatim by every rewrite and WARNed on every close, so a
+ * stray blank would otherwise be reported forever (#1489).
+ *
+ * A line that parses but is not a JSON object (`null`, `42`, `[]`, `"x"`) is
+ * malformed too (#1500): `normalizeLearning` turned it into a record of
+ * defaults alone, and every rewrite wrote that invented record back.
+ *
+ * @param {string} raw
+ * @returns {{entries: object[], malformed: string[], malformedLineNumbers: number[]}}
+ *   `malformedLineNumbers[i]` is the 1-based line of `malformed[i]` in `raw`
+ */
+export function parseLearningsText(raw) {
   const entries = [];
   const malformed = [];
-  for (const line of lines) {
+  const malformedLineNumbers = [];
+  raw.split('\n').forEach((line, i) => {
+    if (line.trim().length === 0) return;
     try {
       const parsed = JSON.parse(line);
+      if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        throw new TypeError('not a JSON object');
+      }
       entries.push(normalizeLearning(parsed));
     } catch {
       malformed.push(line);
+      malformedLineNumbers.push(i + 1);
     }
-  }
-  return { entries, malformed };
+  });
+  return { entries, malformed, malformedLineNumbers };
 }
 
 // ---------------------------------------------------------------------------
@@ -143,11 +304,23 @@ export async function readLearnings(filePath) {
  * atomic on POSIX append. For very large insight/evidence fields that
  * might exceed that boundary, use rewriteLearnings() instead.
  *
+ * The append runs under the store lock ({@link withLearningsLock}): an append
+ * that landed between a rewriter's read and its rename was silently lost.
+ * Validation runs first, outside the lock — a rejected record never waits.
+ *
+ * A store whose last line is torn (a crash mid-append left no trailing
+ * newline) gets a `\n` written first, inside the same append: without it the
+ * new record fused onto the torn text into ONE unparseable line, and no reader
+ * ever saw the record (#1489).
+ *
  * @param {string} filePath
  * @param {object} entry
+ * @param {{ lockTimeoutMs?: number }} [opts] — store-lock wait, default
+ *   {@link LEARNINGS_LOCK_TIMEOUT_MS}
  * @returns {Promise<object>} validated entry
+ * @throws {LearningsLockError} when the store lock is not acquired (nothing appended)
  */
-export async function appendLearning(filePath, entry) {
+export async function appendLearning(filePath, entry, { lockTimeoutMs } = {}) {
   // Ensure created_at is set first — many writers omit it, and expires_at
   // derivation depends on it. Use ISO 8601 UTC.
   const createdAt =
@@ -173,12 +346,46 @@ export async function appendLearning(filePath, entry) {
   // parses back AND re-validates before any append touches disk. Throws
   // ValidationError on a non-round-tripping record — file is left untouched.
   const line = serializeLearningLineChecked(validated);
-  await mkdir(path.dirname(filePath), { recursive: true });
-  const { appendFile } = await import('node:fs/promises');
-  await appendFile(filePath, line, 'utf8');
+  // The lock path resolution creates the parent directory. The torn-tail check
+  // reads inside the lock, so no other writer can change the tail in between.
+  await withLearningsLock(
+    filePath,
+    async () => {
+      const lead = (await endsWithoutNewline(filePath)) ? '\n' : '';
+      await appendFile(filePath, lead + line, 'utf8');
+    },
+    { timeoutMs: lockTimeoutMs }
+  );
   return validated;
 }
 
+/**
+ * True when `filePath` is a non-empty file whose last byte is not `\n`. An
+ * absent file is `false` (the append creates it). Reads one byte, so the cost
+ * does not grow with the store.
+ *
+ * @param {string} filePath
+ * @returns {Promise<boolean>}
+ */
+async function endsWithoutNewline(filePath) {
+  let fh;
+  try {
+    fh = await open(filePath, 'r');
+  } catch (err) {
+    if (err?.code === 'ENOENT') return false;
+    throw err;
+  }
+  try {
+    const { size } = await fh.stat();
+    if (size === 0) return false;
+    const buf = Buffer.alloc(1);
+    await fh.read(buf, 0, 1, size - 1);
+    return buf[0] !== 0x0a;
+  } finally {
+    await fh.close();
+  }
+}
+
 /**
  * Number of `${basename}.bak-<ISO>` siblings retained after a backup rotation.
  * The atomic rewrite is destructive on a gitignored store (no VCS restore), so
@@ -281,7 +488,7 @@ async function rotateBackups(dir, baseName, keep = BACKUP_KEEP) {
  * - `backup` (default `true`): before the destructive rename, copy the current
  *   file to `${filePath}.bak-<ISO>`, then rotate to keep only the newest
  *   {@link BACKUP_KEEP}. Rotation is best-effort and never blocks the rewrite.
- * - `legacyTolerant` (default `true`, GitLab #386): this function is a
+ * - `legacyTolerant` (default `true`, EventDrop #386): this function is a
  *   ROUND-TRIP writer — its usual caller (`sweepExpiredLearnings` /
  *   `pruneLearnings` in `expiry-sweep.mjs`) reads the store with
  *   `readLearnings()` first, and that reader already tolerates a legacy
@@ -298,18 +505,36 @@ async function rotateBackups(dir, baseName, keep = BACKUP_KEEP) {
  *   round-trip (a value JSON.stringify drops or coerces, e.g. `undefined`/
  *   `NaN`) is still caught by the #662 checked serializer regardless of this
  *   flag — see {@link serializeLearningLineChecked}. Pass `false` to restore
- *   the pre-#386 fully-strict behaviour.
+ *   the fully-strict behaviour from before EventDrop #386.
+ * - `malformedLines` (#1489): raw text of store lines that did not parse —
+ *   the `malformed` array {@link readLearnings} returns. Written back
+ *   verbatim, one per line, AFTER the validated records. A round-trip writer
+ *   that rewrites from parsed entries alone deletes every such line with no
+ *   archive record (only the keep-3 `.bak` still holds it). OMITTED (the
+ *   default), the store's OWN malformed lines are re-read under the lock and
+ *   kept — so a caller that forgets this option no longer deletes them
+ *   (`export-hw-learnings` did). Pass an array to write exactly those lines;
+ *   `[]` is the explicit opt-out that drops them. Each entry must be a
+ *   non-empty string without a newline; anything else throws a TypeError
+ *   before any disk access, dry run included.
  *
  * @param {string} filePath
  * @param {object[]} entries
- * @param {{dryRun?: boolean, backup?: boolean, legacyTolerant?: boolean}} [opts]
+ * @param {{dryRun?: boolean, backup?: boolean, legacyTolerant?: boolean, malformedLines?: string[]}} [opts]
  * @returns {Promise<object[]>} validated entries (always returned, even dryRun)
  */
 export async function rewriteLearnings(
   filePath,
   entries,
-  { dryRun = false, backup = true, legacyTolerant = true } = {}
+  { dryRun = false, backup = true, legacyTolerant = true, malformedLines } = {}
 ) {
+  if (
+    malformedLines !== undefined &&
+    (!Array.isArray(malformedLines) ||
+      malformedLines.some((l) => typeof l !== 'string' || l.length === 0 || l.includes('\n')))
+  ) {
+    throw new TypeError('rewriteLearnings: malformedLines must be an array of non-empty single-line strings');
+  }
   const validated = entries.map((e) =>
     validateLearning(
       {
@@ -330,25 +555,37 @@ export async function rewriteLearnings(
   // rewrite, no backup — and hand the validated entries back to the caller.
   if (dryRun) return validated;
 
-  const body = lines.join('');
-  await mkdir(path.dirname(filePath), { recursive: true });
+  // Under the store lock (#1447 point 8) so no append lands between the backup
+  // and the rename. The CALLER's read must sit inside the same lock for a
+  // read-modify-write to be safe — this call joins it reentrantly.
+  await withLearningsLock(filePath, async () => {
+    // Default: the store's own malformed lines as they stand right now, read
+    // under the lock that guards the rename below.
+    const kept =
+      malformedLines ??
+      (existsSync(filePath) ? parseLearningsText(await readFile(filePath, 'utf8')).malformed : []);
+    // Malformed lines last, each newline-terminated: a truncated final line
+    // otherwise fuses with the next append into one more unparseable line.
+    const body = lines.join('') + kept.map((l) => `${l}\n`).join('');
 
-  // Backup-before-rewrite (#721): snapshot the current store to a timestamped
-  // sidecar BEFORE the destructive rename, then prune to keep-N. Only meaningful
-  // when the target already exists (a first-time write has nothing to lose).
-  if (backup && existsSync(filePath)) {
-    const ts = new Date().toISOString().replace(/[:.]/g, '-');
-    await copyFile(filePath, `${filePath}.bak-${ts}`);
-    try {
-      await rotateBackups(path.dirname(filePath), path.basename(filePath));
-    } catch {
-      // Rotation is best-effort — a stale/undeletable sibling must never abort
-      // the rewrite. The fresh backup above is already safely on disk.
+
+    // Backup-before-rewrite (#721): snapshot the current store to a timestamped
+    // sidecar BEFORE the destructive rename, then prune to keep-N. Only meaningful
+    // when the target already exists (a first-time write has nothing to lose).
+    if (backup && existsSync(filePath)) {
+      const ts = new Date().toISOString().replace(/[:.]/g, '-');
+      await copyFile(filePath, `${filePath}.bak-${ts}`);
+      try {
+        await rotateBackups(path.dirname(filePath), path.basename(filePath));
+      } catch {
+        // Rotation is best-effort — a stale/undeletable sibling must never abort
+        // the rewrite. The fresh backup above is already safely on disk.
+      }
     }
-  }
 
-  const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`;
-  await writeFile(tmp, body, 'utf8');
-  await rename(tmp, filePath);
+    const tmp = `${filePath}.tmp-${process.pid}-${Date.now()}`;
+    await writeFile(tmp, body, 'utf8');
+    await rename(tmp, filePath);
+  });
   return validated;
 }

package/scripts/lib/learnings/schema.mjs

@@ -230,17 +230,19 @@ export class ValidationError extends Error {
  * Throws ValidationError on contract violations. Does NOT mutate input.
  *
  * @param {object} entry — candidate learning
- * @param {{ legacyTolerant?: boolean }} [opts] — GitLab #386. When `true`, a
+ * @param {{ legacyTolerant?: boolean }} [opts] — EventDrop #386. When `true`, a
  *   value `normalizeLearning()` already passes through UNCHECKED (or merely
  *   DEFAULTED, never validated) on read is no longer rejected here either:
  *   an out-of-enum `schema_version` (measured against the real EventDrop.at
  *   store: 3 live records carry `schema_version: 2`), the
  *   LEGACY_REQUIRED_FIELDS presence check, the `confidence` range/type check
  *   when NO `confidence` key is present, and the `scope`/`host_class`/
- *   `anonymized` shape checks (measured live: a record with `scope:
- *   "src/components/landing/mobile-sticky-cta.tsx"` — a producer bug that
- *   wrote a file path into the scope column, which `normalizeLearning` never
- *   validates and therefore never rejects). A field that IS present keeps
+ *   `anonymized`/`file_paths` shape checks (measured live: a record whose `scope` held a
+ *   file path — e.g. a repo-relative component path such as
+ *   `src/components/sample/sticky-cta.tsx` (the measured value is not
+ *   reproduced here) — a producer bug that wrote a file path into the scope
+ *   column, which `normalizeLearning` never validates and therefore never
+ *   rejects). A field that IS present keeps
  *   being validated regardless when its DATA TYPE can be silently corrupted
  *   by a JSON round-trip (a present-but-malformed `confidence` still throws
  *   — see io.mjs #662): this option relaxes "you must HAVE a valid value",
@@ -296,9 +298,11 @@ export function validateLearning(entry, { legacyTolerant = false } = {}) {
   // scope/host_class/anonymized: `normalizeLearning()` only DEFAULTS an
   // absent value (`d.scope ?? 'local'`, etc.) on read — it never validates a
   // PRESENT one, so a corrupt legacy value round-trips silently today
-  // (measured live in production: a record with `scope:
-  // "src/components/landing/mobile-sticky-cta.tsx"` — evidently a producer
-  // bug that wrote a file path into the scope column). All three checks are
+  // (measured live in production: a record whose `scope` held a file path —
+  // e.g. a repo-relative component path such as
+  // `src/components/sample/sticky-cta.tsx` (the measured value is not
+  // reproduced here) — evidently a producer bug that wrote a file path into
+  // the scope column). All three checks are
   // therefore skipped under legacyTolerant. Unlike `confidence` (a number,
   // where `NaN`/`Infinity` are non-JSON-safe and silently become `null`
   // across a JSON round-trip — the #662 concern), a string/string-or-null/
@@ -319,6 +323,24 @@ export function validateLearning(entry, { legacyTolerant = false } = {}) {
     throw new ValidationError(`anonymized must be boolean, got: ${typeof anonymized}`);
   }
 
+  // file_paths (GitLab #1447): when present, an array of repo-relative path
+  // strings — a bare string (`file_paths: "src/x.ts"`) is the producer slip
+  // this rejects; every reader iterates it as a list. `null` reads as absent,
+  // as in migrateLegacyLearning. Skipped under legacyTolerant like the shape
+  // checks above: the read funnel passes the value through unchecked.
+  const filePaths = entry.file_paths;
+  if (!legacyTolerant && filePaths !== undefined && filePaths !== null) {
+    if (!Array.isArray(filePaths)) {
+      throw new ValidationError(`file_paths must be an array of repo-relative paths, got: ${typeof filePaths}`);
+    }
+    const badIndex = filePaths.findIndex((p) => !isRepoRelativePathEntry(p));
+    if (badIndex !== -1) {
+      throw new ValidationError(
+        `file_paths[${badIndex}] must be a non-empty repo-relative path (not absolute, no '..' segment), got: ${JSON.stringify(filePaths[badIndex])}`
+      );
+    }
+  }
+
   // Privacy contract — NEVER relaxed, legacyTolerant or not: an out-of-enum
   // `scope` (garbage or otherwise) is by definition not `'public'`, so this
   // pair only ever fires for a record that genuinely claims `scope: 'public'`
@@ -476,6 +498,85 @@ export function normalizeDialects(entry, { reserializeTimestamps = true } = {})
   return out;
 }
 
+/**
+ * File extensions a path-like `scope` value must end in to be treated as a
+ * repo-relative file path (GitHub #69 / GitLab #1446). Deliberately a closed
+ * list: a looser "contains a slash and a dot" test would also swallow values
+ * such as `infrastructure/docker-compose` or `node.js`.
+ */
+const PATH_SCOPE_EXT = /\.(mjs|cjs|js|jsx|ts|tsx|mts|cts|json|jsonl|md|mdx|ya?ml|toml|css|scss|html|vue|svelte|astro|py|rb|go|rs|swift|kt|java|sh|sql|prisma)$/i;
+
+/**
+ * Strict predicate: is `s` a repo-relative file path (at least one directory
+ * segment, no `..`, no empty segment, a known source/doc extension)? Rejects
+ * canonical scopes, absolute and home-relative paths, Windows drive paths,
+ * URLs, and anything containing whitespace.
+ *
+ * @param {unknown} s — candidate `scope` value
+ * @returns {boolean}
+ */
+function looksLikeRepoRelativePath(s) {
+  if (typeof s !== 'string' || VALID_SCOPES.includes(s) || s.length > 512) return false;
+  if (/\s/.test(s) || /^[/~]/.test(s) || /^[A-Za-z]:[\\/]/.test(s) || s.includes('://')) return false;
+  const segs = s.split('/');
+  if (segs.length < 2 || segs.includes('..') || segs.includes('')) return false;
+  return PATH_SCOPE_EXT.test(segs[segs.length - 1]);
+}
+
+/**
+ * Loose predicate for ONE `file_paths` entry (GitLab #1447): a non-empty string
+ * that is not absolute (`/`, `\`, `~`, a Windows drive), not a URL, and has no
+ * `..` segment. Deliberately looser than {@link looksLikeRepoRelativePath}:
+ * root-level files (`package.json`), directories (`docs/prd`) and globs
+ * (`skills/**`) are legitimate `file_paths` values.
+ *
+ * Exported because `scripts/memory-propose.mjs` gates `--file-paths` on the same
+ * rule: a proposal this predicate rejects would be queued, approved, and then
+ * refused by {@link validateLearning} at write time.
+ *
+ * @param {unknown} s — candidate entry
+ * @returns {boolean}
+ */
+export function isRepoRelativePathEntry(s) {
+  if (typeof s !== 'string' || s.trim() === '') return false;
+  if (/^[/\\~]/.test(s) || /^[A-Za-z]:[\\/]/.test(s) || s.includes('://')) return false;
+  return !s.split(/[\\/]/).includes('..');
+}
+
+const LEADING_DOT_SLASH = /^(?:\.\/)+/;
+
+/** Non-canonical STRING scopes {@link migrateLegacyLearning} coerces to `'local'`. */
+const LOCAL_ALIAS_SCOPES = new Set(['vault-tools', 'deep-sessions', 'wave-executor', 'coordinator']);
+
+/** Non-canonical STRING scopes {@link migrateLegacyLearning} coerces to `'private'`. */
+const PRIVATE_ALIAS_SCOPES = new Set(['project', 'repo']);
+
+/**
+ * Every word the migration reads as scope vocabulary. An array-scope entry
+ * equal to one of these is a scope word, never a path — the same reading the
+ * string branch gives it, so `['project']` is not turned into
+ * `file_paths: ['project']`.
+ */
+const SCOPE_WORDS = new Set([...VALID_SCOPES, ...LOCAL_ALIAS_SCOPES, ...PRIVATE_ALIAS_SCOPES]);
+
+/**
+ * The repo-relative paths a broken `scope` value carries, or `null` when it is
+ * not a path scope. A single string must pass the strict
+ * {@link looksLikeRepoRelativePath} (a lone string may be a scope word); an
+ * ARRAY is never scope vocabulary, so the loose per-entry predicate suffices
+ * there — an empty array yields `[]`. A leading `./` run is stripped from each
+ * path; case is kept (git paths are case-sensitive).
+ *
+ * @param {unknown} scope
+ * @returns {string[] | null}
+ */
+function pathScopeEntries(scope) {
+  if (looksLikeRepoRelativePath(scope)) return [scope.replace(LEADING_DOT_SLASH, '')];
+  if (!Array.isArray(scope)) return null;
+  const entries = scope.map((e) => (typeof e === 'string' ? e.replace(LEADING_DOT_SLASH, '') : e));
+  return entries.every((e) => isRepoRelativePathEntry(e) && !SCOPE_WORDS.has(e)) ? entries : null;
+}
+
 /**
  * Migrate a legacy learning record to the canonical schema_version:1 shape.
  * Idempotent — calling it on an already-canonical record is a safe no-op.
@@ -483,9 +584,32 @@ export function normalizeDialects(entry, { reserializeTimestamps = true } = {})
  * Alias precedence for insight: insight > description > recommendation > observation > lesson
  *
  * Producer dialects (`files`→`file_paths`, duplicate `session_id`, `last_seen`,
- * `next_review: null`) are normalized by {@link normalizeDialects} as the final
- * step. Timestamp FORMAT is deliberately NOT reformatted here (byte-exact
- * contract preserved); the read funnel + backfill canonicalize timestamps.
+ * `next_review: null`) are normalized by {@link normalizeDialects}. Timestamp
+ * FORMAT is deliberately NOT reformatted here (byte-exact contract preserved);
+ * the read funnel + backfill canonicalize timestamps.
+ *
+ * Broken-consumer-record coercions (GitHub #69 / GitLab #1446 — measured in a
+ * consumer store: 124 of 307 records failed strict validateLearning, 76 on
+ * `scope`, 48 on `schema_version`). Applied AFTER normalizeDialects, because
+ * coercing a path-like scope before the `files`→`file_paths` move would lose
+ * the legacy `files` data. Only this MIGRATION path coerces — the read funnel
+ * (normalizeLearning / normalizeDialects) stays pass-through by design:
+ *   - path-like `scope` (a repo-relative file path a producer wrote into the
+ *     scope column, see {@link looksLikeRepoRelativePath}) → appended to
+ *     `file_paths` (copied, never mutated in place; created when absent/null;
+ *     no duplicate) and `scope: 'private'`. A present non-array `file_paths`
+ *     leaves the record untouched — it stays invalid and is counted. A
+ *     leading `./` run is stripped first (GitLab #1447).
+ *   - array-typed `scope` (GitLab #1447) → the same move when every entry is a
+ *     repo-relative path ({@link isRepoRelativePathEntry}: root files, dirs and
+ *     globs included); an empty array → `scope: 'private'`. An array holding an
+ *     absolute path, a non-string or a scope word — canonical or one of the
+ *     aliases below, e.g. `['project']` — stays untouched (invalid, counted);
+ *     nothing is ever dropped.
+ *   - `scope: 'project'` / `scope: 'repo'` → `'private'`.
+ *   - `schema_version: '1'` (string) or `2` (number) → `1`.
+ * Each rule's output never re-matches that rule, so the whole function stays
+ * idempotent.
  *
  * The caller MUST still run validateLearning() on the result to confirm the
  * migrated record passes the full schema gate before writing.
@@ -534,8 +658,7 @@ export function migrateLegacyLearning(entry) {
     out.evidence = '';
   }
 
-  const COERCIBLE_SCOPES = new Set(['vault-tools', 'deep-sessions', 'wave-executor', 'coordinator']);
-  if (out.scope && !VALID_SCOPES.includes(out.scope) && COERCIBLE_SCOPES.has(out.scope)) {
+  if (out.scope && !VALID_SCOPES.includes(out.scope) && LOCAL_ALIAS_SCOPES.has(out.scope)) {
     out.scope = 'local';
   }
 
@@ -560,12 +683,37 @@ export function migrateLegacyLearning(entry) {
     out.schema_version = CURRENT_SCHEMA_VERSION;
   }
 
-  // Normalize producer dialects last (files→file_paths, session_id/last_seen/
+  // Normalize producer dialects (files→file_paths, session_id/last_seen/
   // next_review reconciliation) — schema_version is preserved by normalizeDialects.
   // reserializeTimestamps:false keeps migrateLegacyLearning's byte-exact timestamp
   // contract (it does schema/alias migration, not timestamp reformatting). The
   // READ funnel (normalizeLearning) and the backfill canonicalize timestamp format.
-  return normalizeDialects(out, { reserializeTimestamps: false });
+  const migrated = normalizeDialects(out, { reserializeTimestamps: false });
+
+  // Broken-consumer-record coercions (GitHub #69 / GitLab #1446, #1447) — AFTER
+  // normalizeDialects so a legacy `files` list has already become `file_paths`.
+  const pathEntries = pathScopeEntries(migrated.scope);
+  if (pathEntries !== null) {
+    const existing = migrated.file_paths;
+    if (existing === undefined || existing === null || Array.isArray(existing)) {
+      // Copy: normalizeDialects' shallow copy still shares the caller's array.
+      const filePaths = Array.isArray(existing) ? [...existing] : [];
+      for (const p of pathEntries) {
+        if (!filePaths.includes(p)) filePaths.push(p);
+      }
+      if (filePaths.length > 0) migrated.file_paths = filePaths;
+      migrated.scope = 'private';
+    }
+    // Present but not an array → leave untouched; the record stays invalid.
+  } else if (PRIVATE_ALIAS_SCOPES.has(migrated.scope)) {
+    migrated.scope = 'private';
+  }
+
+  if (migrated.schema_version === '1' || migrated.schema_version === 2) {
+    migrated.schema_version = CURRENT_SCHEMA_VERSION;
+  }
+
+  return migrated;
 }
 
 // Module-level dedupe sets for warnings (per-process).

package/scripts/lib/learnings/sizing-subject.mjs

@@ -9,8 +9,8 @@
  * 4-5. Before this module, the `effective-sizing` analyzer
  * (`skills/evolve/references/evolve-analyze-mode.md`) keyed its subject on
  * `session_type` alone (e.g. `deep-session-sizing`), so a 7-wave ultradeep
- * session and a 5-wave deep session landed on the SAME row — the over-delivery
- * median for one profile silently absorbed the other's.
+ * session and a 5-wave deep session landed on the SAME row — the agents-per-wave
+ * average for one profile silently absorbed the other's.
  *
  * Pure, stdlib-only, no imports, no clock, no fs.
  *