session-orchestrator 3.24.0 → 4.0.0

package/scripts/lib/learnings/evolve-telemetry.mjs

@@ -0,0 +1,178 @@
+/**
+ * learnings/evolve-telemetry.mjs — mechanical `orchestrator.evolve.completed` /
+ * `orchestrator.dialectic.completed` emitters for the two `/evolve` run
+ * classes (issue #1206).
+ *
+ * Both events were prose-only until this module: five emit sites lived in
+ * `skills/evolve/SKILL.md` as `node scripts/emit-event.mjs ...` bash blocks the
+ * coordinator had to remember to run — a mechanism hiding inside prose, the
+ * same shape #1017 already fixed once for the learnings-store write path
+ * (`pruneLearnings()` in `./expiry-sweep.mjs`). This module gives both events
+ * a CODE call site instead: `emitEvolveCompleted()` is invoked from
+ * `scripts/sweep-expired-learnings.mjs`'s `--prune` exit path (the ONE store
+ * write `/evolve analyze` already performs), and `recordDialecticRun()` is
+ * invoked from `scripts/dialectic-deriver.mjs::runDialecticDeriver()` for the
+ * three abort statuses it can determine on its own, plus dry-run success —
+ * see that module's header for why APPLY-mode success and the two
+ * throw-based abort classes (`unknown-model`, `subagent-crash`) still call
+ * `recordDialecticRun()` from `skills/evolve/SKILL.md` prose (the merge and
+ * the dispatch-agent try/catch both happen one layer up, outside this pure
+ * pipeline).
+ *
+ * Both emitters are best-effort and try/catch-wrapped — same posture as
+ * `emitReconcileCompleted` in `scripts/lib/reconcile/engine.mjs` and
+ * `decideAndRecordAutoDialectic` in `scripts/lib/auto-dialectic.mjs` — because
+ * `emitEvent()` THROWS `EventValidationError` on a malformed record, and a
+ * telemetry failure must never break the `/evolve` pipeline itself.
+ *
+ * Both REFUSE to emit without an explicit `repoRoot` (#1119): most unit tests
+ * call the underlying pipelines (`pruneLearnings`, `runDialecticDeriver`)
+ * without one, and a `SO_PROJECT_DIR` fallback would silently append
+ * synthetic records to the operator's REAL fleet ledger on every `npm test`.
+ */
+
+const REASON_MAX_CHARS = 300;
+
+/**
+ * Record one `/evolve analyze` run — success or abort.
+ *
+ * @param {object} [opts]
+ * @param {string} opts.repoRoot - REQUIRED. No SO_PROJECT_DIR fallback (#1119).
+ * @param {number} [opts.appended=0] - new learnings written (Step 3.5(4)).
+ * @param {number} [opts.boosted=0] - existing learnings reinforced (Step 3.5(2)).
+ * @param {number} [opts.pruned=0] - `$PRUNE.archived` total from the
+ *   `--prune` call this function is invoked alongside.
+ * @param {number} [opts.promoted=0] - always 0 from this call site; promotion
+ *   to `public` scope is a separate CLI (`npm run share:hw-learnings -- --promote`).
+ * @param {number} [opts.durationMs] - elapsed ms since the run's telemetry
+ *   start marker; omitted (never fabricated) when not measured.
+ * @param {string[]} [opts.skipped=[]] - HR-105: optional steps that RAN but
+ *   were themselves skipped this run (e.g. `skill-evolution-off`,
+ *   `vault-mirror-off`) — distinguishes "ran, a step inside it skipped" from
+ *   "did not run at all" (the `aborted` form below). Included only when
+ *   non-empty; success-form counters are ALWAYS present regardless.
+ * @param {string} [opts.aborted] - when given (non-empty string), emit the
+ *   ABORT form instead of the success form: `{aborted, reason, duration_ms}`.
+ * @param {string} [opts.reason] - abort message shown to the user, clamped to
+ *   {@link REASON_MAX_CHARS}.
+ * @param {Function} [opts.emitFn] - DI hook for testing; defaults to the real
+ *   `emitEvent` from `../events.mjs` (dynamically imported, so a caller that
+ *   supplies `emitFn` never touches the real module at all).
+ * @returns {Promise<void>}
+ */
+export async function emitEvolveCompleted({
+  repoRoot,
+  appended = 0,
+  boosted = 0,
+  pruned = 0,
+  promoted = 0,
+  durationMs,
+  skipped = [],
+  aborted,
+  reason,
+  emitFn,
+} = {}) {
+  if (typeof repoRoot !== 'string' || repoRoot.trim() === '') {
+    process.stderr.write(
+      'evolve: skipped orchestrator.evolve.completed — no repoRoot given; ' +
+        'refusing the ambient SO_PROJECT_DIR destination (#1119).\n',
+    );
+    return;
+  }
+
+  const payload =
+    typeof aborted === 'string' && aborted.length > 0
+      ? {
+          aborted,
+          reason: typeof reason === 'string' ? reason.slice(0, REASON_MAX_CHARS) : reason,
+          duration_ms: durationMs,
+        }
+      : {
+          appended,
+          boosted,
+          pruned,
+          promoted,
+          duration_ms: durationMs,
+          ...(Array.isArray(skipped) && skipped.length > 0 ? { skipped } : {}),
+        };
+
+  try {
+    const emit = emitFn ?? (await import('../events.mjs')).emitEvent;
+    await emit('orchestrator.evolve.completed', payload, { repoRoot });
+  } catch (err) {
+    // best-effort — a telemetry failure must never break the caller's pipeline,
+    // but a silently swallowed failure is exactly the 0-records unfalsifiability
+    // #1206 was filed against — so it is surfaced on stderr instead.
+    process.stderr.write(
+      '⚠ evolve-telemetry: emit failed (orchestrator.evolve.completed): ' +
+        (err?.message ?? String(err)) +
+        '\n',
+    );
+  }
+}
+
+/**
+ * Record one `/evolve --dialectic` run — success or abort.
+ *
+ * @param {object} [opts]
+ * @param {string} opts.repoRoot - REQUIRED, same #1119 refusal as {@link emitEvolveCompleted}.
+ * @param {string} opts.status - `ok` for the success form, or one of the abort
+ *   slugs `unknown-model` | `budget-exceeded` | `would-empty-card` |
+ *   `empty-input` | `subagent-crash` (mirrors `skills/evolve/SKILL.md` Step 6.5
+ *   verbatim) — any other non-`ok` string is emitted as the abort form as-is.
+ * @param {'dry-run'|'apply'} [opts.mode] - success form only.
+ * @param {number} [opts.userDeltas=0] - success form only.
+ * @param {number} [opts.agentDeltas=0] - success form only.
+ * @param {number} [opts.tokensIn] - success form only; omitted when not measured.
+ * @param {number} [opts.tokensOut] - success form only; omitted when not measured.
+ * @param {number} [opts.durationMs] - omitted (never fabricated) when not measured.
+ * @param {Function} [opts.emitFn] - DI hook for testing; defaults to the real
+ *   `emitEvent` from `../events.mjs` (dynamically imported, so a caller that
+ *   supplies `emitFn` never touches the real module at all).
+ * @returns {Promise<void>}
+ */
+export async function recordDialecticRun({
+  repoRoot,
+  status,
+  mode,
+  userDeltas = 0,
+  agentDeltas = 0,
+  tokensIn,
+  tokensOut,
+  durationMs,
+  emitFn,
+} = {}) {
+  if (typeof repoRoot !== 'string' || repoRoot.trim() === '') {
+    process.stderr.write(
+      'dialectic: skipped orchestrator.dialectic.completed — no repoRoot given; ' +
+        'refusing the ambient SO_PROJECT_DIR destination (#1119).\n',
+    );
+    return;
+  }
+
+  const payload =
+    status === 'ok'
+      ? {
+          mode,
+          user_deltas: userDeltas,
+          agent_deltas: agentDeltas,
+          tokens_in: tokensIn,
+          tokens_out: tokensOut,
+          duration_ms: durationMs,
+        }
+      : { aborted: status, duration_ms: durationMs };
+
+  try {
+    const emit = emitFn ?? (await import('../events.mjs')).emitEvent;
+    await emit('orchestrator.dialectic.completed', payload, { repoRoot });
+  } catch (err) {
+    // best-effort — a telemetry failure must never break the caller's pipeline,
+    // but a silently swallowed failure is exactly the 0-records unfalsifiability
+    // #1206 was filed against — so it is surfaced on stderr instead.
+    process.stderr.write(
+      '⚠ evolve-telemetry: emit failed (orchestrator.dialectic.completed): ' +
+        (err?.message ?? String(err)) +
+        '\n',
+    );
+  }
+}

package/scripts/lib/learnings/io.mjs

@@ -189,22 +189,76 @@ export async function appendLearning(filePath, entry) {
 const BACKUP_KEEP = 3;
 
 /**
- * Best-effort keep-N rotation of `${baseName}.bak-*` siblings in `dir`.
+ * Timestamp suffix of a backup sibling of `baseName`, or `null` if `fileName`
+ * is not one. Accepts BOTH historical delimiters after `.bak` (#1173): the
+ * canonical `-` this module writes, and the legacy `.` that pre-#721 writers
+ * (e.g. `learnings.jsonl.bak.evolve-<ts>`) still left on disk in consumer
+ * repos — those files were invisible to rotation AND to restore.
+ * The delimiter check is what keeps a `${baseName}.backfill-tmp-*` scratch
+ * file out: `.bak` followed by `f` is not a backup.
+ *
+ * REVISIT-TRIGGER for the legacy `.` branch (BV-004): `-` has been the only
+ * delimiter any writer emits since #721, so the `.` arm exists purely to keep
+ * pre-#721 files on disk visible to rotation and restore. Drop it once no fleet
+ * repo reports a `.bak.` sibling — re-measure via the fleet sweep, do not infer
+ * it from this repo alone.
+ *
+ * @param {string} baseName — basename of the store file (e.g. `learnings.jsonl`)
+ * @param {string} fileName — sibling filename to classify
+ * @returns {string|null} the suffix after the delimiter, or `null`
+ */
+export function backupSuffixOf(baseName, fileName) {
+  const stem = `${baseName}.bak`;
+  if (!fileName.startsWith(stem)) return null;
+  const delim = fileName[stem.length];
+  return delim === '-' || delim === '.' ? fileName.slice(stem.length + 1) : null;
+}
+
+/**
+ * True when `fileName` is a backup sibling of `baseName` in either delimiter
+ * form. Shared by this module's rotation and by `backfill-learnings-from-vault`'s
+ * restore sweep so the two can never drift apart again (#1173).
+ *
+ * @param {string} baseName
+ * @param {string} fileName
+ * @returns {boolean}
+ */
+export function isBackupOf(baseName, fileName) {
+  return backupSuffixOf(baseName, fileName) !== null;
+}
+
+/**
+ * Best-effort keep-N rotation of `${baseName}.bak[-.]*` siblings in `dir`.
  * The `.bak-<ISO>` naming uses ISO 8601 with `:`/`.` swapped for `-`, so a
- * plain lexical sort of the filenames is chronological. Oldest beyond `keep`
- * are unlinked. Errors PROPAGATE — the single caller wraps this in try/catch so
- * a rotation failure can never abort the rewrite it protects.
+ * plain lexical sort of the SUFFIX is chronological. Sorting on the suffix
+ * rather than the whole filename is load-bearing across the two delimiter
+ * forms (#1173): `-` (0x2D) sorts before `.` (0x2E), so a whole-name sort
+ * would group every legacy dot-form file after every hyphen-form one
+ * regardless of age, and rotation would prune only hyphen-form backups.
+ * Oldest beyond `keep` are unlinked. Errors PROPAGATE — the single caller
+ * wraps this in try/catch so a rotation failure can never abort the rewrite it
+ * protects.
  *
  * @param {string} dir — directory holding the store + its backups
  * @param {string} baseName — basename of the store file (e.g. `learnings.jsonl`)
  * @param {number} keep — number of newest backups to retain
  */
 async function rotateBackups(dir, baseName, keep = BACKUP_KEEP) {
-  const prefix = `${baseName}.bak-`;
   const names = await readdir(dir);
   // Lexical sort == chronological for the dash-normalized ISO suffix. Ascending
   // → oldest first, so the head of the list is what we prune.
-  const backups = names.filter((n) => n.startsWith(prefix)).sort();
+  const backups = names
+    .map((n) => ({ name: n, suffix: backupSuffixOf(baseName, n) }))
+    .filter((e) => e.suffix !== null)
+    // Sort key strips a leading non-digit label so a labelled legacy suffix
+    // (`.bak.evolve-<ts>`) compares against a bare one (`.bak-<ts>`) on the
+    // timestamp, not on the label. Ceiling (#1173): this assumes the label
+    // PRECEDES the timestamp, which holds for every form observed on disk; a
+    // suffix carrying no digits at all sorts oldest and is pruned first.
+    // Revisit if a writer ever appends its label after the timestamp.
+    .map((e) => ({ ...e, key: e.suffix.replace(/^\D*/, '') }))
+    .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0))
+    .map((e) => e.name);
   const stale = backups.slice(0, Math.max(0, backups.length - keep));
   for (const name of stale) {
     await unlink(path.join(dir, name));

package/scripts/lib/memory-proposals/store.mjs

@@ -227,11 +227,18 @@ function countWaveLines(jsonlPath, waveId) {
 }
 
 /**
- * Read the per-wave summary JSON, or return a zeroed summary if absent.
+ * Read + parse a per-wave summary JSON file. Returns the parsed summary on
+ * success, or `null` on any failure — missing file (ENOENT) silently, any
+ * other read/parse failure with a stderr WARN (#1210 — ENOENT and other
+ * failures are different facts, same split as `sessions-canonical.mjs`
+ * `readCanonicalSessions`). Shared by `readSummary` (defaults to zeros) and
+ * `readWaveSummary` (surfaces `null` to its own caller) — one read path
+ * instead of two near-identical copies.
+ *
  * @param {string} summaryPath
- * @returns {{ queued: number, dropped: number, below_floor: number, fs_error: number }}
+ * @returns {{ queued: number, dropped: number, below_floor: number, fs_error: number } | null}
  */
-function readSummary(summaryPath) {
+function readSummaryFileSafe(summaryPath) {
   try {
     const raw = fs.readFileSync(summaryPath, 'utf8');
     const parsed = JSON.parse(raw);
@@ -243,10 +250,26 @@ function readSummary(summaryPath) {
         fs_error: parsed.fs_error ?? 0,
       };
     }
-  } catch {
-    /* ENOENT or malformed — return zeros */
+    return null;
+  } catch (err) {
+    if (!err || err.code !== 'ENOENT') {
+      process.stderr.write(
+        `⚠ readSummaryFileSafe: cannot read ${summaryPath} ` +
+          `(${err?.code ?? '?'}: ${err?.message ?? String(err)}) — ` +
+          'treating as EMPTY, counts below are floors\n',
+      );
+    }
+    return null;
   }
-  return { queued: 0, dropped: 0, below_floor: 0, fs_error: 0 };
+}
+
+/**
+ * Read the per-wave summary JSON, or return a zeroed summary if absent.
+ * @param {string} summaryPath
+ * @returns {{ queued: number, dropped: number, below_floor: number, fs_error: number }}
+ */
+function readSummary(summaryPath) {
+  return readSummaryFileSafe(summaryPath) ?? { queued: 0, dropped: 0, below_floor: 0, fs_error: 0 };
 }
 
 /**
@@ -422,20 +445,5 @@ export async function countProposalsForWave({ repoRoot, waveId }) {
  */
 export async function readWaveSummary({ repoRoot, waveId }) {
   const summaryPath = summaryPathFor(repoRoot, waveId);
-  try {
-    const raw = fs.readFileSync(summaryPath, 'utf8');
-    const parsed = JSON.parse(raw);
-    if (parsed && typeof parsed === 'object') {
-      return {
-        queued: parsed.queued ?? 0,
-        dropped: parsed.dropped ?? 0,
-        below_floor: parsed.below_floor ?? 0,
-        fs_error: parsed.fs_error ?? 0,
-      };
-    }
-    return null;
-  } catch (err) {
-    if (err.code === 'ENOENT') return null;
-    return null;
-  }
+  return readSummaryFileSafe(summaryPath);
 }

package/scripts/lib/owner-config-banner.mjs

@@ -36,7 +36,14 @@
  *  - Issue #820.
  */
 
-import { loadOwnerConfig, OWNER_YAML_PATH } from './owner-yaml.mjs';
+import { loadOwnerConfig, resolveOwnerYamlPath, getDefaults } from './owner-yaml.mjs';
+
+/**
+ * OPTIONAL object sections (mirrors `owner-yaml.mjs`'s internal, unexported
+ * `OPTIONAL_OBJECT_SECTIONS`) — the only sections the #1244 discard-branch
+ * merge rule can carry through a whole-file discard.
+ */
+const OPTIONAL_OBJECT_SECTION_NAMES = /** @type {const} */ (['paths', 'dispatcher']);
 
 /**
  * Format a `{ section, errors }` entry into a short `"name" (first error)` tag.
@@ -52,6 +59,7 @@ function formatSectionTag(entry) {
  * Check owner.yaml load health and produce a session-start banner.
  *
  * @param {{ loader?: (opts?: object) => {
+ *   config?: object,
  *   source: 'file'|'defaults'|'partial',
  *   errors: string[],
  *   droppedSections?: Array<{ section: string, errors: string[] }>,
@@ -86,15 +94,44 @@ export function checkOwnerConfig({ loader = loadOwnerConfig } = {}) {
     }
 
     // Whole-file discard: a REQUIRED section was invalid (legacy behaviour).
+    //
+    // #1244 MERGE RULE (owner-yaml.mjs loadOwnerConfig): a VALID optional
+    // object section (`paths`, `dispatcher`) survives the discard — merged
+    // onto its default in `result.config`. loadOwnerConfig's return carries no
+    // dedicated "survived" field, so it is derived here: a section differs
+    // from `getDefaults()` only when real, validly-parsed values were merged
+    // in. Without this, the banner claimed "the entire file was discarded"
+    // while a host-local `paths.confidential-names-file` the operator DID
+    // configure correctly was still live — the exact silent-fix-that-lies gap
+    // Q3 measured with a tmp owner.yaml (invalid `owner.name`, valid
+    // `paths.confidential-names-file`).
     if (result.source === 'defaults' && errors.length > 0) {
       const firstError = errors[0];
+      const path = resolveOwnerYamlPath();
+      const survivedSections = [];
+      if (result.config && typeof result.config === 'object') {
+        const defaults = getDefaults();
+        for (const name of OPTIONAL_OBJECT_SECTION_NAMES) {
+          if (JSON.stringify(result.config[name]) !== JSON.stringify(defaults[name])) {
+            survivedSections.push(name);
+          }
+        }
+      }
+
+      const message =
+        survivedSections.length > 0
+          ? `⚠ owner-config: ${path} has required section(s) invalid (${firstError}) — ` +
+            `${survivedSections.map((s) => `"${s}"`).join('/')} kept, everything else defaulted. ` +
+            'Fix the file (see .claude/rules/owner-persona.md) to restore your settings.'
+          : `⚠ owner-config: ${path} is invalid (${firstError}) — the entire file was ` +
+            'discarded and defaults are in effect. Fix the file (see .claude/rules/owner-persona.md) ' +
+            'to restore your settings.';
+
       return {
         severity: 'warn',
-        message:
-          `⚠ owner-config: ${OWNER_YAML_PATH} is invalid (${firstError}) — the entire file was ` +
-          'discarded and defaults are in effect. Fix the file (see .claude/rules/owner-persona.md) ' +
-          'to restore your settings.',
+        message,
         discarded: true,
+        ...(droppedSections.length > 0 ? { droppedSections } : {}),
       };
     }
 
@@ -115,7 +152,7 @@ export function checkOwnerConfig({ loader = loadOwnerConfig } = {}) {
 
     return {
       severity: 'warn',
-      message: `⚠ owner-config: ${OWNER_YAML_PATH} — ${parts.join('; ')} — fix the file to restore full config.`,
+      message: `⚠ owner-config: ${resolveOwnerYamlPath()} — ${parts.join('; ')} — fix the file to restore full config.`,
       ...(droppedSections.length > 0 ? { droppedSections } : {}),
       ...(sectionWarnings.length > 0 ? { sectionWarnings } : {}),
     };

package/scripts/lib/owner-config-loader.mjs

@@ -8,7 +8,10 @@
  *
  * ── PATH RESOLUTION ──────────────────────────────────────────────────────
  *
- *   Default path: `${XDG_CONFIG_HOME ?? ${HOME}/.config}/session-orchestrator/owner.yaml`
+ *   Default path (#1223 — delegated to the single resolver
+ *   `config/private-config-dir.mjs`; `SO_CONFIG_HOME` names the private dir
+ *   ITSELF and wins):
+ *   `${SO_CONFIG_HOME ?? ${XDG_CONFIG_HOME ?? ${HOME}/.config}/session-orchestrator}/owner.yaml`
  *   - macOS / Linux: `~/.config/session-orchestrator/owner.yaml`
  *   - Windows:       `%APPDATA%\session-orchestrator\owner.yaml` when
  *                    APPDATA is set and XDG_CONFIG_HOME is not (Node uses
@@ -43,29 +46,37 @@
 import { readFile } from 'node:fs/promises';
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
-import { homedir } from 'node:os';
 import yaml from 'js-yaml';
 
 import { validate } from './owner-config.mjs';
+import { resolvePrivateConfigDir } from './config/private-config-dir.mjs';
 
 // ---------------------------------------------------------------------------
 // Path resolution
 // ---------------------------------------------------------------------------
 
-const APP_DIR = 'session-orchestrator';
-const CONFIG_FILE = 'owner.yaml';
-
 /**
- * Resolve the canonical owner.yaml path on disk. Honours XDG_CONFIG_HOME
- * when set (Linux convention), otherwise falls back to `${HOME}/.config`.
+ * Resolve the canonical owner.yaml path on disk: the ONE private-config-dir
+ * resolver (#1223) plus the file's basename — precedence `SO_CONFIG_HOME` (the
+ * private dir itself) > `XDG_CONFIG_HOME` (its parent) > `${HOME}/.config`, each
+ * `.trim()`ed so a whitespace-only value falls through instead of
+ * short-circuiting (`.claude/rules/development.md` § Error Handling).
+ *
+ * Reaches the leaf DIRECTLY rather than through `owner-yaml.mjs`'s
+ * `resolveOwnerYamlPath()`: that hop exists only to append the same basename and
+ * drags a `js-yaml`-importing module in with it. The precedence — the part #1223
+ * consolidated — still lives in exactly one place; only the literal `'owner.yaml'`
+ * is stated twice.
+ *
+ * Before #1223 this function knew only `XDG_CONFIG_HOME` and did not trim, so
+ * `SO_CONFIG_HOME=<sandbox>` still read the operator's real home.
+ *
  * Returns an absolute path string. Does NOT touch the filesystem.
  *
  * @returns {string} absolute path to owner.yaml
  */
 export function resolveOwnerConfigPath() {
-  const xdg = process.env.XDG_CONFIG_HOME;
-  const base = xdg && xdg.length > 0 ? xdg : join(homedir(), '.config');
-  return join(base, APP_DIR, CONFIG_FILE);
+  return join(resolvePrivateConfigDir(), 'owner.yaml');
 }
 
 // ---------------------------------------------------------------------------

package/scripts/lib/owner-interview.mjs

@@ -37,7 +37,7 @@
 import { randomBytes } from 'node:crypto';
 import { existsSync, copyFileSync } from 'node:fs';
 import {
-  OWNER_YAML_PATH,
+  resolveOwnerYamlPath,
   validateOwnerConfig,
   writeOwnerConfig,
   getDefaults,
@@ -161,7 +161,7 @@ export function getInterviewQuestions() {
  * @returns {{ ok: boolean, path: string, errors: string[] }}
  */
 export function applyInterviewAnswers(answers, opts = {}) {
-  const filePath = opts.path ?? OWNER_YAML_PATH;
+  const filePath = opts.path ?? resolveOwnerYamlPath();
 
   if (!Array.isArray(answers) || answers.length !== 5) {
     return { ok: false, path: filePath, errors: ['applyInterviewAnswers requires exactly 5 answers'] };
@@ -258,7 +258,7 @@ export function runOwnerInterview(opts = {}) {
   const {
     skipIfExists = true,
     force = false,
-    path: filePath = OWNER_YAML_PATH,
+    path: filePath = resolveOwnerYamlPath(),
   } = opts;
 
   const fileExists = existsSync(filePath);