session-orchestrator 3.24.0 → 4.0.1

package/scripts/backfill-learnings-expires.mjs

@@ -1,196 +0,0 @@
-#!/usr/bin/env node
-/**
- * backfill-learnings-expires.mjs — one-shot backfill helper for `expires_at`.
- *
- * Closes issue #323. Surfaced by 2026-05-07 W1 D2 surface-map: 1/129 records
- * in .orchestrator/metrics/learnings.jsonl was missing `expires_at` (id:
- * `archive-en-masse-residual-policy-2026-05-01`).
- *
- * For each record without `expires_at`, derives one from `created_at` + `type`
- * via `deriveExpiresAt()` from `scripts/lib/learnings.mjs`. Records that
- * already have `expires_at` pass through untouched.
- *
- * Forensics: every patched record gets a `_backfilled_expires_at: true` tag
- * so downstream readers can identify backfilled vs originally-stamped values.
- *
- * Usage:
- *   node scripts/backfill-learnings-expires.mjs [--source <path>] [--dry-run] [--apply]
- *
- * Flags:
- *   --source <path>   Path to learnings.jsonl (default: .orchestrator/metrics/learnings.jsonl)
- *   --dry-run         Preview changes without writing (DEFAULT)
- *   --apply           Write patched records back; backup original to <path>.bak.<isoDate>
- *
- * Exit codes:
- *   0  Success (including no-op when nothing needs backfilling)
- *   1  Input/validation error
- *   2  I/O error
- *
- * Idempotent: a second --apply run produces 0 to_backfill (every record now
- * has expires_at; the `_backfilled_expires_at` tag is preserved).
- *
- * Output (stdout — single JSON summary line):
- *   {"total":N,"already_has_expires":A,"to_backfill":B,"parse_errors":P,"applied":bool}
- *
- * Per-record narration goes to stderr so stdout stays machine-parseable.
- */
-
-import { existsSync, readFileSync, writeFileSync, renameSync, copyFileSync, mkdirSync } from 'node:fs';
-import { dirname } from 'node:path';
-import { deriveExpiresAt } from './lib/learnings.mjs';
-
-// ---------------------------------------------------------------------------
-// Arg parsing
-// ---------------------------------------------------------------------------
-
-const args = process.argv.slice(2);
-if (args.includes('--help') || args.includes('-h')) {
-  process.stdout.write(
-    `Usage: backfill-learnings-expires.mjs [--source <path>] [--dry-run] [--apply]
-
-Options:
-  --source <path>   Path to learnings.jsonl (default: .orchestrator/metrics/learnings.jsonl)
-  --dry-run         Preview changes without writing (DEFAULT)
-  --apply           Write patched records; backup original to <path>.bak.<isoDate>
-
-Exit codes:  0 success  1 input error  2 I/O error
-`
-  );
-  process.exit(0);
-}
-
-const applyFlag = args.includes('--apply');
-const dryRun = !applyFlag;
-
-const sourceIdx = args.indexOf('--source');
-const sourcePath =
-  sourceIdx !== -1 && args[sourceIdx + 1]
-    ? args[sourceIdx + 1]
-    : '.orchestrator/metrics/learnings.jsonl';
-
-// ---------------------------------------------------------------------------
-// Load
-// ---------------------------------------------------------------------------
-
-if (!existsSync(sourcePath)) {
-  process.stderr.write(`backfill-learnings-expires: source file not found: ${sourcePath}\n`);
-  process.exit(1);
-}
-
-let raw;
-try {
-  raw = readFileSync(sourcePath, 'utf8');
-} catch (err) {
-  process.stderr.write(
-    `backfill-learnings-expires: failed to read ${sourcePath}: ${err.message}\n`
-  );
-  process.exit(2);
-}
-
-const lines = raw.split('\n').filter((l) => l.trim().length > 0);
-
-// ---------------------------------------------------------------------------
-// Backfill
-// ---------------------------------------------------------------------------
-
-let countAlready = 0;
-let countToBackfill = 0;
-let countParseErrors = 0;
-const outputLines = [];
-
-for (const line of lines) {
-  let parsed;
-  try {
-    parsed = JSON.parse(line);
-  } catch {
-    countParseErrors++;
-    process.stderr.write(
-      `backfill-learnings-expires: WARN skipping malformed JSON line: ${line.slice(0, 80)}\n`
-    );
-    // Preserve original line — never destroy data
-    outputLines.push(line);
-    continue;
-  }
-
-  const hasExpires =
-    typeof parsed.expires_at === 'string' && parsed.expires_at.length > 0;
-  if (hasExpires) {
-    countAlready++;
-    outputLines.push(JSON.stringify(parsed));
-    continue;
-  }
-
-  // Derive expires_at from created_at + type. deriveExpiresAt() falls back
-  // to Date.now() when created_at is missing/unparseable, so we always
-  // produce a valid ISO timestamp.
-  const newExpires = deriveExpiresAt(parsed.created_at, parsed.type);
-  const patched = {
-    ...parsed,
-    expires_at: newExpires,
-    _backfilled_expires_at: true,
-  };
-  countToBackfill++;
-  outputLines.push(JSON.stringify(patched));
-  process.stderr.write(
-    `backfill-learnings-expires: patched id=${parsed.id ?? '<no id>'} type=${parsed.type ?? '<no type>'} expires_at=${newExpires}\n`
-  );
-}
-
-// ---------------------------------------------------------------------------
-// Write (--apply only)
-// ---------------------------------------------------------------------------
-
-let applied = false;
-if (!dryRun && countToBackfill > 0) {
-  // Backup first
-  const isoDate = new Date().toISOString().replace(/[:.]/g, '-');
-  const backupPath = `${sourcePath}.bak.${isoDate}`;
-  try {
-    copyFileSync(sourcePath, backupPath);
-  } catch (err) {
-    process.stderr.write(
-      `backfill-learnings-expires: ERROR failed to create backup ${backupPath}: ${err.message}\n`
-    );
-    process.exit(2);
-  }
-
-  // Atomic write via tmp + rename
-  const body = outputLines.join('\n') + '\n';
-  const tmp = `${sourcePath}.backfill-tmp-${process.pid}-${Date.now()}`;
-  try {
-    mkdirSync(dirname(sourcePath), { recursive: true });
-    writeFileSync(tmp, body, 'utf8');
-    renameSync(tmp, sourcePath);
-    applied = true;
-    process.stderr.write(
-      `backfill-learnings-expires: backup written to ${backupPath}\n`
-    );
-  } catch (err) {
-    process.stderr.write(
-      `backfill-learnings-expires: ERROR failed to write ${sourcePath}: ${err.message}\n`
-    );
-    try {
-      if (existsSync(tmp)) renameSync(tmp, tmp + '.failed');
-    } catch {
-      /* ignore */
-    }
-    process.exit(2);
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Summary (single JSON line on stdout)
-// ---------------------------------------------------------------------------
-
-const summary = {
-  total: lines.length,
-  already_has_expires: countAlready,
-  to_backfill: countToBackfill,
-  parse_errors: countParseErrors,
-  applied,
-  dry_run: dryRun,
-  source: sourcePath,
-};
-process.stdout.write(JSON.stringify(summary) + '\n');
-
-process.exit(0);

package/scripts/backfill-learnings.mjs

@@ -1,203 +0,0 @@
-#!/usr/bin/env node
-/**
- * backfill-learnings.mjs — normalize producer DIALECTS in learnings.jsonl.
- *
- * Epic #723 B2. The live corpus historically carried the rule-conversion scope
- * under `files` (not the canonical `file_paths`), duplicated `source_session`
- * into `session_id`, used `last_seen` instead of `updated_at`, and left
- * `next_review: null` / non-canonical (no-millis) timestamps. This one-shot
- * backfill routes every line through the schema SSOT (`migrateLegacyLearning`,
- * which applies `normalizeDialects` + the insight/subject alias chain + a
- * `schema_version:1` stamp), gates the result with `validateLearning`, and
- * rewrites the file where a safe normalization exists.
- *
- * Data-loss safety (three guarantees):
- *   - A line that fails `JSON.parse` is preserved byte-identical and counted
- *     (`parse_errors`) — never dropped.
- *   - A record that still fails `validateLearning` AFTER migration is passed
- *     through byte-identical and counted (`invalid_after_migration`) — never
- *     mutated, never flagged.
- *   - `--apply` writes atomically (copy-to-`.bak-<ISO>` → tmp → rename), so the
- *     canonical file always exists holding either the old or the new content.
- *
- * This script duplicates NO normalization logic — all shape work lives in
- * `scripts/lib/learnings/schema.mjs`.
- *
- * Usage:
- *   node scripts/backfill-learnings.mjs [--file PATH] [--apply|--dry-run]
- *
- * Modes:
- *   (default / --dry-run)   compute + report intended rewrites per line; does
- *                           not touch the file
- *   --apply                 write the normalized file atomically via tmp+rename,
- *                           after copying the original to `${file}.bak-<ISO>`
- *
- * Exit codes:
- *   0 — completed (dry-run or apply)
- *   1 — I/O error (file not found, unreadable, write failed)
- *   2 — usage error (unknown argument)
- *
- * Output: JSON summary on stdout (single line)
- *   {"scanned":N,"normalized":N,"unchanged":N,"parse_errors":N,
- *    "invalid_after_migration":N,"applied":bool,"dry_run":bool,"backup":"<path>|null","file":"<path>"}
- */
-
-import { readFileSync, writeFileSync, renameSync, copyFileSync, existsSync } from 'node:fs';
-import { join, dirname, basename } from 'node:path';
-import {
-  migrateLegacyLearning,
-  normalizeDialects,
-  validateLearning,
-  ValidationError,
-} from './lib/learnings/schema.mjs';
-
-const DEFAULT_FILE = '.orchestrator/metrics/learnings.jsonl';
-
-function parseArgs(argv) {
-  const args = { file: DEFAULT_FILE, apply: false };
-  for (let i = 0; i < argv.length; i++) {
-    const a = argv[i];
-    if (a === '--file') args.file = argv[++i];
-    else if (a === '--apply') args.apply = true;
-    else if (a === '--dry-run') args.apply = false;
-    else if (a === '--help' || a === '-h') {
-      process.stdout.write(
-        'Usage: node scripts/backfill-learnings.mjs [--file PATH] [--apply|--dry-run]\n' +
-          '  --file      target learnings.jsonl (default: .orchestrator/metrics/learnings.jsonl)\n' +
-          '  --dry-run   report normalizations; do not modify file (default)\n' +
-          '  --apply     rewrite file atomically with a .bak-<ISO> backup\n'
-      );
-      process.exit(0);
-    } else {
-      process.stderr.write(`backfill-learnings: unknown argument: ${a}\n`);
-      process.exit(2);
-    }
-  }
-  return args;
-}
-
-/**
- * Classify + transform a single parsed record via the schema SSOT.
- *
- * Returns { outcome, line } where outcome is one of:
- *   - 'unchanged'   : migration is a semantic no-op (serialized form matches input)
- *   - 'normalized'  : migration changed the record AND it validates
- *   - 'invalid'     : migration result still fails validateLearning → keep original
- *
- * @param {object} parsed — the JSON.parse'd record
- * @param {string} originalLine — the verbatim source line (returned untouched on 'invalid')
- * @returns {{ outcome: 'unchanged'|'normalized'|'invalid', line: string }}
- */
-function processRecord(parsed, originalLine) {
-  // Two SSOT calls, no duplicated normalization logic:
-  //   1. migrateLegacyLearning — aliases + schema_version:1 stamp + dialects
-  //      (files→file_paths, session_id/last_seen/next_review), byte-exact timestamps.
-  //   2. normalizeDialects (full) — canonicalize timestamp FORMAT to millis+Z.
-  //      Idempotent for every other dialect (already handled in step 1).
-  const migrated = normalizeDialects(migrateLegacyLearning(parsed));
-
-  // Gate: a record that cannot pass the schema after migration is passed
-  // through byte-identical (never mutated, never flagged).
-  try {
-    validateLearning(migrated);
-  } catch (err) {
-    if (err instanceof ValidationError) {
-      return { outcome: 'invalid', line: originalLine };
-    }
-    throw err;
-  }
-
-  const migratedLine = JSON.stringify(migrated);
-  // Semantic-equality against the re-serialized parsed input: a migration that
-  // reorders nothing and changes no value is a no-op.
-  if (migratedLine === JSON.stringify(parsed)) {
-    return { outcome: 'unchanged', line: migratedLine };
-  }
-  return { outcome: 'normalized', line: migratedLine };
-}
-
-function main() {
-  const args = parseArgs(process.argv.slice(2));
-
-  if (!args.file || !existsSync(args.file)) {
-    process.stderr.write(`backfill-learnings: file not found: ${args.file}\n`);
-    process.exit(1);
-  }
-
-  let lines;
-  try {
-    lines = readFileSync(args.file, 'utf8').split('\n').filter((l) => l.length > 0);
-  } catch (err) {
-    process.stderr.write(`backfill-learnings: read failed: ${err.message}\n`);
-    process.exit(1);
-  }
-
-  const outputLines = [];
-  const counts = {
-    scanned: 0,
-    normalized: 0,
-    unchanged: 0,
-    parse_errors: 0,
-    invalid_after_migration: 0,
-  };
-
-  for (const line of lines) {
-    counts.scanned++;
-    let parsed;
-    try {
-      parsed = JSON.parse(line);
-    } catch {
-      counts.parse_errors++;
-      outputLines.push(line); // byte-identical preservation — never drop
-      continue;
-    }
-
-    const { outcome, line: outLine } = processRecord(parsed, line);
-    if (outcome === 'invalid') counts.invalid_after_migration++;
-    else counts[outcome]++;
-    outputLines.push(outLine);
-  }
-
-  const summary = {
-    scanned: counts.scanned,
-    normalized: counts.normalized,
-    unchanged: counts.unchanged,
-    parse_errors: counts.parse_errors,
-    invalid_after_migration: counts.invalid_after_migration,
-    applied: false,
-    dry_run: !args.apply,
-    backup: null,
-    file: args.file,
-  };
-
-  if (!args.apply) {
-    process.stdout.write(JSON.stringify(summary) + '\n');
-    return;
-  }
-
-  // Apply path — POSIX atomic replace, self-contained (no io.mjs dependency).
-  //   1. copyFileSync: canonical → .bak-<ISO>   (backup; canonical still present)
-  //   2. writeFileSync: normalized → tmp        (new content; canonical still present)
-  //   3. renameSync: tmp → canonical            (atomic replace per POSIX)
-  // The canonical file exists at every step, holding old or new content.
-  const iso = new Date().toISOString().replace(/[:.]/g, '-');
-  const backupPath = `${args.file}.bak-${iso}`;
-  const tmpPath = join(dirname(args.file), `.${basename(args.file)}.tmp-${process.pid}`);
-  try {
-    copyFileSync(args.file, backupPath);
-    writeFileSync(tmpPath, outputLines.join('\n') + '\n', 'utf8');
-    renameSync(tmpPath, args.file);
-    summary.applied = true;
-    summary.backup = backupPath;
-  } catch (err) {
-    const recoveryHint = existsSync(backupPath)
-      ? `Canonical file is intact; backup at ${backupPath}.`
-      : 'Backup was not created.';
-    process.stderr.write(`backfill-learnings: write failed: ${err.message}. ${recoveryHint}\n`);
-    process.exit(1);
-  }
-
-  process.stdout.write(JSON.stringify(summary) + '\n');
-}
-
-main();

package/scripts/fleet-instruction-scan.mjs

@@ -1,141 +0,0 @@
-#!/usr/bin/env node
-/**
- * fleet-instruction-scan.mjs — always-on instruction load across every repo on the host.
- *
- * `scripts/measure-context-overhead.sh` measures one directory precisely but
- * costs an API call per measurement. This scans the whole fleet for free by
- * counting bytes and classifying them the way `scripts/lib/rule-loader.mjs`
- * does: a rule file is always-on unless its frontmatter carries `globs:` or
- * `paths:` (issue #795), and it reaches wave agents unless it is tagged
- * `tier: coordinator-only` (issue #692).
- *
- * Token estimates use the ratio measured 2026-07-30 on claude-opus-5[1m]:
- * 108589 always-on bytes in this repo produced roughly 67396 context tokens,
- * i.e. about 1 token per 1.6 bytes. Treat the estimate as an order of
- * magnitude and confirm a specific repo with measure-context-overhead.sh.
- *
- * Usage:
- *   node scripts/fleet-instruction-scan.mjs
- *   node scripts/fleet-instruction-scan.mjs ~/Projects/intern ~/Projects/extern
- *   node scripts/fleet-instruction-scan.mjs --json
- */
-
-import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
-import { homedir } from 'node:os';
-import { basename, join } from 'node:path';
-
-const BYTES_PER_TOKEN = 1.6;
-
-const argv = process.argv.slice(2);
-const asJson = argv.includes('--json');
-const roots = argv.filter((a) => !a.startsWith('--'));
-const ROOTS = roots.length
-  ? roots
-  : [
-      join(homedir(), 'Projects', 'extern'),
-      join(homedir(), 'Projects', 'intern'),
-      join(homedir(), 'Projects', 'Ventures'),
-      join(homedir(), 'Projects', 'Bernhard'),
-    ];
-
-/** Classify one rule file the way rule-loader.mjs does. */
-function classify(path) {
-  const body = readFileSync(path, 'utf8');
-  const fmMatch = body.match(/^---\n([\s\S]*?)\n---/);
-  const fm = fmMatch ? fmMatch[1] : '';
-  const tierMatch = fm.match(/^tier:\s*(\S+)/m);
-  return {
-    scoped: /^(globs|paths):/m.test(fm),
-    tier: tierMatch ? tierMatch[1] : 'always',
-    bytes: Buffer.byteLength(body),
-  };
-}
-
-function scanRepo(dir) {
-  let alwaysBytes = 0;
-  let waveBytes = 0;
-  let files = 0;
-  let scopedFiles = 0;
-  const rulesDir = join(dir, '.claude', 'rules');
-  if (existsSync(rulesDir)) {
-    for (const f of readdirSync(rulesDir).filter((n) => n.endsWith('.md'))) {
-      const c = classify(join(rulesDir, f));
-      files++;
-      if (c.scoped) {
-        scopedFiles++;
-        continue;
-      }
-      alwaysBytes += c.bytes;
-      if (c.tier !== 'coordinator-only') waveBytes += c.bytes;
-    }
-  }
-  const instructionFile = ['CLAUDE.md', 'AGENTS.md'].map((f) => join(dir, f)).find(existsSync);
-  const cmBytes = instructionFile ? statSync(instructionFile).size : 0;
-  return { files, scopedFiles, alwaysBytes, waveBytes, cmBytes, total: alwaysBytes + cmBytes };
-}
-
-const rows = [];
-for (const root of ROOTS) {
-  if (!existsSync(root)) continue;
-  for (const name of readdirSync(root)) {
-    const dir = join(root, name);
-    try {
-      if (!statSync(dir).isDirectory()) continue;
-    } catch {
-      continue;
-    }
-    if (!existsSync(join(dir, '.git'))) continue;
-    rows.push({ repo: `${basename(root)}/${name}`, ...scanRepo(dir) });
-  }
-}
-rows.sort((a, b) => b.total - a.total);
-
-if (asJson) {
-  console.log(JSON.stringify({ bytesPerToken: BYTES_PER_TOKEN, repos: rows }, null, 2));
-  process.exit(0);
-}
-
-const pad = (s, n) => String(s).padStart(n);
-console.log(
-  'REPO'.padEnd(36),
-  pad('RULES', 6),
-  pad('SCOPED', 7),
-  pad('ALWAYS_B', 9),
-  pad('WAVE_B', 8),
-  pad('INSTR_B', 8),
-  pad('EST_TOK', 8),
-);
-for (const r of rows) {
-  console.log(
-    r.repo.padEnd(36),
-    pad(r.files, 6),
-    pad(r.scopedFiles, 7),
-    pad(r.alwaysBytes, 9),
-    pad(r.waveBytes, 8),
-    pad(r.cmBytes, 8),
-    pad(Math.round(r.total / BYTES_PER_TOKEN), 8),
-  );
-}
-
-const totalFiles = rows.reduce((a, r) => a + r.files, 0);
-const totalScoped = rows.reduce((a, r) => a + r.scopedFiles, 0);
-const median = (xs) => {
-  const s = [...xs].sort((a, b) => a - b);
-  return s.length ? s[Math.floor(s.length / 2)] : 0;
-};
-const withRules = rows.filter((r) => r.files > 0);
-
-console.log(`\nrepos scanned            : ${rows.length} (${withRules.length} with rule files)`);
-console.log(`rule files               : ${totalFiles}  (${totalScoped} glob/path-scoped, ${totalFiles - totalScoped} always-on)`);
-console.log(`median always-on bytes   : ${median(withRules.map((r) => r.alwaysBytes))}`);
-console.log(`median est. tokens/session: ${Math.round(median(withRules.map((r) => r.total)) / BYTES_PER_TOKEN)}`);
-
-// A value shared by many repos means a vendored rule set: trimming it at the
-// source propagates everywhere, which is the cheapest lever in the fleet.
-const byBytes = new Map();
-for (const r of withRules) byBytes.set(r.alwaysBytes, (byBytes.get(r.alwaysBytes) || 0) + 1);
-const shared = [...byBytes.entries()].filter(([, n]) => n > 1).sort((a, b) => b[1] - a[1]);
-if (shared.length) {
-  console.log('\nidentical always-on totals (vendored rule sets — trim once, propagate widely):');
-  for (const [bytes, n] of shared.slice(0, 5)) console.log(`  ${bytes} B  in ${n} repos`);
-}