redlinegate 0.0.1

package/scripts/measure-context.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+// Measures what the skills render target actually saves, per profile.
+//
+// The roadmap makes this piece conditional on the measurement: it is a cost
+// optimisation, it is rated the highest-exposure item in the plan, and it dies if
+// the reduction is marginal. So the number is produced by a script anyone can
+// re-run rather than asserted once in a changelog.
+//
+// What is compared:
+//   composed  — AGENTS.md, which a Claude session loads in full every turn for the
+//               life of the session, whatever file is being edited
+//   per-stack — the core skill plus the ONE stack whose files are in play, which
+//               is what a skills-rendered repository loads instead
+//
+// Env: [ROOT=.], [OUT] (write JSON here as well as printing)
+
+import { cpSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { render } from '../dist/render/standards.js';
+
+const { ROOT = process.cwd(), OUT } = process.env;
+
+// This measures the standards package, so it needs one. Running it in a product
+// repository is a reasonable mistake — the command surface does not hide that it
+// exists — and an ENOENT stack trace is the worst possible answer to it.
+if (!existsSync(join(ROOT, 'standards/manifest.json'))) {
+  console.error(`No standards/manifest.json under ${resolve(ROOT)}.`);
+  console.error('This measures the Redline standards themselves, so it runs in the source repo.');
+  console.error('Point it somewhere else with --root <path>.');
+  process.exit(2);
+}
+
+// A scratch root whose manifest has skills enabled, so the measurement does not
+// depend on whether the org has switched it on yet.
+const root = mkdtempSync(join(tmpdir(), 'redline-measure-'));
+cpSync(join(ROOT, 'standards'), join(root, 'standards'), { recursive: true });
+const manifestPath = join(root, 'standards/manifest.json');
+const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
+manifest.vendors.skills.enabled = true;
+writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+
+const rows = [];
+for (const profile of Object.keys(manifest.profiles)) {
+  const skillsOut = mkdtempSync(join(tmpdir(), 'redline-skills-'));
+  render({ root, profile, out: skillsOut, vendors: ['skills'] });
+  const dir = join(skillsOut, '.claude/skills');
+  const size = (name) => readFileSync(join(dir, name, 'SKILL.md'), 'utf8').length;
+  const names = readdirSync(dir);
+  const core = size('redline-core');
+  const stacks = names.filter((n) => n !== 'redline-core');
+  // The mean stack, not the smallest: quoting the best case would be marketing.
+  const perStack =
+    core + (stacks.length ? Math.round(stacks.reduce((a, n) => a + size(n), 0) / stacks.length) : 0);
+
+  const composedOut = mkdtempSync(join(tmpdir(), 'redline-composed-'));
+  render({ root, profile, out: composedOut, vendors: ['claude', 'agents'] });
+  const composed = readFileSync(join(composedOut, 'AGENTS.md'), 'utf8').length;
+
+  rows.push({
+    profile,
+    stacks: stacks.length,
+    composedBytes: composed,
+    perStackBytes: perStack,
+    reduction: 1 - perStack / composed,
+  });
+
+  rmSync(skillsOut, { recursive: true, force: true });
+  rmSync(composedOut, { recursive: true, force: true });
+}
+rmSync(root, { recursive: true, force: true });
+
+rows.sort((a, b) => b.reduction - a.reduction);
+
+const pct = (n) => `${(n * 100).toFixed(1)}%`;
+console.log('profile           stacks   composed   per-stack   reduction');
+for (const row of rows) {
+  console.log(
+    `${row.profile.padEnd(17)}${String(row.stacks).padStart(6)}   ${String(row.composedBytes).padStart(8)}   ${String(row.perStackBytes).padStart(9)}   ${pct(row.reduction).padStart(9)}`
+  );
+}
+
+// The finding, stated rather than left for a reader to notice: a single-stack
+// profile pays the skill frontmatter and gains nothing, because there is no
+// second stack to avoid loading. Selecting skills there makes the repository
+// worse, and no amount of rollout enthusiasm changes that.
+const helped = rows.filter((r) => r.reduction > 0);
+const hurt = rows.filter((r) => r.reduction <= 0);
+console.log('');
+console.log(`Helps ${helped.length} profile(s): ${pct(Math.min(...helped.map((r) => r.reduction)))} to ${pct(Math.max(...helped.map((r) => r.reduction)))}.`);
+if (hurt.length > 0) {
+  console.log(
+    `Costs ${hurt.length} profile(s) — all single-stack (${hurt.map((r) => r.profile).join(', ')}): ` +
+      'there is no second stack to avoid loading, so the frontmatter is pure overhead. Do not select skills there.'
+  );
+}
+
+if (OUT) {
+  writeFileSync(OUT, `${JSON.stringify({ generatedAt: new Date().toISOString(), rows }, null, 2)}\n`);
+  console.log(`\nWritten to ${OUT}.`);
+}

package/scripts/render-self.mjs

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+// Renders this repository's own standards artifacts. The estate-wide renderer lives
+// in cli/render/standards.ts; this is the two-line shim CI calls.
+import { resolve, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { render } from '../cli/render/standards.ts';
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const check = process.argv.includes('--check');
+const result = render({ root, profile: 'tooling', out: root, check });
+
+if (result.stale.length) {
+  console.error(`Rendered output is stale. Run: node scripts/render-self.mjs\n  ${result.stale.join('\n  ')}`);
+  process.exit(1);
+}
+result.written.forEach((f) => console.log(`  write  ${f}`));
+result.removed.forEach((f) => console.log(`  prune  ${f}`));
+if (!result.written.length && !result.removed.length) console.log('  up to date');

package/scripts/score-seeds.mjs

@@ -0,0 +1,294 @@
+#!/usr/bin/env node
+// Scores an automated reviewer against the seeded corpus.
+//
+//   GH_TOKEN=... node scripts/score-seeds.mjs --repo acme/pilot-web --pr 12
+//   GH_TOKEN=... node scripts/score-seeds.mjs --repo acme/pilot-web --pr 12 --json
+//   GH_TOKEN=... node scripts/score-seeds.mjs --repo acme/pilot-web --pr 12 \
+//     --history data/seed-scores.jsonl --baseline
+//
+// Three numbers, because a review system fails in three ways:
+//   recall     — seeded defects flagged at the expected severity or higher
+//   precision  — comments on seeded/clean/**, which should never happen
+//   attribution— flagged defects that cited the correct rule id
+//
+// Vendor-neutral: findings are attributed by reviewer login, so the same command scores
+// Copilot, Claude or Codex on identical input.
+
+import { readFileSync, readdirSync, statSync, appendFileSync, existsSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { ROOT, RANK, loadRules, parseFinding, isReviewBot, RESERVED_RULE_IDS } from './lib/rules.mjs';
+
+const args = process.argv.slice(2);
+const arg = (name) => {
+  const i = args.indexOf(`--${name}`);
+  return i === -1 ? undefined : args[i + 1];
+};
+
+const REPO = arg('repo');
+const PR = arg('pr');
+const AS_JSON = args.includes('--json');
+const HISTORY = arg('history');
+const BASELINE = args.includes('--baseline');
+// A finding rarely lands on the exact marker line; accept the surrounding block.
+const WINDOW = Number(arg('window') ?? 4);
+const { GH_TOKEN } = process.env;
+
+if (!REPO || !PR || !GH_TOKEN) {
+  console.error(
+    'usage: GH_TOKEN=... score-seeds.mjs --repo <org>/<repo> --pr <number> [--json] [--window 4] [--history <file>] [--baseline]'
+  );
+  process.exit(2);
+}
+
+const rules = loadRules();
+const SEED_MARKER = /SEED\s+(\d+)\s*\[(BLOCKER|HIGH|SUGGESTION)\]\s*(?:\(([^)]+)\))?\s*(.*)$/;
+
+function walk(dir) {
+  const out = [];
+  for (const entry of readdirSync(dir)) {
+    const full = join(dir, entry);
+    if (statSync(full).isDirectory()) out.push(...walk(full));
+    else if (!entry.endsWith('.md')) out.push(full);
+  }
+  return out;
+}
+
+// --- expected findings, read straight out of the corpus ---------------------
+const expected = [];
+const cleanFiles = new Set();
+for (const file of walk(join(ROOT, 'seeded'))) {
+  const rel = relative(ROOT, file);
+  if (rel.startsWith('seeded/clean/')) {
+    cleanFiles.add(rel);
+    continue;
+  }
+  readFileSync(file, 'utf8')
+    .split('\n')
+    .forEach((line, i) => {
+      const m = SEED_MARKER.exec(line);
+      if (!m) return;
+      expected.push({
+        file: rel,
+        stack: rel.split('/')[1],
+        line: i + 1,
+        id: Number(m[1]),
+        severity: m[2],
+        ruleId: m[3] ?? null,
+        note: m[4].trim(),
+      });
+    });
+}
+
+for (const seed of expected) {
+  if (seed.ruleId && !rules.has(seed.ruleId) && !RESERVED_RULE_IDS.has(seed.ruleId)) {
+    console.error(`FAIL ${seed.file}:${seed.line} cites unknown rule id "${seed.ruleId}"`);
+    process.exit(2);
+  }
+}
+
+// --- what the reviewer actually posted --------------------------------------
+async function api(path) {
+  const results = [];
+  let url = `https://api.github.com${path}${path.includes('?') ? '&' : '?'}per_page=100`;
+  while (url) {
+    const res = await fetch(url, {
+      headers: { Authorization: `Bearer ${GH_TOKEN}`, Accept: 'application/vnd.github+json' },
+    });
+    if (!res.ok) throw new Error(`${url}: ${res.status} ${await res.text()}`);
+    results.push(...(await res.json()));
+    const next = /<([^>]+)>;\s*rel="next"/.exec(res.headers.get('link') ?? '');
+    url = next?.[1] ?? null;
+  }
+  return results;
+}
+
+const comments = (await api(`/repos/${REPO}/pulls/${PR}/comments`)).filter((c) =>
+  isReviewBot(c.user?.login ?? '')
+);
+
+const findings = comments.map((c) => {
+  const parsed = parseFinding(c.body ?? '');
+  return {
+    path: c.path,
+    line: c.line ?? c.original_line ?? 0,
+    ...parsed,
+    reviewer: c.user?.login,
+    body: (c.body ?? '').split('\n')[0].slice(0, 120),
+  };
+});
+
+// --- match ------------------------------------------------------------------
+const used = new Set();
+const caught = [];
+const missed = [];
+
+for (const seed of expected) {
+  // Prefer a finding that cites the right rule; fall back to position + severity, which
+  // still counts as caught. Attribution is scored separately so a reviewer that finds
+  // the bug but names the wrong rule is not marked as having missed it.
+  const candidates = findings
+    .map((f, i) => ({ f, i }))
+    .filter(
+      ({ f, i }) =>
+        !used.has(i) &&
+        seed.file.endsWith(f.path) &&
+        Math.abs(f.line - seed.line) <= WINDOW &&
+        RANK[f.severity] >= RANK[seed.severity]
+    );
+
+  const exact = candidates.find(({ f }) => seed.ruleId && f.ruleId === seed.ruleId);
+  const hit = exact ?? candidates[0];
+
+  if (!hit) {
+    missed.push(seed);
+    continue;
+  }
+  used.add(hit.i);
+  caught.push({
+    ...seed,
+    got: hit.f.severity,
+    gotRuleId: hit.f.ruleId,
+    ruleMatch: Boolean(seed.ruleId && hit.f.ruleId === seed.ruleId),
+    reviewer: hit.f.reviewer,
+  });
+}
+
+const falsePositives = findings.filter(
+  (f, i) => !used.has(i) && [...cleanFiles].some((c) => c.endsWith(f.path))
+);
+const fpSet = new Set(falsePositives);
+const unmatched = findings.filter((f, i) => !used.has(i) && !fpSet.has(f));
+
+// Findings citing a rule id that is not in the catalogue: the reviewer invented one.
+const invalidRuleIds = findings.filter(
+  (f) => f.hasRuleId && !rules.has(f.ruleId) && !RESERVED_RULE_IDS.has(f.ruleId)
+);
+
+const byStack = new Map();
+for (const seed of expected) {
+  const entry = byStack.get(seed.stack) ?? { total: 0, caught: 0, ruleMatched: 0 };
+  entry.total += 1;
+  const hit = caught.find((c) => c.file === seed.file && c.id === seed.id);
+  if (hit) {
+    entry.caught += 1;
+    if (hit.ruleMatch) entry.ruleMatched += 1;
+  }
+  byStack.set(seed.stack, entry);
+}
+
+const blockers = expected.filter((s) => s.severity === 'BLOCKER');
+const blockersCaught = caught.filter((s) => s.severity === 'BLOCKER');
+const ruleMatched = caught.filter((c) => c.ruleMatch);
+const withRuleId = findings.filter((f) => f.hasRuleId).length;
+const untagged = findings.filter((f) => !f.tagged).length;
+
+const ratio = (n, d) => (d ? +(n / d).toFixed(3) : 0);
+
+const report = {
+  repo: REPO,
+  pr: Number(PR),
+  scored_at: new Date().toISOString(),
+  standards_version: JSON.parse(readFileSync(join(ROOT, 'standards/manifest.json'), 'utf8')).version,
+  reviewers: [...new Set(findings.map((f) => f.reviewer))],
+  totals: {
+    seeded: expected.length,
+    caught: caught.length,
+    recall: ratio(caught.length, expected.length),
+    blocker_seeded: blockers.length,
+    blocker_caught: blockersCaught.length,
+    blocker_recall: ratio(blockersCaught.length, blockers.length),
+    rule_attribution: ratio(ruleMatched.length, caught.length),
+    comments_posted: findings.length,
+    findings_with_rule_id: withRuleId,
+    invalid_rule_ids: invalidRuleIds.length,
+    false_positives_on_clean: falsePositives.length,
+    unmatched_findings: unmatched.length,
+    untagged_findings: untagged,
+  },
+  by_stack: Object.fromEntries(
+    [...byStack].map(([k, v]) => [
+      k,
+      { ...v, recall: ratio(v.caught, v.total), rule_attribution: ratio(v.ruleMatched, v.caught) },
+    ])
+  ),
+  missed: missed.map((m) => ({ file: m.file, line: m.line, severity: m.severity, rule: m.ruleId, note: m.note })),
+  misattributed: caught
+    .filter((c) => c.ruleId && !c.ruleMatch)
+    .map((c) => ({ file: c.file, line: c.line, expected: c.ruleId, got: c.gotRuleId })),
+  false_positives: falsePositives.map((f) => ({ path: f.path, line: f.line, severity: f.severity, body: f.body })),
+  invalid_rule_ids: invalidRuleIds.map((f) => ({ path: f.path, line: f.line, ruleId: f.ruleId })),
+};
+
+if (HISTORY) {
+  appendFileSync(HISTORY, JSON.stringify(report) + '\n');
+  console.error(`appended to ${HISTORY}`);
+}
+
+// --- regression check against the previous run ------------------------------
+let regression = null;
+if (BASELINE && HISTORY && existsSync(HISTORY)) {
+  const previous = readFileSync(HISTORY, 'utf8')
+    .split('\n')
+    .filter(Boolean)
+    .map((l) => JSON.parse(l))
+    .filter((r) => r.repo === REPO && r.scored_at !== report.scored_at)
+    .pop();
+  if (previous) {
+    const drops = [];
+    if (report.totals.blocker_recall < previous.totals.blocker_recall)
+      drops.push(`BLOCKER recall ${previous.totals.blocker_recall} → ${report.totals.blocker_recall}`);
+    if (report.totals.false_positives_on_clean > previous.totals.false_positives_on_clean)
+      drops.push(`false positives ${previous.totals.false_positives_on_clean} → ${report.totals.false_positives_on_clean}`);
+    if (report.totals.rule_attribution < previous.totals.rule_attribution - 0.05)
+      drops.push(`rule attribution ${previous.totals.rule_attribution} → ${report.totals.rule_attribution}`);
+    regression = { since: previous.scored_at, drops };
+    report.regression = regression;
+  }
+}
+
+if (AS_JSON) {
+  console.log(JSON.stringify(report, null, 2));
+} else {
+  const t = report.totals;
+  const pct = (n) => `${Math.round(n * 100)}%`;
+  console.log(`\nRedline seed score — ${REPO}#${PR} (standards v${report.standards_version})`);
+  console.log(`reviewers: ${report.reviewers.join(', ') || 'none — no bot comments found'}\n`);
+  console.log(`  BLOCKER recall     ${t.blocker_caught}/${t.blocker_seeded}  (${pct(t.blocker_recall)})`);
+  console.log(`  overall recall     ${t.caught}/${t.seeded}  (${pct(t.recall)})`);
+  console.log(`  rule attribution   ${pct(t.rule_attribution)} of caught seeds cited the right rule`);
+  console.log(`  comments posted    ${t.comments_posted} (${t.findings_with_rule_id} carried a rule id)`);
+  console.log(`  false positives    ${t.false_positives_on_clean} on seeded/clean/**`);
+  console.log(`  unmatched          ${t.unmatched_findings} on a seed file but not near a marker`);
+  console.log(`  untagged           ${t.untagged_findings} missing the Redline/<SEVERITY> prefix`);
+  console.log(`  invalid rule ids   ${t.invalid_rule_ids}\n`);
+  console.log('  by stack:');
+  for (const [stack, s] of Object.entries(report.by_stack)) {
+    console.log(
+      `    ${stack.padEnd(16)} ${String(s.caught).padStart(2)}/${String(s.total).padEnd(3)} recall ${pct(s.recall).padStart(4)}   attribution ${pct(s.rule_attribution).padStart(4)}`
+    );
+  }
+  if (report.missed.length) {
+    console.log('\n  missed:');
+    for (const m of report.missed) console.log(`    ${m.file}:${m.line} [${m.severity}] ${m.rule ?? '?'} — ${m.note}`);
+  }
+  if (report.misattributed.length) {
+    console.log('\n  found but wrong rule cited:');
+    for (const m of report.misattributed) console.log(`    ${m.file}:${m.line} expected ${m.expected}, got ${m.got ?? 'none'}`);
+  }
+  if (report.false_positives.length) {
+    console.log('\n  false positives on clean code:');
+    for (const f of report.false_positives) console.log(`    ${f.path}:${f.line} [${f.severity}] ${f.body}`);
+  }
+  if (regression?.drops.length) {
+    console.log(`\n  REGRESSION against ${regression.since}:`);
+    for (const d of regression.drops) console.log(`    ${d}`);
+  }
+}
+
+// A pilot that cannot flag its own BLOCKER seeds is not ready to widen, and a regression
+// against the last recorded run must stop the rollout.
+const failed =
+  report.totals.blocker_recall < 1 ||
+  report.totals.false_positives_on_clean > 0 ||
+  Boolean(regression?.drops.length);
+process.exit(failed ? 1 : 0);