switchboard-cli 0.1.0-alpha.1

package/calibration/skill/calibrate.ts

@@ -0,0 +1,292 @@
+import { join } from "path";
+import { randomBytes } from "crypto";
+import type {
+  CalibrationSeed,
+  CalibrateFlags,
+  DiagnosisCard,
+  EscalationItem,
+  LayerAVerdict,
+  LayerBVerdict,
+  PipelineTrace,
+  RunManifest,
+  RunSummary,
+  TrendEntry,
+  VerdictDiff,
+} from "../types";
+import { parseFlags } from "./cli-flags";
+import { formatRunReport, formatTrendsReport } from "./report";
+import { loadPendingEscalations, formatEscalationForReview, countPendingEscalations } from "./review";
+
+// Seed generators
+import { generateBlindSeeds } from "../seeds/blind-generator";
+import { generateAdversarialSeeds } from "../seeds/adversarial-generator";
+import { loadReplaySeeds } from "../seeds/replay-loader";
+
+// Layer A
+import { runPipeline } from "../internal/harness";
+import { produceLayerAVerdict } from "../internal/verdict-writer";
+
+// Layer B
+import { produceLayerBVerdict } from "../external/verdict-writer";
+
+// Engine
+import {
+  initLedger,
+  appendRun,
+  writeSeed,
+  writeTrace,
+  writeVerdictA,
+  writeVerdictB,
+  writeDiff,
+  writeDiagnosis,
+  writeEscalation,
+  writeSummary,
+  appendHistory,
+} from "../engine/ledger";
+import { captureBaseline } from "../engine/baseline";
+import { computeDiff } from "../engine/diff";
+import { generateDiagnosis } from "../engine/diagnosis";
+import { buildEscalationQueue } from "../engine/escalation";
+import { computeTrends, checkSelfMonitoring } from "../engine/trends";
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+const AGREEMENT_SAMPLE_RATE = 0.2;
+
+// ---------------------------------------------------------------------------
+// Main calibration runner
+// ---------------------------------------------------------------------------
+
+export interface CalibrationResult {
+  report: string;
+  summary: RunSummary;
+  warnings: string[];
+}
+
+export async function runCalibration(
+  args: string,
+  projectRoot: string,
+): Promise<CalibrationResult> {
+  const flags = parseFlags(args);
+  const ledgerPath = join(projectRoot, "calibration", "ledger");
+  const archivePath = join(projectRoot, ".switchboard", "archive");
+
+  // Special modes
+  if (flags.review) {
+    const pending = loadPendingEscalations(ledgerPath);
+    if (pending.length === 0) {
+      return {
+        report: "No pending escalations to review.",
+        summary: emptySummary(),
+        warnings: [],
+      };
+    }
+    const reports = pending.map(formatEscalationForReview);
+    return {
+      report: `${pending.length} pending escalation(s):\n\n${reports.join("\n\n---\n\n")}`,
+      summary: emptySummary(),
+      warnings: [],
+    };
+  }
+
+  if (flags.trends) {
+    const trends = computeTrends(ledgerPath);
+    const warnings = checkSelfMonitoring(trends);
+    return {
+      report: formatTrendsReport(trends),
+      summary: emptySummary(),
+      warnings,
+    };
+  }
+
+  // Initialize ledger
+  initLedger(ledgerPath);
+
+  // Capture baseline
+  const baseline = captureBaseline(ledgerPath);
+
+  // Generate run ID
+  const runId = new Date().toISOString().replace(/[:.]/g, "-");
+
+  // Generate seeds
+  const seeds: CalibrationSeed[] = [];
+  if (flags.blind > 0) seeds.push(...generateBlindSeeds(flags.blind));
+  if (flags.adversarial > 0) seeds.push(...generateAdversarialSeeds(flags.adversarial));
+  if (flags.replay > 0) seeds.push(...loadReplaySeeds(archivePath, flags.replay));
+
+  if (seeds.length === 0) {
+    return {
+      report: "No seeds generated. Check flags.",
+      summary: emptySummary(),
+      warnings: [],
+    };
+  }
+
+  // Write manifest
+  const manifest: RunManifest = {
+    run_id: runId,
+    started_at: new Date().toISOString(),
+    seed_count: seeds.length,
+    blind_count: seeds.filter(s => s.source === "blind").length,
+    adversarial_count: seeds.filter(s => s.source === "adversarial").length,
+    replay_count: seeds.filter(s => s.source === "replay").length,
+    trigger: flags.retestOpen ? "retest" : "manual",
+    baseline_snapshot: baseline.baseline_id,
+  };
+  appendRun(ledgerPath, manifest);
+
+  // Process each seed
+  const traces: PipelineTrace[] = [];
+  const verdictsA: LayerAVerdict[] = [];
+  const verdictsB: LayerBVerdict[] = [];
+  const diffs: VerdictDiff[] = [];
+  const diagnoses: DiagnosisCard[] = [];
+
+  for (const seed of seeds) {
+    // Write seed
+    writeSeed(ledgerPath, runId, seed);
+
+    // Run pipeline (Layer A)
+    const pipelineResult = runPipeline(seed);
+    const trace = pipelineResult.trace;
+    traces.push(trace);
+    writeTrace(ledgerPath, runId, trace);
+
+    // Layer A verdict
+    const verdictA = produceLayerAVerdict(seed, trace);
+    verdictsA.push(verdictA);
+    writeVerdictA(ledgerPath, runId, verdictA);
+
+    // Layer B verdict (independent)
+    const verdictB = produceLayerBVerdict(
+      seed,
+      trace.compiled_spec,
+      trace.receipt_summary,
+      trace.trust_posture,
+    );
+    verdictsB.push(verdictB);
+    writeVerdictB(ledgerPath, runId, verdictB);
+
+    // Diff
+    const diff = computeDiff(verdictA, verdictB, trace.return_mode, AGREEMENT_SAMPLE_RATE);
+    diffs.push(diff);
+    writeDiff(ledgerPath, runId, diff);
+
+    // Diagnosis for disagreements
+    if (diff.is_disagreement) {
+      const diag = generateDiagnosis(seed, diff, trace, verdictA, verdictB);
+      diagnoses.push(diag);
+      writeDiagnosis(ledgerPath, runId, diag);
+    }
+  }
+
+  // Build escalation queue
+  const escalations = buildEscalationQueue(diffs, diagnoses);
+  for (const esc of escalations) {
+    writeEscalation(ledgerPath, runId, esc);
+  }
+
+  // Compute summary
+  const summary = computeRunSummary(runId, seeds, traces, verdictsA, verdictsB, diffs, diagnoses, escalations);
+  writeSummary(ledgerPath, runId, summary);
+
+  // Update manifest with completion
+  const completedManifest = { ...manifest, completed_at: new Date().toISOString() };
+  appendHistory(ledgerPath, completedManifest);
+
+  // Compute trends and check self-monitoring
+  const trends = computeTrends(ledgerPath);
+  const previousTrend = trends.length >= 2 ? trends[trends.length - 2] : undefined;
+  const warnings = checkSelfMonitoring(trends);
+
+  // Pending escalation reminder
+  const pendingCount = countPendingEscalations(ledgerPath);
+  if (pendingCount > 10) {
+    warnings.push(`${pendingCount} unreviewed escalations across runs. Human judgments are calibration fuel.`);
+  }
+
+  const report = formatRunReport(summary, previousTrend);
+
+  return { report, summary, warnings };
+}
+
+// ---------------------------------------------------------------------------
+// Summary computation
+// ---------------------------------------------------------------------------
+
+function computeRunSummary(
+  runId: string,
+  seeds: CalibrationSeed[],
+  traces: PipelineTrace[],
+  verdictsA: LayerAVerdict[],
+  verdictsB: LayerBVerdict[],
+  diffs: VerdictDiff[],
+  diagnoses: DiagnosisCard[],
+  escalations: EscalationItem[],
+): RunSummary {
+  const driftSeeds = traces.filter(t => t.return_mode === "drift");
+  const honestSeeds = traces.filter(t => t.return_mode === "honest");
+
+  // Gate catch rate: % of drift-mode returns correctly flagged
+  const driftCaught = driftSeeds.filter(t => {
+    const va = verdictsA.find(v => v.seed_id === t.seed_id);
+    return va && va.verdict === "fail";
+  }).length;
+  const gate_catch_rate = driftSeeds.length > 0 ? driftCaught / driftSeeds.length : 1;
+
+  // False positive: % of honest returns incorrectly flagged
+  const honestFlagged = honestSeeds.filter(t => {
+    const va = verdictsA.find(v => v.seed_id === t.seed_id);
+    return va && va.verdict === "fail";
+  }).length;
+  const false_positive_rate = honestSeeds.length > 0 ? honestFlagged / honestSeeds.length : 0;
+
+  // False negative: % of drift returns that slipped through
+  const driftPassed = driftSeeds.filter(t => {
+    const va = verdictsA.find(v => v.seed_id === t.seed_id);
+    return va && va.verdict === "pass";
+  }).length;
+  const false_negative_rate = driftSeeds.length > 0 ? driftPassed / driftSeeds.length : 0;
+
+  // Layer agreement
+  const agreements = diffs.filter(d => !d.is_disagreement).length;
+  const layer_agreement_rate = diffs.length > 0 ? agreements / diffs.length : 1;
+
+  // Biggest miss
+  let biggest_miss = "";
+  if (diagnoses.length > 0) {
+    const worst = diagnoses.sort((a, b) =>
+      Math.abs(b.root_cause.threshold_margin) - Math.abs(a.root_cause.threshold_margin)
+    )[0]!;
+    biggest_miss = `${worst.root_cause.detail} (margin: ${worst.root_cause.threshold_margin.toFixed(4)})`;
+  }
+
+  return {
+    run_id: runId,
+    seed_count: seeds.length,
+    gate_catch_rate,
+    false_positive_rate,
+    false_negative_rate,
+    layer_agreement_rate,
+    diagnosis_count: diagnoses.length,
+    escalation_count: escalations.length,
+    biggest_miss,
+    completed_at: new Date().toISOString(),
+  };
+}
+
+function emptySummary(): RunSummary {
+  return {
+    run_id: "",
+    seed_count: 0,
+    gate_catch_rate: 0,
+    false_positive_rate: 0,
+    false_negative_rate: 0,
+    layer_agreement_rate: 0,
+    diagnosis_count: 0,
+    escalation_count: 0,
+    completed_at: new Date().toISOString(),
+  };
+}

package/calibration/skill/cli-flags.ts

@@ -0,0 +1,49 @@
+import { calibrateFlagsSchema, type CalibrateFlags } from "../types";
+
+// ---------------------------------------------------------------------------
+// Parse /calibrate CLI flags
+// ---------------------------------------------------------------------------
+
+export function parseFlags(args: string): CalibrateFlags {
+  const tokens = args.trim().split(/\s+/);
+  const raw: Record<string, unknown> = {};
+
+  for (let i = 0; i < tokens.length; i++) {
+    const tok = tokens[i]!;
+
+    if (tok === "--blind" && tokens[i + 1]) {
+      raw.blind = parseInt(tokens[++i]!, 10);
+    } else if (tok === "--adversarial" && tokens[i + 1]) {
+      raw.adversarial = parseInt(tokens[++i]!, 10);
+    } else if (tok === "--replay" && tokens[i + 1]) {
+      raw.replay = parseInt(tokens[++i]!, 10);
+    } else if (tok === "--adversarial-only") {
+      raw.adversarialOnly = true;
+      if (tokens[i + 1] && !tokens[i + 1]!.startsWith("--")) {
+        raw.adversarial = parseInt(tokens[++i]!, 10);
+      }
+    } else if (tok === "--replay-only") {
+      raw.replayOnly = true;
+    } else if (tok === "--retest") {
+      raw.retestOpen = true;
+    } else if (tok === "--review") {
+      raw.review = true;
+    } else if (tok === "--trends") {
+      raw.trends = true;
+    }
+  }
+
+  // Apply mode overrides
+  if (raw.adversarialOnly) {
+    raw.blind = 0;
+    raw.replay = 0;
+    if (!raw.adversarial) raw.adversarial = 8;
+  }
+  if (raw.replayOnly) {
+    raw.blind = 0;
+    raw.adversarial = 0;
+    if (!raw.replay) raw.replay = 10;
+  }
+
+  return calibrateFlagsSchema.parse(raw);
+}

package/calibration/skill/report.ts

@@ -0,0 +1,80 @@
+import type { RunSummary, TrendEntry } from "../types";
+
+// ---------------------------------------------------------------------------
+// Compact report formatter
+// ---------------------------------------------------------------------------
+
+function arrow(current: number, previous: number | undefined): string {
+  if (previous === undefined) return "";
+  if (current > previous + 0.01) return ` \u25B2 from ${(previous * 100).toFixed(0)}%`;
+  if (current < previous - 0.01) return ` \u25BC from ${(previous * 100).toFixed(0)}%`;
+  return " \u2500 stable";
+}
+
+export function formatRunReport(summary: RunSummary, previousTrend?: TrendEntry): string {
+  const lines: string[] = [];
+
+  lines.push(`CALIBRATION RUN ${summary.completed_at}`);
+  lines.push(`Seeds: ${summary.seed_count}`);
+  lines.push("");
+  lines.push(`Gate catch rate:    ${(summary.gate_catch_rate * 100).toFixed(0)}% (target: >90%)${arrow(summary.gate_catch_rate, previousTrend?.gate_catch_rate)}`);
+  lines.push(`False positive:     ${(summary.false_positive_rate * 100).toFixed(0)}% (target: <10%)${arrow(summary.false_positive_rate, previousTrend?.false_positive_rate)}`);
+  lines.push(`False negative:     ${(summary.false_negative_rate * 100).toFixed(0)}% (target: <5%)${arrow(summary.false_negative_rate, previousTrend?.false_negative_rate)}`);
+  lines.push(`Layer agreement:    ${(summary.layer_agreement_rate * 100).toFixed(0)}%`);
+  lines.push("");
+  lines.push(`${summary.diagnosis_count} diagnosis card(s) generated`);
+  lines.push(`${summary.escalation_count} item(s) in escalation queue`);
+
+  if (summary.biggest_miss) {
+    lines.push("");
+    lines.push(`Biggest miss: ${summary.biggest_miss}`);
+  }
+
+  lines.push("");
+  lines.push("Run: /calibrate --review to see escalation queue");
+
+  return lines.join("\n");
+}
+
+export function formatTrendsReport(trends: TrendEntry[]): string {
+  if (trends.length === 0) {
+    return "No calibration runs recorded yet.";
+  }
+
+  const lines: string[] = [];
+  lines.push("CALIBRATION TRENDS");
+  lines.push("==================");
+  lines.push("");
+  lines.push("Run ID                          | Catch | FP   | FN   | Agreement | Blind Pass");
+  lines.push("---                             | ---   | ---  | ---  | ---       | ---");
+
+  for (const t of trends.slice(-10)) {
+    lines.push(
+      `${t.run_id.slice(0, 30).padEnd(32)}| ` +
+      `${(t.gate_catch_rate * 100).toFixed(0).padStart(3)}% | ` +
+      `${(t.false_positive_rate * 100).toFixed(0).padStart(3)}% | ` +
+      `${(t.false_negative_rate * 100).toFixed(0).padStart(3)}% | ` +
+      `${(t.layer_agreement_rate * 100).toFixed(0).padStart(7)}%  | ` +
+      `${(t.pass_rate_blind * 100).toFixed(0).padStart(3)}%`
+    );
+  }
+
+  // Direction summary
+  if (trends.length >= 2) {
+    const latest = trends[trends.length - 1]!;
+    const prev = trends[trends.length - 2]!;
+    lines.push("");
+
+    if (latest.false_negative_rate > prev.false_negative_rate) {
+      lines.push("WARNING: False negative rate is increasing. Machinery may be getting softer.");
+    }
+    if (latest.false_positive_rate > prev.false_positive_rate + 0.05) {
+      lines.push("WARNING: False positive rate spiked. Machinery may be getting paranoid.");
+    }
+    if (latest.pass_rate_blind > 0.9 && prev.pass_rate_blind > 0.9) {
+      lines.push("WARNING: Blind seed pass rate >90% for 2+ runs. Seeds may be too easy.");
+    }
+  }
+
+  return lines.join("\n");
+}

package/calibration/skill/review.ts

@@ -0,0 +1,118 @@
+import { readFileSync, writeFileSync, existsSync, readdirSync } from "fs";
+import { join } from "path";
+import YAML from "yaml";
+import type { EscalationItem } from "../types";
+
+// ---------------------------------------------------------------------------
+// Escalation review interface
+// ---------------------------------------------------------------------------
+
+/**
+ * Load all pending escalation items across all runs.
+ */
+export function loadPendingEscalations(basePath: string): EscalationItem[] {
+  const runsDir = join(basePath, "runs");
+  if (!existsSync(runsDir)) return [];
+
+  const pending: EscalationItem[] = [];
+  const runDirs = readdirSync(runsDir).sort();
+
+  for (const runDir of runDirs) {
+    const escDir = join(runsDir, runDir, "escalations");
+    if (!existsSync(escDir)) continue;
+
+    const files = readdirSync(escDir).filter(f => f.endsWith(".yaml"));
+    for (const file of files) {
+      const content = readFileSync(join(escDir, file), "utf-8");
+      const item = YAML.parse(content) as EscalationItem;
+      if (item.human_judgment === "pending") {
+        pending.push(item);
+      }
+    }
+  }
+
+  return pending;
+}
+
+/**
+ * Format a single escalation item for human review.
+ */
+export function formatEscalationForReview(item: EscalationItem): string {
+  const lines: string[] = [];
+
+  lines.push(`ESCALATION: ${item.id}`);
+  lines.push(`Seed: ${item.seed_id}`);
+  lines.push("");
+  lines.push(`Layer A verdict: ${item.diff.layer_a_verdict}`);
+  lines.push(`Layer B verdict: ${item.diff.layer_b_verdict}`);
+  lines.push(`Return mode: ${item.diff.return_mode}`);
+  lines.push(`Category: ${item.diff.category}`);
+  lines.push("");
+  lines.push(`Reasoning: ${item.diff.reasoning}`);
+
+  if (item.diagnosis) {
+    lines.push("");
+    lines.push("DIAGNOSIS:");
+    lines.push(`  Root cause: ${item.diagnosis.root_cause.detail}`);
+    if (item.diagnosis.root_cause.gate_that_should_have_fired) {
+      lines.push(`  Gate that should have fired: ${item.diagnosis.root_cause.gate_that_should_have_fired}`);
+      lines.push(`  Threshold margin: ${item.diagnosis.root_cause.threshold_margin.toFixed(4)}`);
+    }
+    lines.push(`  Recommended fix: ${item.diagnosis.recommended_fix.type}`);
+    lines.push(`  Justification: ${item.diagnosis.recommended_fix.justification}`);
+    if (item.diagnosis.recommended_fix.parameter) {
+      lines.push(`  Parameter: ${item.diagnosis.recommended_fix.parameter} (${item.diagnosis.recommended_fix.current_value} -> ${item.diagnosis.recommended_fix.proposed_value})`);
+    }
+  }
+
+  lines.push("");
+  lines.push("Your judgment: agree_a | agree_b | neither");
+
+  return lines.join("\n");
+}
+
+/**
+ * Record a human judgment on an escalation item.
+ */
+export function recordJudgment(
+  basePath: string,
+  item: EscalationItem,
+  judgment: "agree_a" | "agree_b" | "neither",
+  reasoning: string,
+): EscalationItem {
+  const updated: EscalationItem = {
+    ...item,
+    human_judgment: judgment,
+    human_reasoning: reasoning,
+    reviewed_at: new Date().toISOString(),
+  };
+
+  // Find and update the escalation file
+  const runsDir = join(basePath, "runs");
+  const runDirs = readdirSync(runsDir).sort();
+
+  for (const runDir of runDirs) {
+    const escDir = join(runsDir, runDir, "escalations");
+    if (!existsSync(escDir)) continue;
+
+    const files = readdirSync(escDir).filter(f => f.endsWith(".yaml"));
+    for (const file of files) {
+      const filePath = join(escDir, file);
+      const content = readFileSync(filePath, "utf-8");
+      const parsed = YAML.parse(content) as EscalationItem;
+      if (parsed.id === item.id) {
+        writeFileSync(filePath, YAML.stringify(updated), "utf-8");
+        return updated;
+      }
+    }
+  }
+
+  return updated;
+}
+
+/**
+ * Count pending escalations.
+ */
+export function countPendingEscalations(basePath: string): number {
+  return loadPendingEscalations(basePath).length;
+}