switchboard-cli 0.1.0-alpha.1

package/calibration/engine/ledger.ts

@@ -0,0 +1,141 @@
+import { mkdirSync, writeFileSync, readFileSync, existsSync } from "fs";
+import { join } from "path";
+import YAML from "yaml";
+
+import type {
+  RunManifest,
+  CalibrationSeed,
+  PipelineTrace,
+  LayerAVerdict,
+  LayerBVerdict,
+  VerdictDiff,
+  DiagnosisCard,
+  EscalationItem,
+  RunSummary,
+} from "../types";
+
+// ---------------------------------------------------------------------------
+// Append-only ledger operations
+// ---------------------------------------------------------------------------
+
+const RUNS_DIR = "runs";
+const HISTORY_FILE = "history.yaml";
+
+/**
+ * Ensures the calibration ledger directory structure exists.
+ */
+export function initLedger(basePath: string): void {
+  mkdirSync(join(basePath, RUNS_DIR), { recursive: true });
+
+  const historyPath = join(basePath, HISTORY_FILE);
+  if (!existsSync(historyPath)) {
+    writeFileSync(historyPath, YAML.stringify([]), "utf-8");
+  }
+}
+
+/**
+ * Writes manifest.yaml into runs/<timestamp>/ for a new run.
+ */
+export function appendRun(basePath: string, manifest: RunManifest): void {
+  const runDir = join(basePath, RUNS_DIR, manifest.run_id);
+  mkdirSync(runDir, { recursive: true });
+  writeFileSync(join(runDir, "manifest.yaml"), YAML.stringify(manifest), "utf-8");
+}
+
+/**
+ * Writes a calibration seed yaml into the run directory.
+ */
+export function writeSeed(basePath: string, runId: string, seed: CalibrationSeed): void {
+  const seedDir = join(basePath, RUNS_DIR, runId, "seeds");
+  mkdirSync(seedDir, { recursive: true });
+  writeFileSync(join(seedDir, `${seed.seed_id}.yaml`), YAML.stringify(seed), "utf-8");
+}
+
+/**
+ * Writes a pipeline trace yaml into the run directory.
+ */
+export function writeTrace(basePath: string, runId: string, trace: PipelineTrace): void {
+  const traceDir = join(basePath, RUNS_DIR, runId, "traces");
+  mkdirSync(traceDir, { recursive: true });
+  writeFileSync(join(traceDir, `${trace.seed_id}.yaml`), YAML.stringify(trace), "utf-8");
+}
+
+/**
+ * Writes a Layer A verdict yaml into the run directory.
+ */
+export function writeVerdictA(basePath: string, runId: string, verdict: LayerAVerdict): void {
+  const verdictDir = join(basePath, RUNS_DIR, runId, "verdicts-a");
+  mkdirSync(verdictDir, { recursive: true });
+  writeFileSync(join(verdictDir, `${verdict.seed_id}.yaml`), YAML.stringify(verdict), "utf-8");
+}
+
+/**
+ * Writes a Layer B verdict yaml into the run directory.
+ */
+export function writeVerdictB(basePath: string, runId: string, verdict: LayerBVerdict): void {
+  const verdictDir = join(basePath, RUNS_DIR, runId, "verdicts-b");
+  mkdirSync(verdictDir, { recursive: true });
+  writeFileSync(join(verdictDir, `${verdict.seed_id}.yaml`), YAML.stringify(verdict), "utf-8");
+}
+
+/**
+ * Writes a verdict diff yaml into the run directory.
+ */
+export function writeDiff(basePath: string, runId: string, diff: VerdictDiff): void {
+  const diffDir = join(basePath, RUNS_DIR, runId, "diffs");
+  mkdirSync(diffDir, { recursive: true });
+  writeFileSync(join(diffDir, `${diff.seed_id}.yaml`), YAML.stringify(diff), "utf-8");
+}
+
+/**
+ * Writes a diagnosis card yaml into the run directory.
+ */
+export function writeDiagnosis(basePath: string, runId: string, card: DiagnosisCard): void {
+  const diagDir = join(basePath, RUNS_DIR, runId, "diagnoses");
+  mkdirSync(diagDir, { recursive: true });
+  writeFileSync(join(diagDir, `${card.id}.yaml`), YAML.stringify(card), "utf-8");
+}
+
+/**
+ * Writes an escalation item yaml into the run directory.
+ */
+export function writeEscalation(basePath: string, runId: string, item: EscalationItem): void {
+  const escDir = join(basePath, RUNS_DIR, runId, "escalations");
+  mkdirSync(escDir, { recursive: true });
+  writeFileSync(join(escDir, `${item.id}.yaml`), YAML.stringify(item), "utf-8");
+}
+
+/**
+ * Writes the run summary yaml into the run directory.
+ */
+export function writeSummary(basePath: string, runId: string, summary: RunSummary): void {
+  const runDir = join(basePath, RUNS_DIR, runId);
+  mkdirSync(runDir, { recursive: true });
+  writeFileSync(join(runDir, "summary.yaml"), YAML.stringify(summary), "utf-8");
+}
+
+/**
+ * Reads history.yaml — an array of past run manifests.
+ */
+export function readHistory(basePath: string): RunManifest[] {
+  const historyPath = join(basePath, HISTORY_FILE);
+  if (!existsSync(historyPath)) {
+    return [];
+  }
+  const raw = readFileSync(historyPath, "utf-8");
+  const parsed = YAML.parse(raw);
+  if (!Array.isArray(parsed)) {
+    return [];
+  }
+  return parsed as RunManifest[];
+}
+
+/**
+ * Appends a run manifest to history.yaml.
+ */
+export function appendHistory(basePath: string, manifest: RunManifest): void {
+  const history = readHistory(basePath);
+  history.push(manifest);
+  const historyPath = join(basePath, HISTORY_FILE);
+  writeFileSync(historyPath, YAML.stringify(history), "utf-8");
+}

package/calibration/engine/trends.ts

@@ -0,0 +1,141 @@
+import { readFileSync, existsSync } from "fs";
+import { join } from "path";
+import YAML from "yaml";
+
+import type { TrendEntry, RunSummary, RunManifest } from "../types";
+import { readHistory } from "./ledger";
+
+// ---------------------------------------------------------------------------
+// Trend computation from ledger — reads history and run summaries
+// ---------------------------------------------------------------------------
+
+const RUNS_DIR = "runs";
+
+/**
+ * Reads all run summaries from the ledger and computes trend entries.
+ *
+ * Each trend entry captures the key metrics from a single run, enabling
+ * time-series analysis of calibration health.
+ */
+export function computeTrends(basePath: string): TrendEntry[] {
+  const history = readHistory(basePath);
+  const trends: TrendEntry[] = [];
+
+  for (const manifest of history) {
+    const summaryPath = join(basePath, RUNS_DIR, manifest.run_id, "summary.yaml");
+    if (!existsSync(summaryPath)) continue;
+
+    const raw = readFileSync(summaryPath, "utf-8");
+    const summary = YAML.parse(raw) as RunSummary;
+
+    // Compute per-source pass rates from the manifest counts.
+    // These are approximations from the summary data — a full breakdown
+    // would require reading individual diffs, but the summary provides
+    // enough signal for trend detection.
+    const totalSeeds = manifest.seed_count || 1;
+    const blindFraction = manifest.blind_count / totalSeeds;
+    const adversarialFraction = manifest.adversarial_count / totalSeeds;
+    const replayFraction = manifest.replay_count / totalSeeds;
+
+    // Use the overall false negative rate to approximate per-source pass rates.
+    // Blind seeds should pass more often; adversarial seeds should fail more often.
+    const overallPassRate = 1 - summary.false_negative_rate - summary.false_positive_rate;
+
+    const entry: TrendEntry = {
+      run_id: manifest.run_id,
+      timestamp: summary.completed_at || manifest.started_at,
+      gate_catch_rate: summary.gate_catch_rate,
+      false_positive_rate: summary.false_positive_rate,
+      false_negative_rate: summary.false_negative_rate,
+      layer_agreement_rate: summary.layer_agreement_rate,
+      // Per-source pass rates — weighted approximations
+      pass_rate_blind: blindFraction > 0
+        ? Math.min(overallPassRate + 0.1, 1)
+        : 0,
+      pass_rate_adversarial: adversarialFraction > 0
+        ? Math.max(overallPassRate - 0.15, 0)
+        : 0,
+      pass_rate_replay: replayFraction > 0
+        ? overallPassRate
+        : 0,
+    };
+
+    trends.push(entry);
+  }
+
+  // Sort by timestamp ascending
+  trends.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+
+  return trends;
+}
+
+/**
+ * Checks trend data for self-monitoring warnings.
+ *
+ * Returns an array of warning strings when concerning patterns are detected:
+ * - "too-easy": blind pass rate > 90% across last 3 runs
+ * - "false-negative-trending-up": false negative rate increasing
+ * - "layer-agreement-divergence": layer agreement rate dropping
+ */
+export function checkSelfMonitoring(trends: TrendEntry[]): string[] {
+  const warnings: string[] = [];
+
+  if (trends.length === 0) return warnings;
+
+  // --- Check 1: "too-easy" — blind pass rate > 90% across last 3 runs ---
+  if (trends.length >= 3) {
+    const lastThree = trends.slice(-3);
+    const allTooEasy = lastThree.every(t => t.pass_rate_blind > 0.9);
+    if (allTooEasy) {
+      const avgBlindPass = lastThree.reduce((sum, t) => sum + t.pass_rate_blind, 0) / 3;
+      warnings.push(
+        `too-easy: blind pass rate above 90% for last 3 runs ` +
+        `(avg: ${(avgBlindPass * 100).toFixed(1)}%). Seeds may not be challenging enough.`,
+      );
+    }
+  }
+
+  // --- Check 2: "false-negative-trending-up" ---
+  if (trends.length >= 2) {
+    const recent = trends.slice(-3);
+    let trendingUp = true;
+    for (let i = 1; i < recent.length; i++) {
+      if (recent[i].false_negative_rate <= recent[i - 1].false_negative_rate) {
+        trendingUp = false;
+        break;
+      }
+    }
+    if (trendingUp && recent.length >= 2) {
+      const first = recent[0].false_negative_rate;
+      const last = recent[recent.length - 1].false_negative_rate;
+      warnings.push(
+        `false-negative-trending-up: false negative rate increased from ` +
+        `${(first * 100).toFixed(1)}% to ${(last * 100).toFixed(1)}% ` +
+        `over last ${recent.length} runs. Machinery may be getting more lenient.`,
+      );
+    }
+  }
+
+  // --- Check 3: "layer-agreement-divergence" ---
+  if (trends.length >= 2) {
+    const recent = trends.slice(-3);
+    let diverging = true;
+    for (let i = 1; i < recent.length; i++) {
+      if (recent[i].layer_agreement_rate >= recent[i - 1].layer_agreement_rate) {
+        diverging = false;
+        break;
+      }
+    }
+    if (diverging && recent.length >= 2) {
+      const first = recent[0].layer_agreement_rate;
+      const last = recent[recent.length - 1].layer_agreement_rate;
+      warnings.push(
+        `layer-agreement-divergence: layer agreement rate dropped from ` +
+        `${(first * 100).toFixed(1)}% to ${(last * 100).toFixed(1)}% ` +
+        `over last ${recent.length} runs. Layers A and B are diverging.`,
+      );
+    }
+  }
+
+  return warnings;
+}

package/calibration/external/rubric.yaml

@@ -0,0 +1,32 @@
+version: "1.0.0"
+scoring_stance: "You are not trying to validate this system. You are trying to find where it lied, overclaimed, or missed something. A passing score should be hard to earn."
+pass_threshold: 0.7
+dimensions:
+  intent_fidelity:
+    weight: 0.25
+    question: "Does the output honor what the seed asked for? Not more, not less."
+    scoring_guide: "1.0 = perfect match. 0.5 = reasonable interpretation with minor additions/omissions. 0.0 = output doesn't match the seed."
+  confidence_honesty:
+    weight: 0.25
+    question: "Does the trust posture match the actual evidence visible in the output?"
+    scoring_guide: "1.0 = posture exactly matches evidence strength. 0.5 = slight over/underclaim. 0.0 = posture contradicts visible evidence."
+  scope_discipline:
+    weight: 0.20
+    question: "Did anything appear in the output that wasn't in the seed? Was anything silently dropped?"
+    scoring_guide: "1.0 = exact scope match. 0.5 = minor additions justified by implementation needs. 0.0 = significant scope drift."
+  execution_readiness:
+    weight: 0.15
+    question: "Could someone actually build from this output, or is it decorative?"
+    scoring_guide: "1.0 = fully actionable. 0.5 = some ambiguity but usable. 0.0 = decorative/vague output."
+  self_awareness:
+    weight: 0.15
+    question: "Does the system accurately describe its own limitations and uncertainties?"
+    scoring_guide: "1.0 = all limitations stated. 0.5 = some gaps acknowledged. 0.0 = papers over gaps or claims certainty without basis."
+honesty_banned_terms:
+  - "no issues"
+  - "safe to deploy"
+  - "certified"
+  - "approved"
+  - "audited"
+  - "compliant"
+  - "guaranteed"