switchboard-cli 0.1.0-alpha.1

package/README.md

@@ -0,0 +1,122 @@
+# @switchboard/cli
+
+Portable governance substrate for AI workflows.
+
+Compile specs, dispatch to Claude/ChatGPT/Cursor/Codex, ingest structured returns, issue trust receipts, evaluate independently, and calibrate your pipeline.
+
+> **Alpha release.** Expect breaking changes. [Report issues](https://github.com/switchboard-foundation/switchboard/issues).
+
+## Install
+
+```bash
+npm install -g @switchboard/cli
+```
+
+Requires Node >= 18.
+
+## Quick start
+
+```bash
+# Initialize a governed project
+sb init
+
+# Compile project contract + spec from canonical state
+sb compile
+
+# Generate a dispatch packet for a surface
+sb packet claude    # or: chatgpt, cursor, codex
+
+# Launch governed dispatch to Claude Code
+sb run claude
+
+# Ingest the structured return
+sb ingest
+
+# Issue an immutable trust receipt
+sb receipt
+
+# Export receipt as standalone JSON
+sb receipt --export json
+
+# Run independent evaluation
+sb evaluate
+
+# Run the two-layer calibration harness
+sb calibrate
+```
+
+## Pipeline
+
+```
+init -> compile -> packet -> run -> ingest -> receipt -> evaluate
+                                                            |
+                                                        calibrate
+```
+
+Each step reads from and writes to `.switchboard/` in your project root. All state is file-based (YAML + Markdown). No database, no hosted service.
+
+## Commands
+
+| Command | What it does |
+|---------|-------------|
+| `sb init` | Scaffold `.switchboard/` directory with contract, working state, and spec |
+| `sb compile` | Compile a loop contract from canonical state |
+| `sb packet <surface>` | Generate a governed dispatch packet (SBX bundle) |
+| `sb run claude` | Dispatch to Claude Code via Agent SDK with proof capture |
+| `sb ingest` | Ingest structured SB_RETURN.yaml, run reconciliation + gates |
+| `sb receipt` | Build immutable ReceiptIssuedV2 from persisted ingest artifacts |
+| `sb receipt --export json` | Export receipt as standalone JSON file |
+| `sb receipt --export yaml` | Export receipt as standalone YAML file |
+| `sb evaluate` | Run independent evaluation on the latest receipt |
+| `sb audit codex` | Generate Codex audit packet for independent review |
+| `sb calibrate` | Run two-layer calibration harness |
+
+## Calibration
+
+The calibration harness tests the trust machinery itself:
+
+- **Layer A** (internal): Runs seeds through the real governance pipeline with full trace access
+- **Layer B** (external): Scores independently with zero imports from core — only sees raw output
+- **Blind seeds**: Fresh ideas the system has never seen
+- **Adversarial seeds**: Designed to break the machinery (scope traps, non-goal evasion, evidence gaps)
+- **Living ledger**: Every run appends to an auditable history with trends
+
+```bash
+# Default run: 5 blind + 3 adversarial seeds
+sb calibrate
+
+# Custom mix
+sb calibrate --blind 10 --adversarial 5
+
+# Stress test only
+sb calibrate --adversarial-only 8
+
+# Review pending escalations
+sb calibrate --review
+
+# Show calibration trends
+sb calibrate --trends
+```
+
+## Trust machinery
+
+The governance pipeline enforces:
+
+- **Honesty guardrails**: Banned terms ("certified", "guaranteed", "no issues") in receipts
+- **Reconciliation**: Six drift checks (scope inflation, non-goal violation, goal drift, target-user shift, untracked decisions, evidence contradiction)
+- **Gates**: Approval gates for scope changes, unparented decisions, weak evidence, trust boundary changes
+- **Claim spread**: Deterministic measurement of how far executor claims drift from proven evidence
+- **Receipts**: Immutable trust artifacts with claim assessments, verification results, and closure basis
+
+## What this is NOT
+
+- Not a chat wrapper
+- Not an LLM gateway
+- Not an eval platform
+- Not an autonomous agent
+
+It is a governance substrate. It preserves project coherence while you bounce work across AI surfaces.
+
+## License
+
+MIT

package/bin/switchboard.mjs

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+
+/**
+ * Switchboard CLI bin shim.
+ *
+ * Spawns tsx on the real TypeScript entrypoint so the CLI is invocable
+ * without a build step. Works from any cwd — resolves paths relative
+ * to this file, not the caller.
+ */
+
+import { execFileSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { dirname, resolve } from "node:path";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const entry = resolve(__dirname, "..", "src", "index.ts");
+
+// Resolve tsx from this package's own node_modules first, then fall
+// back to a global/hoisted location.
+const tsxPaths = [
+  resolve(__dirname, "..", "node_modules", ".bin", "tsx"),
+  "tsx",
+];
+
+let tsxBin;
+for (const candidate of tsxPaths) {
+  try {
+    execFileSync(candidate, ["--version"], { stdio: "ignore" });
+    tsxBin = candidate;
+    break;
+  } catch {
+    // try next
+  }
+}
+
+if (!tsxBin) {
+  console.error("switchboard: could not find tsx. Run `pnpm install` in the monorepo root.");
+  process.exit(1);
+}
+
+try {
+  execFileSync(tsxBin, [entry, ...process.argv.slice(2)], {
+    stdio: "inherit",
+    cwd: process.cwd(),
+  });
+} catch (err) {
+  // execFileSync throws on non-zero exit; the child already printed output
+  process.exit(err.status ?? 1);
+}

package/calibration/engine/baseline.ts

@@ -0,0 +1,93 @@
+import { mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync } from "fs";
+import { join } from "path";
+import YAML from "yaml";
+
+import type { BaselineSnapshot } from "../types";
+import { getAxisWeights } from "@switchboard/core";
+
+// ---------------------------------------------------------------------------
+// Threshold snapshot management
+// ---------------------------------------------------------------------------
+
+const BASELINES_DIR = "baselines";
+
+/**
+ * Captures a baseline snapshot of current axis weights from the
+ * core claim-spread module and persists it to the baselines directory.
+ *
+ * Baseline IDs follow the format: baseline-<ISO-date>
+ */
+export function captureBaseline(basePath: string): BaselineSnapshot {
+  const baselinesDir = join(basePath, BASELINES_DIR);
+  mkdirSync(baselinesDir, { recursive: true });
+
+  const now = new Date().toISOString();
+  const baselineId = `baseline-${now.split("T")[0]}`;
+
+  const axisWeights = getAxisWeights();
+
+  // Gate thresholds derived from axis weights —
+  // each axis weight implies a threshold at which that axis triggers concern.
+  // We use the weights themselves as the threshold baseline since they represent
+  // the relative importance (and therefore the sensitivity boundary) of each axis.
+  const gateThresholds: Record<string, number> = {};
+  for (const [axis, weight] of Object.entries(axisWeights)) {
+    gateThresholds[axis] = weight;
+  }
+
+  // Reconcile thresholds — the composite spread score thresholds
+  // for reconciliation decisions.
+  const reconcileThresholds: Record<string, number> = {
+    drift_warning: 0.3,
+    drift_block: 0.6,
+    reconcile_required: 0.5,
+  };
+
+  const snapshot: BaselineSnapshot = {
+    baseline_id: baselineId,
+    created_at: now,
+    axis_weights: axisWeights as Record<string, number>,
+    gate_thresholds: gateThresholds,
+    reconcile_thresholds: reconcileThresholds,
+    notes: "",
+  };
+
+  writeFileSync(
+    join(baselinesDir, `${baselineId}.yaml`),
+    YAML.stringify(snapshot),
+    "utf-8",
+  );
+
+  return snapshot;
+}
+
+/**
+ * Reads the most recent baseline snapshot, or null if none exist.
+ */
+export function readLatestBaseline(basePath: string): BaselineSnapshot | null {
+  const baselines = listBaselines(basePath);
+  if (baselines.length === 0) return null;
+
+  // Sort by created_at descending and return the most recent
+  baselines.sort((a, b) => b.created_at.localeCompare(a.created_at));
+  return baselines[0] ?? null;
+}
+
+/**
+ * Lists all baseline snapshots from the baselines directory.
+ */
+export function listBaselines(basePath: string): BaselineSnapshot[] {
+  const baselinesDir = join(basePath, BASELINES_DIR);
+  if (!existsSync(baselinesDir)) return [];
+
+  const files = readdirSync(baselinesDir).filter(f => f.endsWith(".yaml"));
+  const snapshots: BaselineSnapshot[] = [];
+
+  for (const file of files) {
+    const raw = readFileSync(join(baselinesDir, file), "utf-8");
+    const parsed = YAML.parse(raw) as BaselineSnapshot;
+    snapshots.push(parsed);
+  }
+
+  return snapshots;
+}

package/calibration/engine/diagnosis.ts

@@ -0,0 +1,191 @@
+import { randomBytes } from "crypto";
+
+import type {
+  CalibrationSeed,
+  VerdictDiff,
+  PipelineTrace,
+  LayerAVerdict,
+  LayerBVerdict,
+  DiagnosisCard,
+} from "../types";
+
+// ---------------------------------------------------------------------------
+// Diagnosis card generation — root cause analysis for disagreements
+// ---------------------------------------------------------------------------
+
+/**
+ * Generates a diagnosis card for a disagreement between Layer A and Layer B.
+ *
+ * Only called when the diff is a disagreement (a_pass_b_fail, a_fail_b_pass,
+ * or both_fail_different). Analyzes the trace, verdicts, and seed to determine
+ * root cause and recommend a fix.
+ */
+export function generateDiagnosis(
+  seed: CalibrationSeed,
+  diff: VerdictDiff,
+  trace: PipelineTrace,
+  verdictA: LayerAVerdict,
+  verdictB: LayerBVerdict,
+): DiagnosisCard {
+  const id = `diag-${randomBytes(8).toString("hex")}`;
+  const now = new Date().toISOString();
+
+  // --- Root cause analysis ---
+  let gateThatShouldHaveFired = "";
+  let thresholdMargin = 0;
+  let evidenceMissed = "";
+  let detail = "";
+
+  // --- Recommended fix ---
+  let fixType: DiagnosisCard["recommended_fix"]["type"];
+  let fixParameter = "";
+  let fixCurrentValue = 0;
+  let fixProposedValue = 0;
+  let fixJustification = "";
+
+  if (diff.category === "a_pass_b_fail") {
+    // Machinery missed something — Layer A was too lenient
+    // Check which gate should have fired based on trace data
+    detail = `Layer A passed but Layer B failed. The trust machinery missed a problem ` +
+      `that independent scoring caught. Seed source: ${seed.source}, ` +
+      `complexity: ${seed.complexity_target}.`;
+
+    // Find the closest threshold proximity — the gate that almost fired
+    type ProxEntry = { threshold: number; actual: number; margin: number };
+    const proximities = Object.entries(verdictA.threshold_proximity) as Array<[string, ProxEntry]>;
+    if (proximities.length > 0) {
+      // Sort by margin ascending — smallest margin = closest to firing
+      proximities.sort(([, a], [, b]) => a.margin - b.margin);
+      const [closestGate, closestData] = proximities[0]!;
+      gateThatShouldHaveFired = closestGate;
+      thresholdMargin = closestData.margin;
+
+      // Find what Layer B caught that A missed
+      type DimEntry = { score: number; justification: string };
+      const dims = verdictB.dimensions as Record<string, DimEntry>;
+      const lowDimensions = Object.entries(dims)
+        .filter(([, dim]) => dim.score < 0.5)
+        .map(([name, dim]) => `${name} (${dim.score.toFixed(2)}): ${dim.justification}`);
+      evidenceMissed = lowDimensions.join("; ");
+    } else {
+      // No threshold proximity data — the machinery had no close calls
+      gateThatShouldHaveFired = "unknown";
+      thresholdMargin = 1;
+      evidenceMissed = `Layer B justification: ${verdictB.justification}`;
+    }
+
+    // Determine fix type based on threshold proximity
+    if (thresholdMargin < 0.1) {
+      fixType = "threshold_tuning";
+      fixParameter = gateThatShouldHaveFired;
+      const prox = verdictA.threshold_proximity[gateThatShouldHaveFired];
+      fixCurrentValue = prox?.threshold ?? 0;
+      // Tighten the threshold — lower it so this gate fires
+      fixProposedValue = fixCurrentValue > 0
+        ? fixCurrentValue * 0.9
+        : 0.1;
+      fixJustification =
+        `Gate "${gateThatShouldHaveFired}" was within ${(thresholdMargin * 100).toFixed(1)}% ` +
+        `of firing. Lowering threshold should catch this class of miss.`;
+    } else {
+      // Margin is wide — the existing gates are not close to catching this
+      const hasRelevantGates = trace.gates_triggered.length > 0;
+      if (hasRelevantGates) {
+        fixType = "scoring_dimension_gap";
+        fixJustification =
+          `Gates fired (${trace.gates_triggered.join(", ")}) but still passed. ` +
+          `The scoring dimensions may not weight this failure mode heavily enough.`;
+      } else {
+        fixType = "new_gate";
+        fixJustification =
+          `No gates were close to firing (margin: ${(thresholdMargin * 100).toFixed(1)}%). ` +
+          `A new gate may be needed to catch this class of failure.`;
+      }
+    }
+  } else if (diff.category === "a_fail_b_pass") {
+    // Machinery was too strict — Layer A failed but B passed
+    detail = `Layer A failed but Layer B passed. The trust machinery was too strict. ` +
+      `Gates triggered: [${verdictA.gates_triggered.join(", ")}]. ` +
+      `Layer B composite: ${verdictB.composite_score.toFixed(3)}.`;
+
+    // The gate that fired was too aggressive
+    if (verdictA.gates_triggered.length > 0) {
+      gateThatShouldHaveFired = verdictA.gates_triggered[0];
+    }
+
+    // Check threshold proximity for the triggered gate
+    const triggeredProx = verdictA.threshold_proximity[gateThatShouldHaveFired];
+    if (triggeredProx) {
+      thresholdMargin = triggeredProx.margin;
+    }
+
+    evidenceMissed = `Layer B passed with composite ${verdictB.composite_score.toFixed(3)}. ` +
+      `All dimensions above threshold.`;
+
+    if (thresholdMargin < 0.1) {
+      fixType = "threshold_tuning";
+      fixParameter = gateThatShouldHaveFired;
+      fixCurrentValue = triggeredProx?.threshold ?? 0;
+      // Loosen the threshold — raise it so this gate doesn't fire
+      fixProposedValue = fixCurrentValue > 0
+        ? fixCurrentValue * 1.1
+        : 0.5;
+      fixJustification =
+        `Gate "${gateThatShouldHaveFired}" fired with margin ${(thresholdMargin * 100).toFixed(1)}%. ` +
+        `Raising threshold should reduce false positives for this pattern.`;
+    } else {
+      fixType = "scoring_dimension_gap";
+      fixJustification =
+        `Layer A rejected with gates [${verdictA.gates_triggered.join(", ")}] but ` +
+        `Layer B passed convincingly (composite: ${verdictB.composite_score.toFixed(3)}). ` +
+        `Gate scoring may be miscalibrated for this seed type.`;
+    }
+  } else {
+    // both_fail_different — different failure models
+    detail = `Both layers failed but for different reasons. ` +
+      `Layer A gates: [${verdictA.gates_triggered.join(", ")}]. ` +
+      `Layer B composite: ${verdictB.composite_score.toFixed(3)}. ` +
+      `The failure models diverge, suggesting a scoring dimension gap.`;
+
+    const bothDims = verdictB.dimensions as Record<string, { score: number; justification: string }>;
+    const lowDimensions = Object.entries(bothDims)
+      .filter(([, dim]) => dim.score < 0.5)
+      .map(([name]) => name);
+
+    evidenceMissed = `Layer A focused on gates [${verdictA.gates_triggered.join(", ")}], ` +
+      `Layer B flagged dimensions [${lowDimensions.join(", ")}].`;
+
+    fixType = "scoring_dimension_gap";
+    fixJustification =
+      `Failure models diverge: machinery gates and independent scoring ` +
+      `are catching different problems. Alignment of scoring dimensions needed.`;
+  }
+
+  return {
+    id,
+    seed_id: seed.seed_id,
+    layer_a_verdict: verdictA.verdict,
+    layer_b_verdict: verdictB.verdict,
+    return_mode: diff.return_mode,
+    root_cause: {
+      gate_that_should_have_fired: gateThatShouldHaveFired,
+      threshold_margin: thresholdMargin,
+      evidence_missed: evidenceMissed,
+      detail,
+    },
+    recommended_fix: {
+      type: fixType,
+      parameter: fixParameter,
+      current_value: fixCurrentValue,
+      proposed_value: fixProposedValue,
+      justification: fixJustification,
+    },
+    verification_plan: {
+      retest_this_seed: true,
+      retest_full_suite: true,
+      regression_check: true,
+    },
+    status: "open",
+    created_at: now,
+  };
+}

package/calibration/engine/diff.ts

@@ -0,0 +1,118 @@
+import type { LayerAVerdict, LayerBVerdict, VerdictDiff } from "../types";
+
+type ReturnMode = "honest" | "drift";
+
+// ---------------------------------------------------------------------------
+// Verdict comparison — computes the diff between Layer A and Layer B verdicts
+// ---------------------------------------------------------------------------
+
+/**
+ * Computes the diff between a Layer A verdict (machinery) and a Layer B verdict
+ * (independent scoring). Determines agreement/disagreement category and whether
+ * the result should be escalated for human review.
+ *
+ * Disagreements are always escalated.
+ * Agreements are escalated at `sampleRate` probability for calibration sampling.
+ */
+export function computeDiff(
+  a: LayerAVerdict,
+  b: LayerBVerdict,
+  returnMode: ReturnMode,
+  sampleRate: number,
+): VerdictDiff {
+  const aPass = a.verdict === "pass";
+  const bPass = b.verdict === "pass";
+  const aFail = a.verdict === "fail";
+  const bFail = b.verdict === "fail";
+
+  let category: VerdictDiff["category"];
+  let reasoning: string;
+  let isDisagreement: boolean;
+
+  if (aPass && bPass) {
+    // Both layers agree the seed passes
+    category = "agreement_pass";
+    reasoning = "Both Layer A and Layer B agree: pass.";
+    isDisagreement = false;
+  } else if (aFail && bFail) {
+    // Both fail — check if for similar reasons
+    const aGates = new Set(a.gates_triggered);
+    const dims = b.dimensions as Record<string, { score: number; justification: string }>;
+    const bDimensions = Object.entries(dims)
+      .filter(([, dim]) => dim.score < 0.5)
+      .map(([name]) => name);
+
+    // If Layer A triggered gates and Layer B has low-scoring dimensions,
+    // check for overlap in failure signal. If gates triggered and dimensions
+    // failed, they may be pointing at the same issue (agreement) or different
+    // issues (disagreement).
+    const hasOverlap = bDimensions.length > 0 && aGates.size > 0;
+
+    if (hasOverlap) {
+      category = "agreement_fail";
+      reasoning =
+        `Both layers fail. Layer A triggered gates: [${[...aGates].join(", ")}]. ` +
+        `Layer B low dimensions: [${bDimensions.join(", ")}]. Failure signals overlap.`;
+      isDisagreement = false;
+    } else {
+      category = "both_fail_different";
+      reasoning =
+        `Both layers fail but for different reasons. ` +
+        `Layer A gates: [${[...aGates].join(", ")}], justification: ${a.justification}. ` +
+        `Layer B low dimensions: [${bDimensions.join(", ")}], justification: ${b.justification}.`;
+      isDisagreement = true;
+    }
+  } else if (aPass && bFail) {
+    // Layer A too lenient — machinery missed something
+    category = "a_pass_b_fail";
+    reasoning =
+      `Layer A passed but Layer B failed (composite: ${b.composite_score.toFixed(3)}). ` +
+      `Machinery may be too lenient. Layer B justification: ${b.justification}`;
+    isDisagreement = true;
+  } else if (aFail && bPass) {
+    // Layer A too strict
+    category = "a_fail_b_pass";
+    reasoning =
+      `Layer A failed but Layer B passed (composite: ${b.composite_score.toFixed(3)}). ` +
+      `Machinery may be too strict. Layer A gates: [${a.gates_triggered.join(", ")}]. ` +
+      `Layer A justification: ${a.justification}`;
+    isDisagreement = true;
+  } else {
+    // Inconclusive verdicts — treat as agreement for the dominant signal
+    if (aPass || bPass) {
+      category = "agreement_pass";
+      reasoning = "One or both layers inconclusive, leaning pass.";
+    } else {
+      category = "agreement_fail";
+      reasoning = "One or both layers inconclusive, leaning fail.";
+    }
+    isDisagreement = false;
+  }
+
+  // Determine escalation
+  let escalated: boolean;
+  let escalationReason: string;
+
+  if (isDisagreement) {
+    escalated = true;
+    escalationReason = `Disagreement: ${category}`;
+  } else if (Math.random() < sampleRate) {
+    escalated = true;
+    escalationReason = `Sampled agreement for calibration (rate: ${sampleRate})`;
+  } else {
+    escalated = false;
+    escalationReason = "";
+  }
+
+  return {
+    seed_id: a.seed_id,
+    category,
+    layer_a_verdict: a.verdict,
+    layer_b_verdict: b.verdict,
+    return_mode: returnMode,
+    reasoning,
+    is_disagreement: isDisagreement,
+    escalated,
+    escalation_reason: escalationReason,
+  };
+}

package/calibration/engine/escalation.ts

@@ -0,0 +1,49 @@
+import { randomBytes } from "crypto";
+
+import type { VerdictDiff, DiagnosisCard, EscalationItem } from "../types";
+
+// ---------------------------------------------------------------------------
+// Human review queue — builds escalation items from diffs and diagnoses
+// ---------------------------------------------------------------------------
+
+/**
+ * Builds the escalation queue from verdict diffs and diagnosis cards.
+ *
+ * All disagreements and sampled agreements that were marked as escalated
+ * become escalation items. Diagnosis cards are linked to their corresponding
+ * diffs by seed_id.
+ */
+export function buildEscalationQueue(
+  diffs: VerdictDiff[],
+  diagnoses: DiagnosisCard[],
+): EscalationItem[] {
+  // Index diagnoses by seed_id for fast lookup
+  const diagBySeed = new Map<string, DiagnosisCard>();
+  for (const diag of diagnoses) {
+    diagBySeed.set(diag.seed_id, diag);
+  }
+
+  const items: EscalationItem[] = [];
+
+  for (const diff of diffs) {
+    // Only escalated diffs become escalation items
+    if (!diff.escalated) continue;
+
+    const id = `esc-${randomBytes(8).toString("hex")}`;
+    const diagnosis = diagBySeed.get(diff.seed_id);
+
+    const item: EscalationItem = {
+      id,
+      seed_id: diff.seed_id,
+      diff,
+      diagnosis,
+      human_judgment: "pending",
+      human_reasoning: "",
+      reviewed_at: "",
+    };
+
+    items.push(item);
+  }
+
+  return items;
+}