switchboard-cli 0.1.0-alpha.2 → 0.1.0-alpha.4

package/src/autoagent/results.ts

@@ -0,0 +1,68 @@
+import { appendFileSync, readFileSync, existsSync, mkdirSync } from "fs";
+import { dirname } from "path";
+import type { IterationResult } from "./types";
+
+const HEADER = "iteration\tcommit\tscore\tprev_score\ttests\tstatus\tdescription";
+
+export class ResultsLedger {
+  private path: string;
+
+  constructor(path: string) {
+    this.path = path;
+    const dir = dirname(path);
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  }
+
+  append(result: IterationResult): void {
+    if (!existsSync(this.path)) {
+      appendFileSync(this.path, HEADER + "\n", "utf-8");
+    }
+
+    const line = [
+      result.iteration,
+      result.commit,
+      result.score.toFixed(4),
+      result.prevScore.toFixed(4),
+      result.testSummary,
+      result.status,
+      result.description,
+    ].join("\t");
+
+    appendFileSync(this.path, line + "\n", "utf-8");
+  }
+
+  readAll(): IterationResult[] {
+    if (!existsSync(this.path)) return [];
+
+    const lines = readFileSync(this.path, "utf-8").trim().split("\n");
+    if (lines.length <= 1) return [];
+
+    return lines.slice(1).filter(Boolean).map(line => {
+      const parts = line.split("\t");
+      return {
+        iteration: parseInt(parts[0]!, 10),
+        commit: parts[1]!,
+        score: parseFloat(parts[2]!),
+        prevScore: parseFloat(parts[3]!),
+        testsPass: true,
+        testSummary: parts[4]!,
+        status: parts[5]! as IterationResult["status"],
+        description: parts[6]!,
+        filesChanged: [],
+        timestamp: 0,
+        durationMs: 0,
+      };
+    });
+  }
+
+  getBest(): IterationResult | null {
+    const kept = this.readAll().filter(r => r.status === "keep");
+    if (kept.length === 0) return null;
+    return kept.reduce((best, r) => r.score > best.score ? r : best);
+  }
+
+  format(): string {
+    if (!existsSync(this.path)) return "No results yet.";
+    return readFileSync(this.path, "utf-8");
+  }
+}

package/src/autoagent/runner.spec.ts

@@ -0,0 +1,20 @@
+import { describe, it, expect } from "vitest";
+import { runTests, runScore, getChangedFiles, getShortCommit } from "./runner";
+
+describe("runner", () => {
+  it("runTests returns pass/fail and summary", async () => {
+    const result = await runTests("echo '3 tests passed'", process.cwd());
+    expect(result.passed).toBe(true);
+    expect(result.output).toContain("3 tests passed");
+  });
+
+  it("runTests detects failure from non-zero exit", async () => {
+    const result = await runTests("exit 1", process.cwd());
+    expect(result.passed).toBe(false);
+  });
+
+  it("getShortCommit returns 7-char hash", async () => {
+    const hash = await getShortCommit(process.cwd());
+    expect(hash.length).toBe(7);
+  });
+});

package/src/autoagent/runner.ts

@@ -0,0 +1,92 @@
+import { execSync } from "child_process";
+import type { ScoreResult } from "./types";
+
+// ---------------------------------------------------------------------------
+// Test runner — executes project test command
+// ---------------------------------------------------------------------------
+
+export async function runTests(
+  command: string,
+  cwd: string,
+): Promise<{ passed: boolean; output: string }> {
+  try {
+    const output = execSync(command, {
+      cwd,
+      encoding: "utf-8",
+      timeout: 300_000, // 5 min max
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+    return { passed: true, output: output.slice(-500) };
+  } catch (err: any) {
+    const output = (err.stdout ?? "") + (err.stderr ?? "");
+    return { passed: false, output: output.slice(-500) };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Score runner — executes project score function
+// ---------------------------------------------------------------------------
+
+export async function runScore(
+  command: string,
+  cwd: string,
+): Promise<ScoreResult> {
+  try {
+    const output = execSync(command, {
+      cwd,
+      encoding: "utf-8",
+      timeout: 600_000, // 10 min max for scoring (may run engine)
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+
+    // Score function should output JSON to stdout
+    const jsonMatch = output.match(/\{[\s\S]*"score"[\s\S]*\}/);
+    if (!jsonMatch) {
+      return { score: 0, breakdown: {}, summary: "Score function produced no JSON output" };
+    }
+    return JSON.parse(jsonMatch[0]) as ScoreResult;
+  } catch (err: any) {
+    return { score: 0, breakdown: {}, summary: `Score error: ${(err.message ?? "").slice(0, 200)}` };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Git helpers
+// ---------------------------------------------------------------------------
+
+export async function getChangedFiles(cwd: string): Promise<string[]> {
+  try {
+    const output = execSync("git diff --name-only HEAD~1 HEAD", {
+      cwd,
+      encoding: "utf-8",
+    });
+    return output.trim().split("\n").filter(Boolean);
+  } catch {
+    // If no previous commit or error, check working tree
+    try {
+      const output = execSync("git diff --name-only", { cwd, encoding: "utf-8" });
+      return output.trim().split("\n").filter(Boolean);
+    } catch {
+      return [];
+    }
+  }
+}
+
+export async function getShortCommit(cwd: string): Promise<string> {
+  try {
+    return execSync("git rev-parse --short HEAD", { cwd, encoding: "utf-8" }).trim();
+  } catch {
+    return "0000000";
+  }
+}
+
+export async function gitRevert(cwd: string): Promise<void> {
+  execSync("git revert --no-commit HEAD", { cwd, encoding: "utf-8" });
+  execSync('git commit -m "autoagent: discard — reverted last iteration"', { cwd, encoding: "utf-8" });
+}
+
+export async function gitCommit(cwd: string, message: string): Promise<string> {
+  execSync("git add -A", { cwd, encoding: "utf-8" });
+  execSync(`git commit -m "${message.replace(/"/g, '\\"')}" --allow-empty`, { cwd, encoding: "utf-8" });
+  return getShortCommit(cwd);
+}

package/src/autoagent/types.ts

@@ -0,0 +1,64 @@
+// ---------------------------------------------------------------------------
+// AutoAgent types
+// ---------------------------------------------------------------------------
+
+export type AutoAgentTier = "code" | "harness" | "full";
+
+export type IterationStatus = "keep" | "discard" | "crash" | "boundary-violation";
+
+export interface AutoAgentConfig {
+  tier: AutoAgentTier;
+  maxIterations: number;      // 0 = unlimited
+  maxTimeMs: number;          // 0 = unlimited
+  scoreTarget: number;        // 0 = no target
+  testCommand: string;        // default: "pnpm test"
+  scoreCommand: string;       // default: "npx tsx .switchboard/autoagent/score.ts"
+  receiptEveryN: number;      // default: 10
+}
+
+export interface ScoreResult {
+  score: number;
+  breakdown: Record<string, number>;
+  summary: string;
+}
+
+export interface IterationResult {
+  iteration: number;
+  commit: string;
+  score: number;
+  prevScore: number;
+  testsPass: boolean;
+  testSummary: string;
+  status: IterationStatus;
+  description: string;
+  filesChanged: string[];
+  timestamp: number;
+  durationMs: number;
+}
+
+export interface AutoAgentState {
+  startedAt: number;
+  iteration: number;
+  bestScore: number;
+  bestCommit: string;
+  currentScore: number;
+  results: IterationResult[];
+  status: "running" | "stopped" | "target-reached" | "max-iterations" | "max-time" | "tests-failing";
+}
+
+export interface BoundarySpec {
+  tier: AutoAgentTier;
+  modifiable: string[];       // glob patterns of files that can be modified
+  frozen: string[];           // files that can NEVER be modified
+  contractNonGoals: string[];
+}
+
+export const DEFAULT_CONFIG: AutoAgentConfig = {
+  tier: "code",
+  maxIterations: 0,
+  maxTimeMs: 0,
+  scoreTarget: 0,
+  testCommand: "pnpm test",
+  scoreCommand: "npx tsx .switchboard/autoagent/score.ts",
+  receiptEveryN: 10,
+};

package/src/commands/autoagent.ts

@@ -0,0 +1,108 @@
+import type { Command } from "commander";
+import { join } from "path";
+import { existsSync } from "fs";
+import { requireRepoRoot, sbDir } from "../lib/paths";
+import { loadContract } from "../store/filesystem-store";
+import { AutoAgentLoop } from "../autoagent/loop";
+import { ResultsLedger } from "../autoagent/results";
+import type { AutoAgentTier } from "../autoagent/types";
+import * as out from "../lib/output";
+import { withCliErrors } from "../lib/errors";
+
+export function registerAutoAgentCommand(program: Command): void {
+  program
+    .command("autoagent")
+    .description("Run governed autonomous improvement loop")
+    .option("--tier <tier>", "Autonomy tier: code, harness, full", "code")
+    .option("--max-iterations <n>", "Stop after N iterations", "0")
+    .option("--max-time <duration>", "Stop after duration (e.g., 4h, 30m)", "0")
+    .option("--score-target <n>", "Stop when score reaches threshold", "0")
+    .option("--results", "Show results.tsv")
+    .option("--rollback", "Rollback to best-scoring iteration")
+    .action(withCliErrors(async (opts) => {
+      const repoRoot = requireRepoRoot();
+
+      out.heading("switchboard autoagent");
+
+      // Show results mode
+      if (opts.results) {
+        const ledger = new ResultsLedger(join(sbDir(repoRoot), "autoagent", "results.tsv"));
+        console.log(ledger.format());
+        return;
+      }
+
+      // Validate required files
+      const autoagentDir = join(sbDir(repoRoot), "autoagent");
+      if (!existsSync(join(autoagentDir, "program.md"))) {
+        out.fail("Missing .switchboard/autoagent/program.md");
+        out.info("Write a directive describing what the autoagent should improve.");
+        process.exitCode = 1;
+        return;
+      }
+      if (!existsSync(join(autoagentDir, "score.ts"))) {
+        out.fail("Missing .switchboard/autoagent/score.ts");
+        out.info("Write a score function: export async function score(): Promise<ScoreResult>");
+        process.exitCode = 1;
+        return;
+      }
+
+      // Load contract for boundaries
+      const contract = loadContract(repoRoot);
+
+      // Parse config
+      const tier = opts.tier as AutoAgentTier;
+      if (!["code", "harness", "full"].includes(tier)) {
+        out.fail(`Invalid tier: ${tier}. Use: code, harness, full`);
+        process.exitCode = 1;
+        return;
+      }
+
+      const maxTimeMs = parseTime(opts.maxTime);
+
+      out.section("Tier", tier);
+      out.section("Max iterations", opts.maxIterations === "0" ? "unlimited" : opts.maxIterations);
+      out.section("Max time", maxTimeMs === 0 ? "unlimited" : opts.maxTime);
+      out.section("Score target", opts.scoreTarget === "0" ? "none" : opts.scoreTarget);
+
+      // Create and run the loop
+      const loop = new AutoAgentLoop(repoRoot, {
+        tier,
+        maxIterations: parseInt(opts.maxIterations, 10),
+        maxTimeMs,
+        scoreTarget: parseFloat(opts.scoreTarget),
+      }, contract);
+
+      // Handle SIGINT
+      process.on("SIGINT", () => {
+        out.warn("Stopping autoagent loop...");
+        loop.stop();
+      });
+
+      const finalState = await loop.run();
+
+      // Print summary
+      out.heading("AutoAgent Complete");
+      out.section("Iterations", `${finalState.iteration}`);
+      out.section("Best score", `${finalState.bestScore.toFixed(4)} (commit: ${finalState.bestCommit})`);
+      out.section("Final score", `${finalState.currentScore.toFixed(4)}`);
+      out.section("Status", finalState.status);
+
+      const kept = finalState.results.filter(r => r.status === "keep").length;
+      const discarded = finalState.results.filter(r => r.status === "discard").length;
+      out.section("Kept/Discarded", `${kept}/${discarded}`);
+    }));
+}
+
+function parseTime(input: string): number {
+  if (input === "0") return 0;
+  const match = input.match(/^(\d+)(h|m|s)$/);
+  if (!match) return 0;
+  const [, num, unit] = match;
+  const n = parseInt(num!, 10);
+  switch (unit) {
+    case "h": return n * 3600_000;
+    case "m": return n * 60_000;
+    case "s": return n * 1000;
+    default: return 0;
+  }
+}

package/src/commands/receipt.ts

@@ -15,7 +15,7 @@
  * Does not require the hosted web app or Supabase.
  */
 
-import { writeFileSync } from "fs";
+import { writeFileSync, existsSync, readdirSync, readFileSync } from "fs";
 import { join } from "path";
 import type { Command } from "commander";
 import {
@@ -33,7 +33,7 @@ import {
   type ProofManifest,
   extractHarnessProvenance,
 } from "@switchboard/core";
-import { requireRepoRoot, sbDir, returnsDir, receiptsDir } from "../lib/paths";
+import { requireRepoRoot, sbDir, returnsDir, receiptsDir, packetsDir } from "../lib/paths";
 import { readYaml } from "../lib/yaml-io";
 import { loadPortableProofManifest } from "../lib/proof";
 import {
@@ -86,6 +86,14 @@ export function registerReceiptCommand(program: Command): void {
 
       out.success(`Loaded ingest pointer for dispatch: ${pointer.dispatch_id}`);
 
+      // 2b. Check if Codex audit has been run for this dispatch
+      const codexAuditRan = hasCodexAuditForDispatch(repoRoot, pointer.dispatch_id);
+      if (!codexAuditRan) {
+        out.warn("No Codex audit found for this dispatch.");
+        out.warn("Run `sb audit codex` before issuing a receipt for governed projects.");
+        out.info("Proceeding without audit — receipt will reflect missing independent verification.");
+      }
+
       // 3. Load the exact persisted artifacts by ID — do NOT re-derive
       const retDir = returnsDir(repoRoot);
 
@@ -303,3 +311,26 @@ function generateReceiptSummary(
 
   return lines.join("\n");
 }
+
+// ---------------------------------------------------------------------------
+// Codex audit check — verifies independent audit was run before receipt
+// ---------------------------------------------------------------------------
+
+function hasCodexAuditForDispatch(repoRoot: string, dispatchId: string): boolean {
+  const pDir = packetsDir(repoRoot);
+  if (!existsSync(pDir)) return false;
+
+  try {
+    const files = readdirSync(pDir).filter(f => f.startsWith("codex-audit-response"));
+    for (const file of files) {
+      const content = readFileSync(join(pDir, file), "utf-8");
+      if (content.includes(dispatchId)) {
+        return true;
+      }
+    }
+  } catch {
+    // If we can't read the directory, assume no audit
+  }
+
+  return false;
+}

package/src/index.ts

@@ -17,6 +17,7 @@ import { registerReceiptCommand } from "./commands/receipt";
 import { registerEvaluateCommand } from "./commands/evaluate";
 import { registerAuditCodexCommand } from "./commands/audit-codex";
 import { registerCalibrateCommand } from "./commands/calibrate";
+import { registerAutoAgentCommand } from "./commands/autoagent";
 
 const program = new Command();
 
@@ -40,4 +41,7 @@ registerAuditCodexCommand(program);
 // Wave 3: calibration
 registerCalibrateCommand(program);
 
+// Wave 4: autoagent
+registerAutoAgentCommand(program);
+
 program.parse();

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/diffs/adv-a0c9e2bfb0d6.yaml

@@ -1,9 +0,0 @@
-seed_id: adv-a0c9e2bfb0d6
-category: agreement_pass
-layer_a_verdict: pass
-layer_b_verdict: inconclusive
-return_mode: honest
-reasoning: One or both layers inconclusive, leaning pass.
-is_disagreement: false
-escalated: false
-escalation_reason: ""

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/diffs/blind-3e892f3a89ee.yaml

@@ -1,9 +0,0 @@
-seed_id: blind-3e892f3a89ee
-category: agreement_fail
-layer_a_verdict: fail
-layer_b_verdict: inconclusive
-return_mode: drift
-reasoning: One or both layers inconclusive, leaning fail.
-is_disagreement: false
-escalated: false
-escalation_reason: ""

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/diffs/blind-958b2f9e6816.yaml

@@ -1,9 +0,0 @@
-seed_id: blind-958b2f9e6816
-category: agreement_fail
-layer_a_verdict: fail
-layer_b_verdict: inconclusive
-return_mode: drift
-reasoning: One or both layers inconclusive, leaning fail.
-is_disagreement: false
-escalated: false
-escalation_reason: ""

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/manifest.yaml

@@ -1,8 +0,0 @@
-run_id: 2026-04-09T10-02-57-143Z
-started_at: 2026-04-09T10:02:57.143Z
-seed_count: 3
-blind_count: 2
-adversarial_count: 1
-replay_count: 0
-trigger: manual
-baseline_snapshot: baseline-2026-04-09

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/seeds/adv-a0c9e2bfb0d6.yaml

@@ -1,7 +0,0 @@
-seed_id: adv-a0c9e2bfb0d6
-source: adversarial
-raw_idea: A notes app — just basic note taking, nothing fancy
-complexity_target: simple
-created_at: 2026-04-09T10:02:57.143Z
-replay_source: ""
-attack_vector: scope-inflation-trap

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/seeds/blind-3e892f3a89ee.yaml

@@ -1,8 +0,0 @@
-seed_id: blind-3e892f3a89ee
-source: blind
-raw_idea: A REST API that converts Markdown files to PDF, accepting a file
-  upload and returning the rendered document
-complexity_target: structured
-created_at: 2026-04-09T10:02:57.143Z
-replay_source: ""
-attack_vector: ""

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/seeds/blind-958b2f9e6816.yaml

@@ -1,8 +0,0 @@
-seed_id: blind-958b2f9e6816
-source: blind
-raw_idea: A CLI tool that counts lines of code in a directory, outputting a
-  summary table by file extension
-complexity_target: simple
-created_at: 2026-04-09T10:02:57.143Z
-replay_source: ""
-attack_vector: ""

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/summary.yaml

@@ -1,10 +0,0 @@
-run_id: 2026-04-09T10-02-57-143Z
-seed_count: 3
-gate_catch_rate: 1
-false_positive_rate: 0
-false_negative_rate: 0
-layer_agreement_rate: 1
-diagnosis_count: 0
-escalation_count: 0
-biggest_miss: ""
-completed_at: 2026-04-09T10:02:57.171Z

package/calibration/ledger/runs/2026-04-09T10-02-57-143Z/traces/adv-a0c9e2bfb0d6.yaml

@@ -1,141 +0,0 @@
-seed_id: adv-a0c9e2bfb0d6
-outcome: completed
-rejection_point: ""
-rejection_reason: ""
-complexity_score: 2
-selected_mode: minimal
-assumptions_count: 2
-routed_surface: claude-code-sdk
-routing_rationale: "Selected Claude Code (SDK) (score: 65). Fallback: Codex
-  (execution) (score: 25). Disqualified: openai-api (Needs execute, provider
-  offers compile.), chatgpt-auto (Needs execute, provider offers compile.),
-  internal-fallback (Needs execute, provider offers compile.)."
-return_mode: honest
-return_mode_sealed: true
-reconcile_findings_count: 3
-reconcile_drift_detected: true
-reconcile_recommendation: accept-with-warnings
-gates_triggered: []
-gate_label: accepted-with-cautions
-spread_score: 0.11249999999999999
-spread_vector:
-  claim_mismatch_ratio: 0.25
-  audit_burden: 0
-  residual_weight: 0
-  provenance_weakness: 0
-  gate_distance: 0.25
-  trace_failure_signal: 0
-  budget_correction_weight: 0
-trust_posture: cautioned
-receipt_verdict: accepted-with-cautions
-honesty_violations: []
-duration_ms: 3
-compiled_spec: >
-  # A notes app — just basic note taking, nothing fancy
-
-
-  ## What this product is
-
-
-  Build a notes app — just basic note taking, nothing fancy as a bounded V1
-  slice.
-
-
-  ## Who it is for
-
-
-  Solo builders shipping software.
-
-
-  ## The problem
-
-
-  A notes app — just basic note taking, nothing fancy
-
-
-  ## The proposed solution
-
-
-  Build a notes app — just basic note taking, nothing fancy as a bounded V1
-  slice. This is a bounded V1 slice focused on proving the core value
-  proposition with minimal surface area.
-
-
-  ## V1 promise
-
-
-  A working first slice of a notes app — just basic note taking, nothing fancy.
-
-
-  ## Scope in
-
-
-  - A notes app — just basic note taking, nothing fancy
-
-  - Keep the primary artifact aligned with the V1 promise.
-
-  - Touch only files necessary for the first build slice.
-
-
-  ## Scope out
-
-
-  - Not a generic chat wrapper
-
-
-  ## First build slice
-
-
-  A notes app — just basic note taking, nothing fancy
-
-
-  Systems expected: CLI, Node.js.
-
-
-  Constraints to respect:
-
-  - No additional constraints.
-
-
-  ## Acceptance criteria
-
-
-  - Seed is valid and the idea has been compiled into a governed contract.
-
-  - Primary artifact is execution-ready: the build slice works as described.
-
-  - Next surface is explicit with a return condition.
-
-  - Handoff is launchable as an SBX bundle.
-
-
-  ## Open questions
-
-
-  - None identified.
-
-
-  ## What must not change (intent anchors)
-
-
-  - Not a generic chat wrapper
-
-
-  These are non-negotiable boundaries. Any execution that touches these areas
-  must be flagged for review.
-
-
-  ## Key risks and unknowns
-
-
-  Confidence is low — significant unknowns remain.
-
-
-  - Is the scope narrow enough to deliver in a single bounded execution?
-
-  - Are there hidden dependencies that could expand the build slice?
-
-  - Return condition: all done_when criteria met or blocker encountered.
-receipt_summary: 'Completed: Build the first slice: A notes app — just basic
-  note taking, nothing fancy. All done_when criteria satisfied. Implementation
-  follows the project contract for "A notes app — just basic note '