switchboard-cli 0.1.0-alpha.1

package/calibration/internal/harness.ts

@@ -0,0 +1,697 @@
+import { randomBytes } from "crypto";
+import type {
+  ActivationInput,
+  AuditedReceipt,
+  CanonicalDelta,
+  DriftDiff,
+  GateState,
+  HandoffRecord,
+  NormalizedReturnEnvelope,
+  ProjectContract,
+  ProjectLedger,
+  ReconcileReport,
+  ReconcileV2Result,
+  ReturnReport,
+  RoutingConstraints,
+  SynthesizedSeed,
+} from "@switchboard/core";
+import {
+  projectContractSchema,
+  handoffRecordSchema,
+} from "@switchboard/core";
+import { activateV2 } from "@switchboard/core";
+import { routeV2 } from "@switchboard/core";
+import { reconcileReturn } from "@switchboard/core";
+import { evaluateGates } from "@switchboard/core";
+import { buildAuditedReceipt } from "@switchboard/core";
+import { computeClaimSpread } from "@switchboard/core";
+
+import type { CalibrationSeed, PipelineTrace } from "../types";
+import { simulateReturn } from "./return-simulator";
+import { collectTrace } from "./trace-collector";
+
+// ---------------------------------------------------------------------------
+// Deterministic seed parser — extracts structured seed from raw_idea
+// ---------------------------------------------------------------------------
+
+function parseSynthesizedSeed(rawIdea: string): SynthesizedSeed {
+  // Title: first sentence or first 80 chars
+  const firstSentence = rawIdea.match(/^[^.!?]+[.!?]?/)?.[0] ?? rawIdea.slice(0, 80);
+  const title = firstSentence.replace(/[.!?]+$/, "").trim();
+
+  // One-liner: short distillation
+  const oneLiner = title.length > 50
+    ? title.slice(0, 50) + "..."
+    : title;
+
+  // V1 outcome: second sentence if available, otherwise derived
+  const sentences = rawIdea.split(/[.!?]+/).map(s => s.trim()).filter(Boolean);
+  const v1Outcome = sentences.length > 1
+    ? sentences[1]!
+    : `A working first slice of ${title.toLowerCase()}.`;
+
+  // Detect task type
+  const isImpl = /\b(build|implement|create|add|set up|scaffold|code|develop)\b/i.test(rawIdea);
+  const taskType = isImpl ? "implementation-request" as const : "product-idea" as const;
+
+  // Confidence: more detail = higher confidence
+  const confidence = rawIdea.length > 200 ? "high" as const
+    : rawIdea.length > 80 ? "medium" as const
+    : "low" as const;
+
+  return {
+    title,
+    one_liner: oneLiner,
+    problem: rawIdea,
+    proposed_solution: `Build ${title.toLowerCase()} as a bounded V1 slice.`,
+    target_user: "Solo builders shipping software.",
+    v1_outcome: v1Outcome,
+    non_goals: ["Not a generic chat wrapper"],
+    missing_inputs: [],
+    open_questions: [],
+    first_build_slice: title,
+    confidence,
+    task_type: taskType,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Contract builder
+// ---------------------------------------------------------------------------
+
+function buildContract(seed: SynthesizedSeed): ProjectContract {
+  return projectContractSchema.parse({
+    project_id: `proj-${randomBytes(6).toString("hex")}`,
+    title: seed.title,
+    one_liner: seed.one_liner,
+    problem: seed.problem,
+    target_user: seed.target_user,
+    desired_v1_outcome: seed.v1_outcome,
+    non_goals: seed.non_goals,
+    repo_status: "repo-not-present",
+    task_clauses: {
+      scope_in: [seed.first_build_slice || seed.title],
+      scope_out: seed.non_goals,
+      systems_expected: [],
+      files_allowed: [],
+      must_not_widen: seed.non_goals,
+      build_constraints: [],
+    },
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Calibrated spec builder — 12-section SB_SPEC from seed + contract
+// ---------------------------------------------------------------------------
+
+function buildCalibratedSpec(seed: SynthesizedSeed, contract: ProjectContract): string {
+  const scopeIn = contract.task_clauses.scope_in.map(s => `- ${s}`).join("\n");
+  const scopeOut = contract.non_goals.map(s => `- ${s}`).join("\n");
+  const openQs = seed.open_questions.length > 0
+    ? seed.open_questions.map(q => `- ${q}`).join("\n")
+    : seed.missing_inputs.length > 0
+      ? seed.missing_inputs.map(m => `- ${m}`).join("\n")
+      : "- None identified.";
+  const constraints = contract.task_clauses.build_constraints.length > 0
+    ? contract.task_clauses.build_constraints.map(c => `- ${c}`).join("\n")
+    : "- No additional constraints.";
+  const systems = contract.task_clauses.systems_expected.length > 0
+    ? contract.task_clauses.systems_expected.join(", ")
+    : "CLI, Node.js";
+  const riskLine = seed.confidence === "high"
+    ? "Confidence is high — minimal unknowns."
+    : seed.confidence === "medium"
+      ? "Confidence is medium — some assumptions remain unconfirmed."
+      : "Confidence is low — significant unknowns remain.";
+
+  return `# ${contract.title}
+
+## What this product is
+
+${seed.proposed_solution}
+
+## Who it is for
+
+${seed.target_user}
+
+## The problem
+
+${seed.problem}
+
+## The proposed solution
+
+${seed.proposed_solution} This is a bounded V1 slice focused on proving the core value proposition with minimal surface area.
+
+## V1 promise
+
+${seed.v1_outcome}
+
+## Scope in
+
+${scopeIn}
+- Keep the primary artifact aligned with the V1 promise.
+- Touch only files necessary for the first build slice.
+
+## Scope out
+
+${scopeOut}
+
+## First build slice
+
+${seed.first_build_slice || contract.title}
+
+Systems expected: ${systems}.
+
+Constraints to respect:
+${constraints}
+
+## Acceptance criteria
+
+- Seed is valid and the idea has been compiled into a governed contract.
+- Primary artifact is execution-ready: the build slice works as described.
+- Next surface is explicit with a return condition.
+- Handoff is launchable as an SBX bundle.
+
+## Open questions
+
+${openQs}
+
+## What must not change (intent anchors)
+
+${scopeOut}
+
+These are non-negotiable boundaries. Any execution that touches these areas must be flagged for review.
+
+## Key risks and unknowns
+
+${riskLine}
+
+- Is the scope narrow enough to deliver in a single bounded execution?
+- Are there hidden dependencies that could expand the build slice?
+- Return condition: all done_when criteria met or blocker encountered.
+`;
+}
+
+// ---------------------------------------------------------------------------
+// Routing constraints builder
+// ---------------------------------------------------------------------------
+
+function buildRoutingConstraints(
+  seed: SynthesizedSeed,
+  mode: string,
+): RoutingConstraints {
+  return {
+    capability_needed: "execute",
+    process_mode: (mode === "minimal" || mode === "structured" || mode === "governance")
+      ? mode as "minimal" | "structured" | "governance"
+      : "structured",
+    task_type: seed.task_type,
+    seed_confidence: seed.confidence,
+    repo_exists: false,
+    blast_radius: "low",
+    non_functional_flags: [],
+    provider_readiness: {
+      "internal-fallback": true,
+      "claude-code-sdk": true,
+      "openai-api": false,
+      "codex": false,
+      "chatgpt-auto": false,
+      "chatgpt-manual": false,
+      "cursor-manual": false,
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Handoff builder
+// ---------------------------------------------------------------------------
+
+function buildHandoff(
+  contract: ProjectContract,
+  route: RoutingDecision,
+): HandoffRecord {
+  // Map provider to surface label
+  const providerId = route.selected.provider_id;
+  const surfaceMap: Record<string, "chatgpt" | "claude-code" | "cursor" | "codex"> = {
+    "claude-code-sdk": "claude-code",
+    "openai-api": "chatgpt",
+    "chatgpt-auto": "chatgpt",
+    "chatgpt-manual": "chatgpt",
+    "cursor-manual": "cursor",
+    "codex": "codex",
+    "internal-fallback": "claude-code",
+  };
+  const surface = surfaceMap[providerId] ?? "claude-code";
+
+  return handoffRecordSchema.parse({
+    handoff_id: `ho-${randomBytes(6).toString("hex")}`,
+    surface,
+    objective: `Build the first slice: ${contract.title}`,
+    included_scope: contract.task_clauses.scope_in.length > 0
+      ? contract.task_clauses.scope_in
+      : [contract.desired_v1_outcome],
+    excluded_scope: contract.non_goals,
+    done_when: [`${contract.desired_v1_outcome} is functional and tested.`],
+    return_condition: "All done_when criteria met or blocker encountered.",
+    loop: {
+      objective: `Build the first slice: ${contract.title}`,
+      scope_in: contract.task_clauses.scope_in.length > 0
+        ? contract.task_clauses.scope_in
+        : [contract.desired_v1_outcome],
+      scope_out: contract.non_goals,
+      done_when: [`${contract.desired_v1_outcome} is functional and tested.`],
+      success_criteria: [`${contract.title} works as described.`],
+      verify_with: ["Manual testing", "Type check passes"],
+      return_when: "All done_when criteria met or blocker encountered.",
+      blocker_escalation: ["Escalate if scope grows beyond V1."],
+      expected_artifacts: ["src/index.ts", "src/lib/core.ts"],
+      fallback_surface: "claude-code",
+      execution_shape: "single",
+    },
+    readiness: {
+      status: "ready",
+      issues: [],
+    },
+    bundle_version: "SBX-v1",
+    launched_at: new Date().toISOString(),
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Minimal DriftDiff builder
+// ---------------------------------------------------------------------------
+
+function buildMinimalDriftDiff(): DriftDiff {
+  return {
+    entries: [],
+    has_drift: false,
+    summary: "No drift detected in calibration run.",
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Minimal ProjectLedger builder
+// ---------------------------------------------------------------------------
+
+function buildMinimalLedger(contract: ProjectContract): ProjectLedger {
+  const now = new Date().toISOString();
+  return {
+    entries: [
+      {
+        id: `goal-${randomBytes(4).toString("hex")}`,
+        kind: "goal",
+        text: contract.desired_v1_outcome,
+        evidence_type: "user-backed",
+        evidence_refs: [],
+        parent_ids: [],
+        risk_level: "low",
+        downstream_artifacts: [],
+        created_at: now,
+      },
+      {
+        id: `constraint-${randomBytes(4).toString("hex")}`,
+        kind: "constraint",
+        text: contract.non_goals[0] ?? "No non-goals specified.",
+        evidence_type: "user-backed",
+        evidence_refs: [],
+        parent_ids: [],
+        risk_level: "low",
+        downstream_artifacts: [],
+        created_at: now,
+      },
+    ],
+    scope_changes: [],
+    version: 1,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Minimal ReconcileV2Result builder
+// ---------------------------------------------------------------------------
+
+function buildMinimalReconcileV2(recommendation: string): ReconcileV2Result {
+  return {
+    checks: [],
+    traceability: [],
+    passed: recommendation === "accept",
+    blockers: 0,
+    warnings: recommendation === "accept" ? 0 : 1,
+    recommendation: recommendation === "accept"
+      ? "accept"
+      : recommendation === "accept-with-warnings"
+        ? "accept-with-warnings"
+        : "review-required",
+    summary: `Reconcile result: ${recommendation}.`,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// NormalizedReturnEnvelope builder
+// ---------------------------------------------------------------------------
+
+function buildEnvelope(
+  report: ReturnReport,
+  handoff: HandoffRecord,
+): NormalizedReturnEnvelope {
+  const now = new Date().toISOString();
+  return {
+    envelope_id: `env-${randomBytes(6).toString("hex")}`,
+    envelope_version: 1 as const,
+    created_at: now,
+    source_surface: report.surface_used,
+    source_role: "agent",
+    objective_used: report.objective_attempted,
+    outcome_class: report.status === "done" ? "completed"
+      : report.status === "partial" ? "partial"
+      : "blocked",
+    changed_files: report.changed_files.map(f => ({ path: f, system: "" })),
+    touched_artifacts: report.artifacts_created,
+    commands_run: [],
+    evidence_items: report.evidence.map(e => ({ text: e, content_mode: "direct" as const })),
+    verification_runs: report.verification_run.map(v => ({ text: v, content_mode: "direct" as const })),
+    criteria_met: report.success_criteria_met.map(c => ({ text: c, content_mode: "direct" as const })),
+    decisions: report.decisions_made.map(d => ({ text: d, content_mode: "direct" as const })),
+    cautions: report.blockers.map(b => ({ text: b, source: "return" as const })),
+    residuals: report.escalations.map(e => ({ text: e, kind: "escalation" as const })),
+    recommended_next_step: report.recommended_next_step,
+    provenance_posture: "captured",
+    confidence_posture: "medium",
+    summary: report.summary,
+    handoff_id: report.handoff_id,
+    dispatch_id: `dsp-${randomBytes(6).toString("hex")}`,
+    return_id: `ret-${randomBytes(6).toString("hex")}`,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// CanonicalDelta builder
+// ---------------------------------------------------------------------------
+
+function buildMinimalDelta(
+  report: ReturnReport,
+  envelope: NormalizedReturnEnvelope,
+): CanonicalDelta {
+  const now = new Date().toISOString();
+  return {
+    delta_id: `delta-${randomBytes(6).toString("hex")}`,
+    project_id: "",
+    parent_contract_id: "",
+    return_id: envelope.return_id,
+    gate_state: "",
+    objective_state: report.objective_met === "yes" ? "met"
+      : report.objective_met === "partial" ? "partial"
+      : "unmet",
+    created_at: now,
+    implemented: report.changed_files,
+    systems_touched: [],
+    file_changes: report.changed_files.map(f => ({ path: f, system: "" })),
+    artifact_changes: report.artifacts_created,
+    tests_or_checks: [],
+    verification_runs: report.verification_run,
+    evidence_items: report.evidence,
+    boundaries_preserved: [],
+    boundaries_violated: [],
+    cautions: report.blockers,
+    unresolved: report.escalations,
+    decisions: report.decisions_made.map(d => ({ text: d, traced: false })),
+    state_fields_advanced: report.status === "done" ? ["objective_state"] : [],
+    summary: report.summary,
+    evidence_accounting: {
+      scope_check_count: 1,
+      scope_check_pass_count: 1,
+      scope_check_caution_count: 0,
+      scope_check_blocker_count: 0,
+      verification_run_count: report.verification_run.length,
+      verification_runs: report.verification_run,
+      evidence_item_count: report.evidence.length,
+      evidence_items: report.evidence,
+      file_change_count: report.changed_files.length,
+      artifact_change_count: report.artifacts_created.length,
+      criteria_met_count: report.success_criteria_met.length,
+      criteria_met: report.success_criteria_met,
+      caution_count: report.blockers.length,
+      unresolved_count: report.escalations.length,
+      boundary_preserved_count: 0,
+      boundary_violated_count: 0,
+      consequence_type: report.status === "done" ? "code-change" : "no-change",
+      consequence_label: report.status === "done"
+        ? "Code change: first slice delivered."
+        : "No meaningful change recorded.",
+    },
+    verified: [],
+    residual_resolutions: [],
+  };
+}
+
+// ---------------------------------------------------------------------------
+// GateState builder
+// ---------------------------------------------------------------------------
+
+function buildGateState(
+  report: ReturnReport,
+  delta: CanonicalDelta,
+  reconcile?: ReconcileReport,
+): GateState {
+  const evidenceCount = report.evidence.length;
+  const verificationCount = report.verification_run.length;
+  const criteriaCount = report.success_criteria_met.length;
+
+  // --- Reconcile-aware escalation (takes priority) ---
+  if (reconcile) {
+    // Rule 1: Explicit review-required recommendation → force held-for-review
+    if (reconcile.recommendation === "review-required") {
+      const reviewFindings = reconcile.findings.filter(f => f.severity === "review-required");
+      return {
+        label: "held-for-review",
+        reason: `Reconcile review-required: ${reviewFindings.map(f => f.category).join(", ")}.`,
+        evidence_summary: {
+          verification_count: verificationCount,
+          evidence_count: evidenceCount,
+          criteria_met_count: criteriaCount,
+          caution_count: reconcile.findings.length,
+          blocker_count: reviewFindings.length,
+        },
+        honest: true,
+      };
+    }
+
+    // Rule 2: 3+ findings (even all warnings) → accepted-with-cautions minimum
+    if (reconcile.findings.length >= 3) {
+      return {
+        label: "accepted-with-cautions",
+        reason: `Reconcile produced ${reconcile.findings.length} findings: ${reconcile.findings.map(f => f.category).join(", ")}.`,
+        evidence_summary: {
+          verification_count: verificationCount,
+          evidence_count: evidenceCount,
+          criteria_met_count: criteriaCount,
+          caution_count: reconcile.findings.length,
+          blocker_count: 0,
+        },
+        honest: true,
+      };
+    }
+
+    // Rule 3: Drift detected with any review-required finding → held-for-review
+    if (reconcile.drift_detected && reconcile.findings.some(f => f.severity === "review-required")) {
+      return {
+        label: "held-for-review",
+        reason: `Drift detected with review-required finding.`,
+        evidence_summary: {
+          verification_count: verificationCount,
+          evidence_count: evidenceCount,
+          criteria_met_count: criteriaCount,
+          caution_count: reconcile.findings.length,
+          blocker_count: reconcile.findings.filter(f => f.severity === "review-required").length,
+        },
+        honest: true,
+      };
+    }
+  }
+  // --- End reconcile-aware escalation, fall through to evidence checks ---
+
+  // Evidence-gap returns get a weaker gate
+  if (verificationCount === 0 && evidenceCount === 0) {
+    return {
+      label: "held-for-review",
+      reason: "No verification or evidence provided despite done claim.",
+      evidence_summary: {
+        verification_count: 0,
+        evidence_count: 0,
+        criteria_met_count: criteriaCount,
+        caution_count: 0,
+        blocker_count: 0,
+      },
+      honest: true,
+    };
+  }
+
+  // Thin evidence gets cautions
+  if (verificationCount <= 1 && evidenceCount <= 1) {
+    return {
+      label: "accepted-with-cautions",
+      reason: "Minimal evidence provided. Accepted with cautions.",
+      evidence_summary: {
+        verification_count: verificationCount,
+        evidence_count: evidenceCount,
+        criteria_met_count: criteriaCount,
+        caution_count: 1,
+        blocker_count: 0,
+      },
+      honest: true,
+    };
+  }
+
+  // Good evidence
+  return {
+    label: "accepted",
+    reason: "Gate accepted. Evidence supports the claims.",
+    evidence_summary: {
+      verification_count: verificationCount,
+      evidence_count: evidenceCount,
+      criteria_met_count: criteriaCount,
+      caution_count: 0,
+      blocker_count: 0,
+    },
+    honest: true,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Main pipeline runner
+// ---------------------------------------------------------------------------
+
+export function runPipeline(seed: CalibrationSeed): {
+  trace: PipelineTrace;
+  contract: ProjectContract;
+  handoff: HandoffRecord;
+  receipt: AuditedReceipt | null;
+} {
+  const startTime = Date.now();
+
+  let contract: ProjectContract | null = null;
+  let handoff: HandoffRecord | null = null;
+  let receipt: AuditedReceipt | null = null;
+  let partialTrace: Partial<PipelineTrace> = {
+    seed_id: seed.seed_id,
+    return_mode: "honest",
+    return_mode_sealed: true,
+  };
+
+  try {
+    // Step 1: Build ActivationInput
+    const activationInput: ActivationInput = {
+      raw_idea: seed.raw_idea,
+      repo_exists: false,
+      what_do_you_want_next: "Build the first slice",
+      constraints: "",
+    };
+
+    // Step 2: Parse synthesized seed
+    const synthesizedSeed = parseSynthesizedSeed(seed.raw_idea);
+
+    // Step 3: Activate
+    const activationResult = activateV2(activationInput, synthesizedSeed);
+    partialTrace.complexity_score = activationResult.mode_scoring.total;
+    partialTrace.selected_mode = activationResult.selected_mode;
+    partialTrace.assumptions_count = activationResult.assumptions.length;
+
+    // Step 4: Build contract + calibrated spec
+    contract = buildContract(synthesizedSeed);
+    const compiledSpec = buildCalibratedSpec(synthesizedSeed, contract);
+    partialTrace.compiled_spec = compiledSpec;
+
+    // Step 5: Route
+    const routingConstraints = buildRoutingConstraints(
+      synthesizedSeed,
+      activationResult.selected_mode,
+    );
+    const routeDecision = routeV2(routingConstraints);
+    partialTrace.routed_surface = routeDecision.selected.provider_id;
+    partialTrace.routing_rationale = routeDecision.rationale;
+
+    // Step 6: Build handoff
+    handoff = buildHandoff(contract, routeDecision);
+
+    // Step 7: Decide return mode
+    const returnMode: "honest" | "drift" = Math.random() < 0.5 ? "honest" : "drift";
+    partialTrace.return_mode = returnMode;
+
+    // Step 8: Simulate return
+    const report = simulateReturn(returnMode, contract, handoff);
+
+    // Step 9: Reconcile (with full spec for untracked-decision context matching)
+    const reconcileResult = reconcileReturn({
+      report,
+      contract,
+      handoff,
+      spec_markdown: compiledSpec,
+    });
+    partialTrace.reconcile_findings_count = reconcileResult.findings.length;
+    partialTrace.reconcile_drift_detected = reconcileResult.drift_detected;
+    partialTrace.reconcile_recommendation = reconcileResult.recommendation;
+
+    // Step 10: Gate evaluation
+    const diff = buildMinimalDriftDiff();
+    const ledger = buildMinimalLedger(contract);
+    const reconcileV2 = buildMinimalReconcileV2(reconcileResult.recommendation);
+    const approvalGates = evaluateGates({
+      diff,
+      ledger,
+      reconcile: reconcileV2,
+      report,
+    });
+    partialTrace.gates_triggered = approvalGates.map(g => g.reason);
+    partialTrace.gate_label = approvalGates.length > 0
+      ? approvalGates.map(g => g.reason).join(", ")
+      : "no-gates";
+
+    // Step 11: Build audited receipt
+    const envelope = buildEnvelope(report, handoff);
+    const delta = buildMinimalDelta(report, envelope);
+    const gate = buildGateState(report, delta, reconcileResult);
+
+    receipt = buildAuditedReceipt({
+      envelope,
+      delta,
+      gate,
+      nextSlice: null,
+    });
+
+    partialTrace.trust_posture = receipt.trust_posture;
+    partialTrace.gate_label = gate.label; // Use the reconcile-aware gate label
+    partialTrace.receipt_verdict = receipt.gate_label;
+    partialTrace.honesty_violations = receipt.audit_cautions;
+
+    // Step 12: Claim spread
+    const spread = computeClaimSpread(receipt);
+    partialTrace.spread_score = spread.spread_score;
+    partialTrace.spread_vector = spread.vector;
+
+    // Step 13: Build final trace
+    partialTrace.outcome = "completed";
+    partialTrace.receipt_summary = receipt.summary_claimed.slice(0, 200);
+    partialTrace.duration_ms = Date.now() - startTime;
+
+    const trace = collectTrace(seed, partialTrace, startTime);
+    return { trace, contract, handoff, receipt };
+
+  } catch (error: unknown) {
+    // Capture error into trace
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    partialTrace.outcome = "error";
+    partialTrace.rejection_point = "pipeline";
+    partialTrace.rejection_reason = errorMessage.slice(0, 200);
+    partialTrace.duration_ms = Date.now() - startTime;
+
+    const trace = collectTrace(seed, partialTrace, startTime);
+    return {
+      trace,
+      contract: contract ?? buildContract(parseSynthesizedSeed(seed.raw_idea)),
+      handoff: handoff ?? buildHandoff(
+        contract ?? buildContract(parseSynthesizedSeed(seed.raw_idea)),
+        { selected: { provider_id: "internal-fallback", score: 0, disqualified: false, disqualification_reason: "", verification_path: "none" }, candidates: [], rationale: "Fallback due to error.", constraints_applied: [], verification_path: "none" },
+      ),
+      receipt: null,
+    };
+  }
+}