switchboard-cli 0.1.0-alpha.1 → 0.1.0-alpha.3

package/node_modules/@switchboard/core/src/evaluation-lane.ts

@@ -0,0 +1,584 @@
+import { randomBytes } from "crypto";
+import type {
+  AuditedReceipt,
+  ClaimAssessment,
+  EvaluationFinding,
+  EvaluationReentrySignal,
+  EvaluationResult,
+  EvaluationTrigger,
+  EvaluationVerdict,
+  ProofManifest,
+  ValidationScope,
+} from "./types";
+import { evaluationResultSchema } from "./types";
+import { classifyScope, scopeChecks } from "./eval-scope";
+
+// ---------------------------------------------------------------------------
+// Independent Evaluation Lane — model-independent governance assessment
+// ---------------------------------------------------------------------------
+//
+// The evaluation lane is structurally separate from the builder/executor lane.
+// It receives an AuditedReceipt (the official record of what happened) plus
+// optional proof artifacts, and produces a governed EvaluationResult.
+//
+// The evaluator:
+//   - NEVER re-executes
+//   - NEVER trusts the builder's self-assessment alone
+//   - Assesses claims against structured evidence
+//   - Checks contract compliance (scope, done-when, criteria)
+//   - Inspects proof chain integrity
+//   - Produces a verdict + re-entry signal for the governed loop
+//
+// The evaluation result persists alongside the receipt and re-enters the
+// governed loop through the live event bus.
+// ---------------------------------------------------------------------------
+
+export interface EvaluationInput {
+  receipt: AuditedReceipt;
+  proofManifest?: ProofManifest | null;
+  trigger: EvaluationTrigger;
+  /** Contract scope items the run was expected to stay within. */
+  contracted_scope_in?: string[];
+  /** Contract criteria the run was expected to satisfy. */
+  contracted_done_when?: string[];
+  /** Explicitly requested validation scope. If omitted, inferred from available data. */
+  requested_scope?: ValidationScope;
+}
+
+// ---------------------------------------------------------------------------
+// Claim coverage — independent re-assessment of the receipt's claim verdicts
+// ---------------------------------------------------------------------------
+
+function evaluateClaimCoverage(
+  claimAssessments: ClaimAssessment[],
+  receipt: AuditedReceipt,
+): {
+  claims_evaluated: number;
+  claims_supported: number;
+  claims_disputed: number;
+  claims_unverifiable: number;
+  findings: EvaluationFinding[];
+} {
+  let supported = 0;
+  let disputed = 0;
+  let unverifiable = 0;
+  const findings: EvaluationFinding[] = [];
+
+  for (const ca of claimAssessments) {
+    switch (ca.disposition) {
+      case "proven":
+      case "supported":
+        supported += 1;
+        break;
+
+      case "partial":
+        // Partial evidence — the evaluator flags but doesn't dispute
+        supported += 1;
+        findings.push({
+          finding_id: `ef-${randomBytes(4).toString("hex")}`,
+          severity: "flag",
+          category: "claim_vs_evidence",
+          detail: `Claim has only partial evidence: "${ca.claim.slice(0, 80)}"`,
+          claim_ref: ca.claim.slice(0, 80),
+          evidence_ref: ca.basis,
+        });
+        break;
+
+      case "inferred":
+        // Inferred evidence — evaluator marks as unverifiable
+        unverifiable += 1;
+        findings.push({
+          finding_id: `ef-${randomBytes(4).toString("hex")}`,
+          severity: "concern",
+          category: "evidence_quality",
+          detail: `Claim relies on inferred evidence only: "${ca.claim.slice(0, 80)}"`,
+          claim_ref: ca.claim.slice(0, 80),
+          evidence_ref: ca.basis,
+        });
+        break;
+
+      case "unsubstantiated": {
+        // Distinguish core implementation claims from design-rationale/narrative claims
+        // using CLAIM-SPECIFIC overlap, not just "some core work exists."
+        //
+        // A claim is "core" if its text overlaps with implementation artifact keywords
+        // (file names, test commands, criteria text). Core claims that are unsubstantiated
+        // stay violations — they're fabricated.
+        //
+        // A claim is "rationale" if its text does NOT overlap with any structured evidence
+        // field. These are design/narrative statements ("Used iterative approach") that
+        // lack structured backing but aren't fabricated. Downgrade to concern.
+        //
+        // Hard fail preserved when:
+        // - No core work at all on the receipt
+        // - Claim text overlaps with implementation artifacts but has no evidence
+        const coreWorkExists = receipt.files_changed_count > 0 || receipt.verification_count > 0;
+
+        // Check claim-specific overlap with all structured evidence fields
+        const claimWords = ca.claim.toLowerCase().split(/\s+/).filter(w => w.length > 3);
+        const allOtherClaims = claimAssessments
+          .filter(other => other !== ca && (other.disposition === "proven" || other.disposition === "supported"))
+          .map(other => other.claim.toLowerCase());
+        const evidencePool = [
+          ...allOtherClaims,
+          receipt.objective_claimed?.toLowerCase() ?? "",
+          receipt.summary_claimed?.toLowerCase() ?? "",
+        ].join(" ");
+
+        const overlapCount = claimWords.filter(w => evidencePool.includes(w)).length;
+        const hasClaimOverlap = overlapCount >= 2 || (claimWords.length <= 2 && overlapCount >= 1);
+
+        // Rationale = core work exists AND this specific claim doesn't overlap with evidence
+        const isRationale = coreWorkExists && !hasClaimOverlap;
+
+        if (isRationale) {
+          // Design/rationale claim without structured backing — concern, not violation.
+          // Does NOT increment disputed — rationale gaps should not block evidence sufficiency.
+          unverifiable += 1;
+          findings.push({
+            finding_id: `ef-${randomBytes(4).toString("hex")}`,
+            severity: "concern",
+            category: "claim_vs_evidence",
+            detail: `Design-rationale claim lacks structured evidence: "${ca.claim.slice(0, 80)}"`,
+            claim_ref: ca.claim.slice(0, 80),
+            evidence_ref: "",
+          });
+        } else {
+          // Either no core work, or this claim overlaps with implementation artifacts — violation
+          disputed += 1;
+          findings.push({
+            finding_id: `ef-${randomBytes(4).toString("hex")}`,
+            severity: "violation",
+            category: "claim_vs_evidence",
+            detail: `Claim is unsubstantiated — no structured evidence found: "${ca.claim.slice(0, 80)}"`,
+            claim_ref: ca.claim.slice(0, 80),
+            evidence_ref: "",
+          });
+        }
+        break;
+      }
+    }
+  }
+
+  // Check for zero-evidence receipt
+  if (receipt.verification_count === 0 && receipt.evidence_count === 0) {
+    findings.push({
+      finding_id: `ef-${randomBytes(4).toString("hex")}`,
+      severity: "violation",
+      category: "evidence_quality",
+      detail: "Receipt has zero verification runs and zero evidence items.",
+      claim_ref: "",
+      evidence_ref: "",
+    });
+  }
+
+  return {
+    claims_evaluated: claimAssessments.length,
+    claims_supported: supported,
+    claims_disputed: disputed,
+    claims_unverifiable: unverifiable,
+    findings,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Contract compliance — check scope and criteria adherence
+// ---------------------------------------------------------------------------
+
+function evaluateContractCompliance(
+  receipt: AuditedReceipt,
+  contracted_scope_in: string[],
+  contracted_done_when: string[],
+): EvaluationFinding[] {
+  const findings: EvaluationFinding[] = [];
+
+  // Scope adherence — check that claimed summary references contracted scope
+  if (contracted_scope_in.length > 0) {
+    const summaryLower = receipt.summary_claimed.toLowerCase();
+    const objectiveLower = receipt.objective_claimed.toLowerCase();
+    const combined = summaryLower + " " + objectiveLower;
+
+    // Check each scope item for at least partial keyword presence
+    const scopeMissing: string[] = [];
+    for (const scopeItem of contracted_scope_in) {
+      const keywords = scopeItem.toLowerCase().split(/\s+/).filter(w => w.length > 3);
+      const matched = keywords.some(kw => combined.includes(kw));
+      if (!matched && keywords.length > 0) {
+        scopeMissing.push(scopeItem);
+      }
+    }
+
+    if (scopeMissing.length > 0) {
+      findings.push({
+        finding_id: `ef-${randomBytes(4).toString("hex")}`,
+        severity: "flag",
+        category: "scope_adherence",
+        detail: `${scopeMissing.length} contracted scope item(s) not referenced in claimed outcome: ${scopeMissing.slice(0, 3).join("; ")}`,
+        claim_ref: receipt.summary_claimed.slice(0, 80),
+        evidence_ref: "",
+      });
+    }
+  }
+
+  // If contract criteria were provided but none were met
+  if (contracted_done_when.length > 0 && receipt.criteria_met_count === 0) {
+    findings.push({
+      finding_id: `ef-${randomBytes(4).toString("hex")}`,
+      severity: "concern",
+      category: "contract_compliance",
+      detail: `Contract has ${contracted_done_when.length} done-when criteria but receipt shows 0 met.`,
+      claim_ref: "",
+      evidence_ref: "",
+    });
+  }
+
+  // Residual risk assessment
+  if (receipt.residual_count > 0) {
+    const severity = receipt.residual_count >= 3 ? "concern" as const : "flag" as const;
+    findings.push({
+      finding_id: `ef-${randomBytes(4).toString("hex")}`,
+      severity,
+      category: "residual_risk",
+      detail: `${receipt.residual_count} residual(s) carry forward: ${receipt.cautions_on_record.length} cautions, ${receipt.unresolved_on_record.length} unresolved.`,
+      claim_ref: "",
+      evidence_ref: "",
+    });
+  }
+
+  // Audit blockers
+  if (receipt.audit_blockers.length > 0) {
+    findings.push({
+      finding_id: `ef-${randomBytes(4).toString("hex")}`,
+      severity: "violation",
+      category: "contract_compliance",
+      detail: `${receipt.audit_blockers.length} audit blocker(s) present on receipt.`,
+      claim_ref: "",
+      evidence_ref: "",
+    });
+  }
+
+  return findings;
+}
+
+// ---------------------------------------------------------------------------
+// Proof chain integrity — check provenance
+// ---------------------------------------------------------------------------
+
+function evaluateProofChain(
+  receipt: AuditedReceipt,
+  proofManifest: ProofManifest | null | undefined,
+): {
+  provenance_intact: boolean;
+  proof_artifacts_checked: number;
+  findings: EvaluationFinding[];
+} {
+  const findings: EvaluationFinding[] = [];
+
+  if (!proofManifest) {
+    return {
+      provenance_intact: false,
+      proof_artifacts_checked: 0,
+      findings: [{
+        finding_id: `ef-${randomBytes(4).toString("hex")}`,
+        severity: "concern",
+        category: "provenance_gap",
+        detail: "No proof manifest available — independent verification impossible without captured artifacts.",
+        claim_ref: "",
+        evidence_ref: "",
+      }],
+    };
+  }
+
+  const artifactCount = proofManifest.artifacts.length;
+
+  // Check for execution transcript (the minimum captured proof)
+  const hasTranscript = proofManifest.artifacts.some(
+    a => a.kind === "execution-transcript" && a.provenance === "captured",
+  );
+  if (!hasTranscript) {
+    findings.push({
+      finding_id: `ef-${randomBytes(4).toString("hex")}`,
+      severity: "concern",
+      category: "provenance_gap",
+      detail: "Proof manifest has no captured execution transcript.",
+      claim_ref: "",
+      evidence_ref: "",
+    });
+  }
+
+  // Check provenance distribution
+  const capturedCount = proofManifest.provenance_summary.captured_count;
+  if (capturedCount === 0) {
+    findings.push({
+      finding_id: `ef-${randomBytes(4).toString("hex")}`,
+      severity: "concern",
+      category: "provenance_gap",
+      detail: "All proof artifacts are derived or reconstructed — zero captured.",
+      claim_ref: "",
+      evidence_ref: "",
+    });
+  }
+
+  // Provenance is intact if there is at least 1 captured artifact and a transcript
+  const provenance_intact = capturedCount > 0 && hasTranscript;
+
+  return {
+    provenance_intact,
+    proof_artifacts_checked: artifactCount,
+    findings,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Budget assessment — flag excessive correction effort
+// ---------------------------------------------------------------------------
+
+function evaluateBudgetPosture(
+  receipt: AuditedReceipt,
+): EvaluationFinding[] {
+  const findings: EvaluationFinding[] = [];
+
+  // Only assess if budget data is present on the receipt
+  if (receipt.budget_posture === "unknown") return findings;
+
+  if (receipt.budget_posture === "over-budget") {
+    findings.push({
+      finding_id: `ef-${randomBytes(4).toString("hex")}`,
+      severity: "concern",
+      category: "budget_concern",
+      detail: `Budget posture is over-budget: ${receipt.budget_posture_basis}. Correction steps: ${receipt.budget_correction_steps}, total steps: ${receipt.budget_task_steps + receipt.budget_verification_steps + receipt.budget_correction_steps}.`,
+      claim_ref: "",
+      evidence_ref: "",
+    });
+  } else if (receipt.budget_posture === "caution" && receipt.budget_had_corrections) {
+    findings.push({
+      finding_id: `ef-${randomBytes(4).toString("hex")}`,
+      severity: "flag",
+      category: "budget_concern",
+      detail: `Budget posture flagged: ${receipt.budget_posture_basis}`,
+      claim_ref: "",
+      evidence_ref: "",
+    });
+  }
+
+  return findings;
+}
+
+// ---------------------------------------------------------------------------
+// Verdict derivation — from findings
+// ---------------------------------------------------------------------------
+
+function deriveVerdict(
+  findings: EvaluationFinding[],
+  receipt: AuditedReceipt,
+  evidenceSufficient: boolean,
+): { verdict: EvaluationVerdict; basis: string } {
+  const violationCount = findings.filter(f => f.severity === "violation").length;
+  const concernCount = findings.filter(f => f.severity === "concern").length;
+
+  // Blocked or held receipts cannot pass
+  if (receipt.gate_label === "blocked-missing-evidence") {
+    return {
+      verdict: "fail",
+      basis: `Gate blocked: ${receipt.gate_reason}`,
+    };
+  }
+
+  if (receipt.gate_label === "held-for-review") {
+    return {
+      verdict: "inconclusive",
+      basis: `Gate held for review: ${receipt.gate_reason}`,
+    };
+  }
+
+  if (violationCount > 0) {
+    return {
+      verdict: "fail",
+      basis: `${violationCount} violation(s) found. Evidence does not support claims.`,
+    };
+  }
+
+  if (!evidenceSufficient) {
+    return {
+      verdict: "inconclusive",
+      basis: "Insufficient evidence for independent verification.",
+    };
+  }
+
+  if (concernCount > 0) {
+    return {
+      verdict: "pass_with_flags",
+      basis: `Pass with ${concernCount} concern(s) and ${findings.filter(f => f.severity === "flag").length} flag(s).`,
+    };
+  }
+
+  if (findings.some(f => f.severity === "flag")) {
+    return {
+      verdict: "pass_with_flags",
+      basis: `Pass with ${findings.filter(f => f.severity === "flag").length} informational flag(s).`,
+    };
+  }
+
+  return {
+    verdict: "pass",
+    basis: "Evidence supports claims. Contract satisfied. No findings.",
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Re-entry signal — from verdict and findings
+// ---------------------------------------------------------------------------
+
+function deriveReentrySignal(
+  verdict: EvaluationVerdict,
+  findings: EvaluationFinding[],
+): { signal: EvaluationReentrySignal; basis: string; cautions: string[] } {
+  const cautions: string[] = [];
+
+  switch (verdict) {
+    case "fail":
+      return {
+        signal: "block_recommended",
+        basis: "Evaluation failed — block recommended until evidence gaps are resolved.",
+        cautions: findings
+          .filter(f => f.severity === "violation")
+          .map(f => `[eval-violation] ${f.detail}`),
+      };
+
+    case "inconclusive":
+      return {
+        signal: "hold_recommended",
+        basis: "Evaluation inconclusive — hold recommended pending further evidence.",
+        cautions: findings
+          .filter(f => f.severity === "concern")
+          .map(f => `[eval-concern] ${f.detail}`),
+      };
+
+    case "pass_with_flags": {
+      const concerns = findings.filter(f => f.severity === "concern");
+      if (concerns.length > 0) {
+        for (const c of concerns) {
+          cautions.push(`[eval-concern] ${c.detail}`);
+        }
+        return {
+          signal: "caution_added",
+          basis: `Pass with ${concerns.length} concern(s) added as cautions.`,
+          cautions,
+        };
+      }
+      return { signal: "none", basis: "Pass with informational flags only.", cautions: [] };
+    }
+
+    case "pass":
+      return { signal: "none", basis: "Clean pass. No governance action needed.", cautions: [] };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Primary builder — runEvaluation
+// ---------------------------------------------------------------------------
+
+export function runEvaluation(input: EvaluationInput): EvaluationResult {
+  const { receipt, proofManifest, trigger } = input;
+  const contracted_scope_in = input.contracted_scope_in ?? [];
+  const contracted_done_when = input.contracted_done_when ?? [];
+
+  // Classify validation scope — explicit request or inferred from available data
+  const validationScope = classifyScope({
+    ...(input.requested_scope != null && { requested_scope: input.requested_scope }),
+    has_contracted_scope: contracted_scope_in.length > 0,
+    has_contracted_done_when: contracted_done_when.length > 0,
+    has_proof_manifest: !!proofManifest,
+  });
+
+  // Phase 1: Claim coverage assessment — always runs (all scopes)
+  const claimCoverage = evaluateClaimCoverage(
+    receipt.claim_assessments,
+    receipt,
+  );
+
+  // Phase 2: Contract compliance — slice-check and above
+  const complianceFindings = validationScope !== "micro-check"
+    ? evaluateContractCompliance(receipt, contracted_scope_in, contracted_done_when)
+    : [];
+
+  // Phase 3: Proof chain integrity — slice-check and above
+  const proofChain = validationScope !== "micro-check"
+    ? evaluateProofChain(receipt, proofManifest)
+    : { provenance_intact: false, proof_artifacts_checked: 0, findings: [] };
+
+  // Phase 4: Budget posture — all scopes (lightweight check)
+  const budgetFindings = evaluateBudgetPosture(receipt);
+
+  // Merge all findings
+  const allFindings: EvaluationFinding[] = [
+    ...claimCoverage.findings,
+    ...complianceFindings,
+    ...proofChain.findings,
+    ...budgetFindings,
+  ];
+
+  // Evidence sufficiency — independent of receipt's own assessment
+  // For micro-check: only claim-level sufficiency (no proof chain required)
+  // For slice-check+: requires structured evidence AND proof chain
+  const evidenceSufficient = validationScope === "micro-check"
+    ? (receipt.verification_count > 0 &&
+       receipt.evidence_count > 0 &&
+       claimCoverage.claims_disputed === 0)
+    : (receipt.verification_count > 0 &&
+       receipt.evidence_count > 0 &&
+       claimCoverage.claims_disputed === 0 &&
+       proofChain.provenance_intact);
+
+  // Derive verdict
+  const { verdict, basis: verdictBasis } = deriveVerdict(
+    allFindings,
+    receipt,
+    evidenceSufficient,
+  );
+
+  // Derive re-entry signal
+  const reentry = deriveReentrySignal(verdict, allFindings);
+
+  // Build scope description — honest about what was actually checked
+  const checksPerformed = scopeChecks(validationScope);
+  const evaluationScope = checksPerformed.join(", ");
+
+  return evaluationResultSchema.parse({
+    evaluation_id: `eval-${randomBytes(8).toString("hex")}`,
+    evaluation_version: 1,
+    created_at: new Date().toISOString(),
+    project_id: receipt.project_id,
+    dispatch_id: receipt.dispatch_id,
+    receipt_id: receipt.receipt_id,
+
+    triggered_by: trigger,
+
+    verdict,
+    verdict_basis: verdictBasis,
+
+    findings: allFindings,
+    finding_count: allFindings.length,
+    violation_count: allFindings.filter(f => f.severity === "violation").length,
+    concern_count: allFindings.filter(f => f.severity === "concern").length,
+    flag_count: allFindings.filter(f => f.severity === "flag").length,
+
+    claims_evaluated: claimCoverage.claims_evaluated,
+    claims_supported: claimCoverage.claims_supported,
+    claims_disputed: claimCoverage.claims_disputed,
+    claims_unverifiable: claimCoverage.claims_unverifiable,
+
+    evidence_sufficient: evidenceSufficient,
+    provenance_intact: proofChain.provenance_intact,
+    proof_artifacts_checked: proofChain.proof_artifacts_checked,
+
+    reentry_signal: reentry.signal,
+    reentry_basis: reentry.basis,
+    cautions_added: reentry.cautions,
+
+    evaluator_role: "independent",
+    evaluation_scope: evaluationScope,
+
+    validation_scope: validationScope,
+  });
+}