switchboard-cli 0.1.0-alpha.3 → 0.1.0-alpha.4

package/node_modules/@switchboard/core/src/evaluation-lane.ts

@@ -1,584 +0,0 @@
-import { randomBytes } from "crypto";
-import type {
-  AuditedReceipt,
-  ClaimAssessment,
-  EvaluationFinding,
-  EvaluationReentrySignal,
-  EvaluationResult,
-  EvaluationTrigger,
-  EvaluationVerdict,
-  ProofManifest,
-  ValidationScope,
-} from "./types";
-import { evaluationResultSchema } from "./types";
-import { classifyScope, scopeChecks } from "./eval-scope";
-
-// ---------------------------------------------------------------------------
-// Independent Evaluation Lane — model-independent governance assessment
-// ---------------------------------------------------------------------------
-//
-// The evaluation lane is structurally separate from the builder/executor lane.
-// It receives an AuditedReceipt (the official record of what happened) plus
-// optional proof artifacts, and produces a governed EvaluationResult.
-//
-// The evaluator:
-//   - NEVER re-executes
-//   - NEVER trusts the builder's self-assessment alone
-//   - Assesses claims against structured evidence
-//   - Checks contract compliance (scope, done-when, criteria)
-//   - Inspects proof chain integrity
-//   - Produces a verdict + re-entry signal for the governed loop
-//
-// The evaluation result persists alongside the receipt and re-enters the
-// governed loop through the live event bus.
-// ---------------------------------------------------------------------------
-
-export interface EvaluationInput {
-  receipt: AuditedReceipt;
-  proofManifest?: ProofManifest | null;
-  trigger: EvaluationTrigger;
-  /** Contract scope items the run was expected to stay within. */
-  contracted_scope_in?: string[];
-  /** Contract criteria the run was expected to satisfy. */
-  contracted_done_when?: string[];
-  /** Explicitly requested validation scope. If omitted, inferred from available data. */
-  requested_scope?: ValidationScope;
-}
-
-// ---------------------------------------------------------------------------
-// Claim coverage — independent re-assessment of the receipt's claim verdicts
-// ---------------------------------------------------------------------------
-
-function evaluateClaimCoverage(
-  claimAssessments: ClaimAssessment[],
-  receipt: AuditedReceipt,
-): {
-  claims_evaluated: number;
-  claims_supported: number;
-  claims_disputed: number;
-  claims_unverifiable: number;
-  findings: EvaluationFinding[];
-} {
-  let supported = 0;
-  let disputed = 0;
-  let unverifiable = 0;
-  const findings: EvaluationFinding[] = [];
-
-  for (const ca of claimAssessments) {
-    switch (ca.disposition) {
-      case "proven":
-      case "supported":
-        supported += 1;
-        break;
-
-      case "partial":
-        // Partial evidence — the evaluator flags but doesn't dispute
-        supported += 1;
-        findings.push({
-          finding_id: `ef-${randomBytes(4).toString("hex")}`,
-          severity: "flag",
-          category: "claim_vs_evidence",
-          detail: `Claim has only partial evidence: "${ca.claim.slice(0, 80)}"`,
-          claim_ref: ca.claim.slice(0, 80),
-          evidence_ref: ca.basis,
-        });
-        break;
-
-      case "inferred":
-        // Inferred evidence — evaluator marks as unverifiable
-        unverifiable += 1;
-        findings.push({
-          finding_id: `ef-${randomBytes(4).toString("hex")}`,
-          severity: "concern",
-          category: "evidence_quality",
-          detail: `Claim relies on inferred evidence only: "${ca.claim.slice(0, 80)}"`,
-          claim_ref: ca.claim.slice(0, 80),
-          evidence_ref: ca.basis,
-        });
-        break;
-
-      case "unsubstantiated": {
-        // Distinguish core implementation claims from design-rationale/narrative claims
-        // using CLAIM-SPECIFIC overlap, not just "some core work exists."
-        //
-        // A claim is "core" if its text overlaps with implementation artifact keywords
-        // (file names, test commands, criteria text). Core claims that are unsubstantiated
-        // stay violations — they're fabricated.
-        //
-        // A claim is "rationale" if its text does NOT overlap with any structured evidence
-        // field. These are design/narrative statements ("Used iterative approach") that
-        // lack structured backing but aren't fabricated. Downgrade to concern.
-        //
-        // Hard fail preserved when:
-        // - No core work at all on the receipt
-        // - Claim text overlaps with implementation artifacts but has no evidence
-        const coreWorkExists = receipt.files_changed_count > 0 || receipt.verification_count > 0;
-
-        // Check claim-specific overlap with all structured evidence fields
-        const claimWords = ca.claim.toLowerCase().split(/\s+/).filter(w => w.length > 3);
-        const allOtherClaims = claimAssessments
-          .filter(other => other !== ca && (other.disposition === "proven" || other.disposition === "supported"))
-          .map(other => other.claim.toLowerCase());
-        const evidencePool = [
-          ...allOtherClaims,
-          receipt.objective_claimed?.toLowerCase() ?? "",
-          receipt.summary_claimed?.toLowerCase() ?? "",
-        ].join(" ");
-
-        const overlapCount = claimWords.filter(w => evidencePool.includes(w)).length;
-        const hasClaimOverlap = overlapCount >= 2 || (claimWords.length <= 2 && overlapCount >= 1);
-
-        // Rationale = core work exists AND this specific claim doesn't overlap with evidence
-        const isRationale = coreWorkExists && !hasClaimOverlap;
-
-        if (isRationale) {
-          // Design/rationale claim without structured backing — concern, not violation.
-          // Does NOT increment disputed — rationale gaps should not block evidence sufficiency.
-          unverifiable += 1;
-          findings.push({
-            finding_id: `ef-${randomBytes(4).toString("hex")}`,
-            severity: "concern",
-            category: "claim_vs_evidence",
-            detail: `Design-rationale claim lacks structured evidence: "${ca.claim.slice(0, 80)}"`,
-            claim_ref: ca.claim.slice(0, 80),
-            evidence_ref: "",
-          });
-        } else {
-          // Either no core work, or this claim overlaps with implementation artifacts — violation
-          disputed += 1;
-          findings.push({
-            finding_id: `ef-${randomBytes(4).toString("hex")}`,
-            severity: "violation",
-            category: "claim_vs_evidence",
-            detail: `Claim is unsubstantiated — no structured evidence found: "${ca.claim.slice(0, 80)}"`,
-            claim_ref: ca.claim.slice(0, 80),
-            evidence_ref: "",
-          });
-        }
-        break;
-      }
-    }
-  }
-
-  // Check for zero-evidence receipt
-  if (receipt.verification_count === 0 && receipt.evidence_count === 0) {
-    findings.push({
-      finding_id: `ef-${randomBytes(4).toString("hex")}`,
-      severity: "violation",
-      category: "evidence_quality",
-      detail: "Receipt has zero verification runs and zero evidence items.",
-      claim_ref: "",
-      evidence_ref: "",
-    });
-  }
-
-  return {
-    claims_evaluated: claimAssessments.length,
-    claims_supported: supported,
-    claims_disputed: disputed,
-    claims_unverifiable: unverifiable,
-    findings,
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Contract compliance — check scope and criteria adherence
-// ---------------------------------------------------------------------------
-
-function evaluateContractCompliance(
-  receipt: AuditedReceipt,
-  contracted_scope_in: string[],
-  contracted_done_when: string[],
-): EvaluationFinding[] {
-  const findings: EvaluationFinding[] = [];
-
-  // Scope adherence — check that claimed summary references contracted scope
-  if (contracted_scope_in.length > 0) {
-    const summaryLower = receipt.summary_claimed.toLowerCase();
-    const objectiveLower = receipt.objective_claimed.toLowerCase();
-    const combined = summaryLower + " " + objectiveLower;
-
-    // Check each scope item for at least partial keyword presence
-    const scopeMissing: string[] = [];
-    for (const scopeItem of contracted_scope_in) {
-      const keywords = scopeItem.toLowerCase().split(/\s+/).filter(w => w.length > 3);
-      const matched = keywords.some(kw => combined.includes(kw));
-      if (!matched && keywords.length > 0) {
-        scopeMissing.push(scopeItem);
-      }
-    }
-
-    if (scopeMissing.length > 0) {
-      findings.push({
-        finding_id: `ef-${randomBytes(4).toString("hex")}`,
-        severity: "flag",
-        category: "scope_adherence",
-        detail: `${scopeMissing.length} contracted scope item(s) not referenced in claimed outcome: ${scopeMissing.slice(0, 3).join("; ")}`,
-        claim_ref: receipt.summary_claimed.slice(0, 80),
-        evidence_ref: "",
-      });
-    }
-  }
-
-  // If contract criteria were provided but none were met
-  if (contracted_done_when.length > 0 && receipt.criteria_met_count === 0) {
-    findings.push({
-      finding_id: `ef-${randomBytes(4).toString("hex")}`,
-      severity: "concern",
-      category: "contract_compliance",
-      detail: `Contract has ${contracted_done_when.length} done-when criteria but receipt shows 0 met.`,
-      claim_ref: "",
-      evidence_ref: "",
-    });
-  }
-
-  // Residual risk assessment
-  if (receipt.residual_count > 0) {
-    const severity = receipt.residual_count >= 3 ? "concern" as const : "flag" as const;
-    findings.push({
-      finding_id: `ef-${randomBytes(4).toString("hex")}`,
-      severity,
-      category: "residual_risk",
-      detail: `${receipt.residual_count} residual(s) carry forward: ${receipt.cautions_on_record.length} cautions, ${receipt.unresolved_on_record.length} unresolved.`,
-      claim_ref: "",
-      evidence_ref: "",
-    });
-  }
-
-  // Audit blockers
-  if (receipt.audit_blockers.length > 0) {
-    findings.push({
-      finding_id: `ef-${randomBytes(4).toString("hex")}`,
-      severity: "violation",
-      category: "contract_compliance",
-      detail: `${receipt.audit_blockers.length} audit blocker(s) present on receipt.`,
-      claim_ref: "",
-      evidence_ref: "",
-    });
-  }
-
-  return findings;
-}
-
-// ---------------------------------------------------------------------------
-// Proof chain integrity — check provenance
-// ---------------------------------------------------------------------------
-
-function evaluateProofChain(
-  receipt: AuditedReceipt,
-  proofManifest: ProofManifest | null | undefined,
-): {
-  provenance_intact: boolean;
-  proof_artifacts_checked: number;
-  findings: EvaluationFinding[];
-} {
-  const findings: EvaluationFinding[] = [];
-
-  if (!proofManifest) {
-    return {
-      provenance_intact: false,
-      proof_artifacts_checked: 0,
-      findings: [{
-        finding_id: `ef-${randomBytes(4).toString("hex")}`,
-        severity: "concern",
-        category: "provenance_gap",
-        detail: "No proof manifest available — independent verification impossible without captured artifacts.",
-        claim_ref: "",
-        evidence_ref: "",
-      }],
-    };
-  }
-
-  const artifactCount = proofManifest.artifacts.length;
-
-  // Check for execution transcript (the minimum captured proof)
-  const hasTranscript = proofManifest.artifacts.some(
-    a => a.kind === "execution-transcript" && a.provenance === "captured",
-  );
-  if (!hasTranscript) {
-    findings.push({
-      finding_id: `ef-${randomBytes(4).toString("hex")}`,
-      severity: "concern",
-      category: "provenance_gap",
-      detail: "Proof manifest has no captured execution transcript.",
-      claim_ref: "",
-      evidence_ref: "",
-    });
-  }
-
-  // Check provenance distribution
-  const capturedCount = proofManifest.provenance_summary.captured_count;
-  if (capturedCount === 0) {
-    findings.push({
-      finding_id: `ef-${randomBytes(4).toString("hex")}`,
-      severity: "concern",
-      category: "provenance_gap",
-      detail: "All proof artifacts are derived or reconstructed — zero captured.",
-      claim_ref: "",
-      evidence_ref: "",
-    });
-  }
-
-  // Provenance is intact if there is at least 1 captured artifact and a transcript
-  const provenance_intact = capturedCount > 0 && hasTranscript;
-
-  return {
-    provenance_intact,
-    proof_artifacts_checked: artifactCount,
-    findings,
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Budget assessment — flag excessive correction effort
-// ---------------------------------------------------------------------------
-
-function evaluateBudgetPosture(
-  receipt: AuditedReceipt,
-): EvaluationFinding[] {
-  const findings: EvaluationFinding[] = [];
-
-  // Only assess if budget data is present on the receipt
-  if (receipt.budget_posture === "unknown") return findings;
-
-  if (receipt.budget_posture === "over-budget") {
-    findings.push({
-      finding_id: `ef-${randomBytes(4).toString("hex")}`,
-      severity: "concern",
-      category: "budget_concern",
-      detail: `Budget posture is over-budget: ${receipt.budget_posture_basis}. Correction steps: ${receipt.budget_correction_steps}, total steps: ${receipt.budget_task_steps + receipt.budget_verification_steps + receipt.budget_correction_steps}.`,
-      claim_ref: "",
-      evidence_ref: "",
-    });
-  } else if (receipt.budget_posture === "caution" && receipt.budget_had_corrections) {
-    findings.push({
-      finding_id: `ef-${randomBytes(4).toString("hex")}`,
-      severity: "flag",
-      category: "budget_concern",
-      detail: `Budget posture flagged: ${receipt.budget_posture_basis}`,
-      claim_ref: "",
-      evidence_ref: "",
-    });
-  }
-
-  return findings;
-}
-
-// ---------------------------------------------------------------------------
-// Verdict derivation — from findings
-// ---------------------------------------------------------------------------
-
-function deriveVerdict(
-  findings: EvaluationFinding[],
-  receipt: AuditedReceipt,
-  evidenceSufficient: boolean,
-): { verdict: EvaluationVerdict; basis: string } {
-  const violationCount = findings.filter(f => f.severity === "violation").length;
-  const concernCount = findings.filter(f => f.severity === "concern").length;
-
-  // Blocked or held receipts cannot pass
-  if (receipt.gate_label === "blocked-missing-evidence") {
-    return {
-      verdict: "fail",
-      basis: `Gate blocked: ${receipt.gate_reason}`,
-    };
-  }
-
-  if (receipt.gate_label === "held-for-review") {
-    return {
-      verdict: "inconclusive",
-      basis: `Gate held for review: ${receipt.gate_reason}`,
-    };
-  }
-
-  if (violationCount > 0) {
-    return {
-      verdict: "fail",
-      basis: `${violationCount} violation(s) found. Evidence does not support claims.`,
-    };
-  }
-
-  if (!evidenceSufficient) {
-    return {
-      verdict: "inconclusive",
-      basis: "Insufficient evidence for independent verification.",
-    };
-  }
-
-  if (concernCount > 0) {
-    return {
-      verdict: "pass_with_flags",
-      basis: `Pass with ${concernCount} concern(s) and ${findings.filter(f => f.severity === "flag").length} flag(s).`,
-    };
-  }
-
-  if (findings.some(f => f.severity === "flag")) {
-    return {
-      verdict: "pass_with_flags",
-      basis: `Pass with ${findings.filter(f => f.severity === "flag").length} informational flag(s).`,
-    };
-  }
-
-  return {
-    verdict: "pass",
-    basis: "Evidence supports claims. Contract satisfied. No findings.",
-  };
-}
-
-// ---------------------------------------------------------------------------
-// Re-entry signal — from verdict and findings
-// ---------------------------------------------------------------------------
-
-function deriveReentrySignal(
-  verdict: EvaluationVerdict,
-  findings: EvaluationFinding[],
-): { signal: EvaluationReentrySignal; basis: string; cautions: string[] } {
-  const cautions: string[] = [];
-
-  switch (verdict) {
-    case "fail":
-      return {
-        signal: "block_recommended",
-        basis: "Evaluation failed — block recommended until evidence gaps are resolved.",
-        cautions: findings
-          .filter(f => f.severity === "violation")
-          .map(f => `[eval-violation] ${f.detail}`),
-      };
-
-    case "inconclusive":
-      return {
-        signal: "hold_recommended",
-        basis: "Evaluation inconclusive — hold recommended pending further evidence.",
-        cautions: findings
-          .filter(f => f.severity === "concern")
-          .map(f => `[eval-concern] ${f.detail}`),
-      };
-
-    case "pass_with_flags": {
-      const concerns = findings.filter(f => f.severity === "concern");
-      if (concerns.length > 0) {
-        for (const c of concerns) {
-          cautions.push(`[eval-concern] ${c.detail}`);
-        }
-        return {
-          signal: "caution_added",
-          basis: `Pass with ${concerns.length} concern(s) added as cautions.`,
-          cautions,
-        };
-      }
-      return { signal: "none", basis: "Pass with informational flags only.", cautions: [] };
-    }
-
-    case "pass":
-      return { signal: "none", basis: "Clean pass. No governance action needed.", cautions: [] };
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Primary builder — runEvaluation
-// ---------------------------------------------------------------------------
-
-export function runEvaluation(input: EvaluationInput): EvaluationResult {
-  const { receipt, proofManifest, trigger } = input;
-  const contracted_scope_in = input.contracted_scope_in ?? [];
-  const contracted_done_when = input.contracted_done_when ?? [];
-
-  // Classify validation scope — explicit request or inferred from available data
-  const validationScope = classifyScope({
-    ...(input.requested_scope != null && { requested_scope: input.requested_scope }),
-    has_contracted_scope: contracted_scope_in.length > 0,
-    has_contracted_done_when: contracted_done_when.length > 0,
-    has_proof_manifest: !!proofManifest,
-  });
-
-  // Phase 1: Claim coverage assessment — always runs (all scopes)
-  const claimCoverage = evaluateClaimCoverage(
-    receipt.claim_assessments,
-    receipt,
-  );
-
-  // Phase 2: Contract compliance — slice-check and above
-  const complianceFindings = validationScope !== "micro-check"
-    ? evaluateContractCompliance(receipt, contracted_scope_in, contracted_done_when)
-    : [];
-
-  // Phase 3: Proof chain integrity — slice-check and above
-  const proofChain = validationScope !== "micro-check"
-    ? evaluateProofChain(receipt, proofManifest)
-    : { provenance_intact: false, proof_artifacts_checked: 0, findings: [] };
-
-  // Phase 4: Budget posture — all scopes (lightweight check)
-  const budgetFindings = evaluateBudgetPosture(receipt);
-
-  // Merge all findings
-  const allFindings: EvaluationFinding[] = [
-    ...claimCoverage.findings,
-    ...complianceFindings,
-    ...proofChain.findings,
-    ...budgetFindings,
-  ];
-
-  // Evidence sufficiency — independent of receipt's own assessment
-  // For micro-check: only claim-level sufficiency (no proof chain required)
-  // For slice-check+: requires structured evidence AND proof chain
-  const evidenceSufficient = validationScope === "micro-check"
-    ? (receipt.verification_count > 0 &&
-       receipt.evidence_count > 0 &&
-       claimCoverage.claims_disputed === 0)
-    : (receipt.verification_count > 0 &&
-       receipt.evidence_count > 0 &&
-       claimCoverage.claims_disputed === 0 &&
-       proofChain.provenance_intact);
-
-  // Derive verdict
-  const { verdict, basis: verdictBasis } = deriveVerdict(
-    allFindings,
-    receipt,
-    evidenceSufficient,
-  );
-
-  // Derive re-entry signal
-  const reentry = deriveReentrySignal(verdict, allFindings);
-
-  // Build scope description — honest about what was actually checked
-  const checksPerformed = scopeChecks(validationScope);
-  const evaluationScope = checksPerformed.join(", ");
-
-  return evaluationResultSchema.parse({
-    evaluation_id: `eval-${randomBytes(8).toString("hex")}`,
-    evaluation_version: 1,
-    created_at: new Date().toISOString(),
-    project_id: receipt.project_id,
-    dispatch_id: receipt.dispatch_id,
-    receipt_id: receipt.receipt_id,
-
-    triggered_by: trigger,
-
-    verdict,
-    verdict_basis: verdictBasis,
-
-    findings: allFindings,
-    finding_count: allFindings.length,
-    violation_count: allFindings.filter(f => f.severity === "violation").length,
-    concern_count: allFindings.filter(f => f.severity === "concern").length,
-    flag_count: allFindings.filter(f => f.severity === "flag").length,
-
-    claims_evaluated: claimCoverage.claims_evaluated,
-    claims_supported: claimCoverage.claims_supported,
-    claims_disputed: claimCoverage.claims_disputed,
-    claims_unverifiable: claimCoverage.claims_unverifiable,
-
-    evidence_sufficient: evidenceSufficient,
-    provenance_intact: proofChain.provenance_intact,
-    proof_artifacts_checked: proofChain.proof_artifacts_checked,
-
-    reentry_signal: reentry.signal,
-    reentry_basis: reentry.basis,
-    cautions_added: reentry.cautions,
-
-    evaluator_role: "independent",
-    evaluation_scope: evaluationScope,
-
-    validation_scope: validationScope,
-  });
-}