vibecheck-ai 2.0.1 → 5.0.0

package/bin/runners/lib/verdict-engine.js

@@ -0,0 +1,628 @@
+/**
+ * Verdict Engine v2
+ * 
+ * Merges all findings from detectors, computes verdict, generates ship report.
+ * Single source of truth for SHIP/WARN/BLOCK decisions.
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+
+const { createFinding, getSeverity, generateFingerprint } = require("./schemas/validator");
+
+// =============================================================================
+// VERDICT COMPUTATION
+// =============================================================================
+
+// Coverage thresholds defaults
+const DEFAULT_COVERAGE_THRESHOLDS = {
+  minActionCoverage: 0,        // --min-action-coverage
+  minRouteCoverage: 0,         // --min-route-coverage
+  requireAuthCoverage: false,  // --require-auth-coverage
+  minAuthCoverage: 80,         // Minimum % when requireAuthCoverage is true
+};
+
+/**
+ * Compute verdict from findings
+ */
+function computeVerdict(findings, options = {}) {
+  const { 
+    failOnWarn = false,
+    coverage = null,
+    thresholds = {},
+  } = options;
+
+  const effectiveThresholds = { ...DEFAULT_COVERAGE_THRESHOLDS, ...thresholds };
+
+  const stats = {
+    total: findings.length,
+    bySeverity: { BLOCK: 0, WARN: 0, INFO: 0 },
+    byCategory: {},
+    byScope: {},
+  };
+
+  const blockReasons = [];
+
+  for (const f of findings) {
+    // Count by severity
+    stats.bySeverity[f.severity] = (stats.bySeverity[f.severity] || 0) + 1;
+
+    // Count by category
+    stats.byCategory[f.category] = (stats.byCategory[f.category] || 0) + 1;
+
+    // Count by scope
+    stats.byScope[f.scope] = (stats.byScope[f.scope] || 0) + 1;
+
+    // Collect block reasons
+    if (f.severity === "BLOCK") {
+      blockReasons.push(f.title);
+    }
+  }
+
+  // Check coverage thresholds
+  const coverageViolations = [];
+  if (coverage) {
+    if (effectiveThresholds.minActionCoverage > 0 && 
+        coverage.uiActionsVerifiedPct < effectiveThresholds.minActionCoverage) {
+      coverageViolations.push(
+        `UI action coverage ${coverage.uiActionsVerifiedPct.toFixed(0)}% < ${effectiveThresholds.minActionCoverage}%`
+      );
+    }
+    if (effectiveThresholds.minRouteCoverage > 0 && 
+        coverage.clientCallsMappedPct < effectiveThresholds.minRouteCoverage) {
+      coverageViolations.push(
+        `Route coverage ${coverage.clientCallsMappedPct.toFixed(0)}% < ${effectiveThresholds.minRouteCoverage}%`
+      );
+    }
+    if (effectiveThresholds.requireAuthCoverage && 
+        coverage.authVerifiedPct !== null &&
+        coverage.authVerifiedPct < effectiveThresholds.minAuthCoverage) {
+      coverageViolations.push(
+        `Auth coverage ${coverage.authVerifiedPct.toFixed(0)}% < ${effectiveThresholds.minAuthCoverage}%`
+      );
+    }
+  }
+
+  // Determine verdict
+  let status = "SHIP";
+  let exitCode = 0;
+
+  if (stats.bySeverity.BLOCK > 0 || coverageViolations.length > 0) {
+    status = "BLOCK";
+    exitCode = 2;
+    if (coverageViolations.length > 0) {
+      blockReasons.push(...coverageViolations.map(v => `Coverage: ${v}`));
+    }
+  } else if (stats.bySeverity.WARN > 0) {
+    status = "WARN";
+    exitCode = failOnWarn ? 1 : 0;
+  }
+
+  // Generate summary
+  const summary = generateVerdictSummary(status, stats);
+
+  return {
+    status,
+    exitCode,
+    summary,
+    blockReasons: blockReasons.slice(0, 5),
+    stats,
+    coverageViolations,
+  };
+}
+
+/**
+ * Generate human-readable verdict summary
+ */
+function generateVerdictSummary(status, stats) {
+  const parts = [];
+
+  if (status === "SHIP") {
+    if (stats.bySeverity.WARN > 0) {
+      parts.push(`✅ SHIP with ${stats.bySeverity.WARN} warning(s)`);
+    } else {
+      parts.push("✅ SHIP - All checks passed");
+    }
+  } else if (status === "WARN") {
+    parts.push(`⚠️ WARN - ${stats.bySeverity.WARN} warning(s) found`);
+  } else {
+    parts.push(`🚫 BLOCK - ${stats.bySeverity.BLOCK} blocker(s) found`);
+  }
+
+  // Add category breakdown
+  const categories = Object.entries(stats.byCategory)
+    .filter(([_, count]) => count > 0)
+    .map(([cat, count]) => `${cat}: ${count}`)
+    .join(", ");
+
+  if (categories) {
+    parts.push(`Categories: ${categories}`);
+  }
+
+  return parts.join("\n");
+}
+
+// =============================================================================
+// COVERAGE METRICS
+// =============================================================================
+
+/**
+ * Compute coverage metrics from truthpack and reality data
+ */
+function computeCoverageMetrics(options = {}) {
+  const { truthpack, realityReport, proofGraph, findings } = options;
+
+  const metrics = {
+    clientCalls: {
+      total: 0,
+      mappedToServerRoutes: 0,
+      mappedToServerRoutesPct: 0,
+    },
+    runtimeRequests: {
+      total: 0,
+      mappedToClientCalls: 0,
+      mappedToClientCallsPct: 0,
+    },
+    uiActions: {
+      total: 0,
+      withMeaningfulChange: 0,
+      withMeaningfulChangePct: 0,
+    },
+    auth: {
+      protectedRoutes: 0,
+      verified: 0,
+      protectedRoutesVerifiedPct: null,
+    },
+    unmappedApiRequests: [],
+  };
+
+  // Client calls coverage
+  if (truthpack?.clientCalls?.calls) {
+    metrics.clientCalls.total = truthpack.clientCalls.calls.length;
+    
+    // Count those with linked server routes (from proof graph or matching)
+    if (proofGraph?.edges) {
+      const clientCallsWithMatches = new Set(
+        proofGraph.edges
+          .filter(e => e.type === "MATCHES" || e.type === "CALLS")
+          .map(e => e.from)
+      );
+      metrics.clientCalls.mappedToServerRoutes = clientCallsWithMatches.size;
+    }
+    
+    metrics.clientCalls.mappedToServerRoutesPct = metrics.clientCalls.total > 0
+      ? (metrics.clientCalls.mappedToServerRoutes / metrics.clientCalls.total) * 100
+      : 100;
+  }
+
+  // Runtime requests coverage
+  if (realityReport?.requests) {
+    metrics.runtimeRequests.total = realityReport.requests.length;
+    
+    // Count mapped requests
+    const mapped = realityReport.requests.filter(r => r.mappedClientCallId || r.matched);
+    metrics.runtimeRequests.mappedToClientCalls = mapped.length;
+    
+    metrics.runtimeRequests.mappedToClientCallsPct = metrics.runtimeRequests.total > 0
+      ? (metrics.runtimeRequests.mappedToClientCalls / metrics.runtimeRequests.total) * 100
+      : 100;
+
+    // Top 5 unmapped requests
+    const unmapped = realityReport.requests
+      .filter(r => !r.mappedClientCallId && !r.matched)
+      .map(r => r.canonicalPath || r.url)
+      .filter(Boolean);
+    
+    // Dedupe and take top 5
+    metrics.unmappedApiRequests = [...new Set(unmapped)].slice(0, 5);
+  }
+
+  // UI actions coverage
+  if (realityReport?.actions) {
+    metrics.uiActions.total = realityReport.actions.length;
+    
+    // Count actions with meaningful UI change
+    const withChange = realityReport.actions.filter(a => 
+      a.meaningfulChange || a.uiChangeScore > 0.5
+    );
+    metrics.uiActions.withMeaningfulChange = withChange.length;
+    
+    metrics.uiActions.withMeaningfulChangePct = metrics.uiActions.total > 0
+      ? (metrics.uiActions.withMeaningfulChange / metrics.uiActions.total) * 100
+      : 100;
+  }
+
+  // Auth coverage (only when verify-auth was used)
+  if (truthpack?.auth?.protectedPatterns?.length > 0 && realityReport?.auth) {
+    metrics.auth.protectedRoutes = truthpack.auth.protectedPatterns.length;
+    metrics.auth.verified = realityReport.auth.verified?.length || 0;
+    
+    metrics.auth.protectedRoutesVerifiedPct = metrics.auth.protectedRoutes > 0
+      ? (metrics.auth.verified / metrics.auth.protectedRoutes) * 100
+      : null;
+  }
+
+  // Flattened percentage fields for easy access
+  return {
+    ...metrics,
+    clientCallsMappedPct: metrics.clientCalls.mappedToServerRoutesPct,
+    runtimeRequestsMappedPct: metrics.runtimeRequests.mappedToClientCallsPct,
+    uiActionsVerifiedPct: metrics.uiActions.withMeaningfulChangePct,
+    authVerifiedPct: metrics.auth.protectedRoutesVerifiedPct,
+  };
+}
+
+// =============================================================================
+// SHIP REPORT GENERATION
+// =============================================================================
+
+/**
+ * Generate ship report from findings and metadata
+ */
+function generateShipReport(options = {}) {
+  const {
+    findings = [],
+    truthpack = null,
+    realityReport = null,
+    proofGraph = null,
+    repoRoot = process.cwd(),
+    failOnWarn = false,
+    startTime = Date.now(),
+  } = options;
+
+  const verdict = computeVerdict(findings, { failOnWarn });
+
+  // Build proof chain for top blockers
+  const proofChain = buildProofChain(findings, proofGraph);
+
+  // Compute coverage metrics
+  const coverage = computeCoverageMetrics({
+    truthpack,
+    realityReport,
+    proofGraph,
+    findings,
+  });
+
+  // Runtime summary
+  const runtime = realityReport ? {
+    ran: true,
+    url: realityReport.meta?.baseUrl,
+    actionsCount: realityReport.actions?.length || 0,
+    requestsCount: realityReport.requests?.length || 0,
+    toastsDetected: realityReport.signals?.filter(s => s.kind?.startsWith("toast_")).length || 0,
+  } : { ran: false };
+
+  const report = {
+    meta: {
+      version: "2.0.0",
+      generatedAt: new Date().toISOString(),
+      repoRoot,
+      commit: {
+        sha: process.env.VIBECHECK_COMMIT_SHA || getGitCommit(repoRoot),
+        branch: process.env.VIBECHECK_BRANCH || getGitBranch(repoRoot),
+      },
+      durationMs: Date.now() - startTime,
+      truthpackHash: truthpack?.index?.hashes?.truthpackHash || null,
+    },
+    verdict: {
+      status: verdict.status,
+      exitCode: verdict.exitCode,
+      summary: verdict.summary,
+      blockReasons: verdict.blockReasons,
+    },
+    findings,
+    stats: verdict.stats,
+    proofChain,
+    coverage,
+    artifacts: {
+      truthpack: ".vibecheck/truthpack.json",
+      realityReport: realityReport ? ".vibecheck/reality/last_reality.json" : null,
+      proofGraph: proofGraph ? ".vibecheck/proof-graph.json" : null,
+      missionPack: null, // Set by fix command
+    },
+    runtime,
+  };
+
+  return report;
+}
+
+/**
+ * Build proof chain for top blockers
+ */
+function buildProofChain(findings, proofGraph) {
+  const blockers = findings
+    .filter(f => f.severity === "BLOCK")
+    .slice(0, 5);
+
+  if (!proofGraph || blockers.length === 0) {
+    return { topBlockers: [] };
+  }
+
+  const topBlockers = blockers.map(finding => {
+    const chain = [];
+
+    // Find proof node if referenced
+    if (finding.proofNode && proofGraph.nodes) {
+      const node = proofGraph.nodes.find(n => n.id === finding.proofNode);
+      if (node) {
+        chain.push({
+          nodeType: node.type,
+          label: node.label,
+          evidence: node.data ? JSON.stringify(node.data).slice(0, 100) : null,
+        });
+
+        // Follow edges to build chain
+        const edges = proofGraph.edges?.filter(e => e.source === node.id || e.target === node.id) || [];
+        for (const edge of edges.slice(0, 3)) {
+          const linkedNode = proofGraph.nodes.find(n => n.id === (edge.source === node.id ? edge.target : edge.source));
+          if (linkedNode) {
+            chain.push({
+              nodeType: linkedNode.type,
+              label: linkedNode.label,
+              evidence: edge.type,
+            });
+          }
+        }
+      }
+    }
+
+    // Add evidence from finding
+    if (chain.length === 0 && finding.evidence?.length > 0) {
+      const ev = finding.evidence[0];
+      chain.push({
+        nodeType: ev.kind || "file",
+        label: ev.file ? `${ev.file}:${ev.lines}` : ev.url || "unknown",
+        evidence: ev.reason,
+      });
+    }
+
+    return {
+      findingId: finding.id,
+      title: finding.title,
+      chain,
+    };
+  });
+
+  return { topBlockers };
+}
+
+/**
+ * Write ship report to disk
+ */
+function writeShipReport(repoRoot, report) {
+  const dir = path.join(repoRoot, ".vibecheck");
+  fs.mkdirSync(dir, { recursive: true });
+
+  // Write last_ship.json
+  const lastShipPath = path.join(dir, "last_ship.json");
+  fs.writeFileSync(lastShipPath, JSON.stringify(report, null, 2));
+
+  // Write timestamped copy
+  const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+  const archiveDir = path.join(dir, "ship");
+  fs.mkdirSync(archiveDir, { recursive: true });
+  fs.writeFileSync(
+    path.join(archiveDir, `ship_${timestamp}.json`),
+    JSON.stringify(report, null, 2)
+  );
+
+  return lastShipPath;
+}
+
+// =============================================================================
+// PR COMMENT RENDERER
+// =============================================================================
+
+/**
+ * Render PR comment markdown from ship report
+ */
+function renderPRComment(report, options = {}) {
+  const { includeDetails = true, maxFindings = 10 } = options;
+
+  const lines = [];
+
+  // Header with verdict
+  const verdictEmoji = {
+    SHIP: "✅",
+    WARN: "⚠️",
+    BLOCK: "🚫",
+  }[report.verdict.status] || "❓";
+
+  lines.push(`## ${verdictEmoji} Vibecheck: ${report.verdict.status}`);
+  lines.push("");
+
+  // Stats summary
+  const { stats } = report;
+  if (stats.total > 0) {
+    lines.push(`**Findings:** ${stats.total} total`);
+    if (stats.bySeverity.BLOCK > 0) lines.push(`- 🚫 ${stats.bySeverity.BLOCK} blocker(s)`);
+    if (stats.bySeverity.WARN > 0) lines.push(`- ⚠️ ${stats.bySeverity.WARN} warning(s)`);
+    if (stats.bySeverity.INFO > 0) lines.push(`- ℹ️ ${stats.bySeverity.INFO} info`);
+    lines.push("");
+  } else {
+    lines.push("**No issues found!**");
+    lines.push("");
+  }
+
+  // Top blockers with proof chain
+  if (report.proofChain?.topBlockers?.length > 0) {
+    lines.push("### Top Blockers");
+    lines.push("");
+
+    for (const blocker of report.proofChain.topBlockers.slice(0, 3)) {
+      lines.push(`**${blocker.title}**`);
+      
+      if (blocker.chain?.length > 0) {
+        const chainStr = blocker.chain
+          .map(c => `${c.nodeType}: ${c.label}`)
+          .join(" → ");
+        lines.push(`> Proof: ${chainStr}`);
+      }
+      lines.push("");
+    }
+  }
+
+  // Detailed findings (collapsible)
+  if (includeDetails && report.findings.length > 0) {
+    lines.push("<details>");
+    lines.push("<summary>All Findings</summary>");
+    lines.push("");
+
+    const shown = report.findings.slice(0, maxFindings);
+    for (const f of shown) {
+      const emoji = f.severity === "BLOCK" ? "🚫" : f.severity === "WARN" ? "⚠️" : "ℹ️";
+      lines.push(`- ${emoji} **${f.category}**: ${f.title}`);
+      
+      if (f.evidence?.[0]?.file) {
+        lines.push(`  - 📁 \`${f.evidence[0].file}:${f.evidence[0].lines || ""}\``);
+      }
+    }
+
+    if (report.findings.length > maxFindings) {
+      lines.push("");
+      lines.push(`_...and ${report.findings.length - maxFindings} more_`);
+    }
+
+    lines.push("");
+    lines.push("</details>");
+    lines.push("");
+  }
+
+  // Runtime coverage
+  if (report.runtime?.ran) {
+    lines.push("### Runtime Verification");
+    lines.push(`- Actions: ${report.runtime.actionsCount}`);
+    lines.push(`- Requests: ${report.runtime.requestsCount}`);
+    lines.push(`- Toasts detected: ${report.runtime.toastsDetected}`);
+    
+    if (report.runtime.coverage?.percent !== undefined) {
+      lines.push(`- Route coverage: ${report.runtime.coverage.percent.toFixed(0)}%`);
+    }
+    lines.push("");
+  }
+
+  // Artifacts
+  lines.push("### Artifacts");
+  lines.push(`- 📄 Ship report: \`${report.artifacts?.truthpack || ".vibecheck/truthpack.json"}\``);
+  if (report.artifacts?.realityReport) {
+    lines.push(`- 🎭 Reality report: \`${report.artifacts.realityReport}\``);
+  }
+  lines.push("");
+
+  // Footer
+  lines.push("---");
+  lines.push(`_Generated by [Vibecheck](https://github.com/guardiavault-oss/Vibecheck) at ${report.meta.generatedAt}_`);
+
+  return lines.join("\n");
+}
+
+/**
+ * Write PR comment to file
+ */
+function writePRComment(repoRoot, comment) {
+  const dir = path.join(repoRoot, ".vibecheck");
+  fs.mkdirSync(dir, { recursive: true });
+
+  const commentPath = path.join(dir, "pr_comment.md");
+  fs.writeFileSync(commentPath, comment);
+
+  return commentPath;
+}
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+function getGitCommit(repoRoot) {
+  try {
+    const headPath = path.join(repoRoot, ".git", "HEAD");
+    const head = fs.readFileSync(headPath, "utf8").trim();
+
+    if (head.startsWith("ref:")) {
+      const refPath = path.join(repoRoot, ".git", head.slice(5).trim());
+      return fs.readFileSync(refPath, "utf8").trim().slice(0, 12);
+    }
+    return head.slice(0, 12);
+  } catch {
+    return "unknown";
+  }
+}
+
+function getGitBranch(repoRoot) {
+  try {
+    const headPath = path.join(repoRoot, ".git", "HEAD");
+    const head = fs.readFileSync(headPath, "utf8").trim();
+
+    if (head.startsWith("ref: refs/heads/")) {
+      return head.slice("ref: refs/heads/".length);
+    }
+    return "HEAD";
+  } catch {
+    return "unknown";
+  }
+}
+
+// =============================================================================
+// FINDING DEDUPLICATION
+// =============================================================================
+
+/**
+ * Deduplicate findings by fingerprint
+ */
+function deduplicateFindings(findings) {
+  const seen = new Set();
+  const deduped = [];
+
+  for (const f of findings) {
+    const fp = f.fingerprint || generateFingerprint([f.detectorId, f.title, f.evidence?.[0]?.file]);
+    if (!seen.has(fp)) {
+      seen.add(fp);
+      deduped.push(f);
+    }
+  }
+
+  return deduped;
+}
+
+/**
+ * Merge findings from multiple sources
+ */
+function mergeFindings(...findingArrays) {
+  const all = [];
+  for (const arr of findingArrays) {
+    if (Array.isArray(arr)) {
+      all.push(...arr);
+    }
+  }
+  return deduplicateFindings(all);
+}
+
+// =============================================================================
+// EXPORTS
+// =============================================================================
+
+module.exports = {
+  // Verdict
+  computeVerdict,
+  generateVerdictSummary,
+  DEFAULT_COVERAGE_THRESHOLDS,
+
+  // Coverage
+  computeCoverageMetrics,
+
+  // Ship report
+  generateShipReport,
+  writeShipReport,
+  buildProofChain,
+
+  // PR comment
+  renderPRComment,
+  writePRComment,
+
+  // Finding utilities
+  deduplicateFindings,
+  mergeFindings,
+};