vibecheck-ai 2.0.1 → 5.0.0

package/bin/runners/lib/agent-firewall/evidence/resolver.js

@@ -0,0 +1,102 @@
+/**
+ * Evidence Resolver
+ * 
+ * Main orchestrator for resolving claims against truthpack.
+ * Returns PROVEN, UNPROVEN, or CONTRADICTS for each claim.
+ */
+
+"use strict";
+
+const routeEvidence = require("./route-evidence");
+const envEvidence = require("./env-evidence");
+const authEvidence = require("./auth-evidence");
+const contractEvidence = require("./contract-evidence");
+const sideEffectEvidence = require("./side-effect-evidence");
+const { CLAIM_TYPES } = require("../claims/claim-types");
+
+/**
+ * Resolve evidence for all claims
+ * @param {string} projectRoot - Project root directory
+ * @param {array} claims - Array of claims to resolve
+ * @returns {array} Array of evidence results
+ */
+function resolveEvidence(projectRoot, claims) {
+  const results = [];
+  
+  for (let i = 0; i < claims.length; i++) {
+    const claim = claims[i];
+    const claimId = `claim_${i}`;
+    
+    let result;
+    
+    switch (claim.type) {
+      case CLAIM_TYPES.ROUTE:
+        result = routeEvidence.resolve(projectRoot, claim);
+        break;
+        
+      case CLAIM_TYPES.ENV:
+        result = envEvidence.resolve(projectRoot, claim);
+        break;
+        
+      case CLAIM_TYPES.AUTH:
+        result = authEvidence.resolve(projectRoot, claim);
+        break;
+        
+      case CLAIM_TYPES.CONTRACT:
+        result = contractEvidence.resolve(projectRoot, claim);
+        break;
+        
+      case CLAIM_TYPES.SIDE_EFFECT:
+        result = sideEffectEvidence.resolve(projectRoot, claim);
+        break;
+        
+      case CLAIM_TYPES.HTTP_CALL:
+        // HTTP calls are checked as routes
+        result = routeEvidence.resolve(projectRoot, {
+          ...claim,
+          type: CLAIM_TYPES.ROUTE,
+          value: extractRouteFromHttpCall(claim.value)
+        });
+        break;
+        
+      case CLAIM_TYPES.UI_SUCCESS:
+        // UI success claims are checked for side effects
+        result = sideEffectEvidence.resolve(projectRoot, claim);
+        break;
+        
+      default:
+        result = {
+          claimId,
+          result: "UNPROVEN",
+          sources: [],
+          reason: `Unknown claim type: ${claim.type}`
+        };
+    }
+    
+    results.push({
+      claimId,
+      ...result
+    });
+  }
+  
+  return results;
+}
+
+/**
+ * Extract route path from HTTP call claim value
+ * @param {string} httpCall - HTTP call string (e.g., "GET /api/users")
+ * @returns {string} Route path
+ */
+function extractRouteFromHttpCall(httpCall) {
+  // Handle "GET /api/users" format
+  const match = httpCall.match(/\s+(.+)$/);
+  if (match) {
+    return match[1];
+  }
+  // Handle "/api/users" format
+  return httpCall.startsWith("/") ? httpCall : `/${httpCall}`;
+}
+
+module.exports = {
+  resolveEvidence
+};

package/bin/runners/lib/agent-firewall/evidence/route-evidence.js

@@ -0,0 +1,213 @@
+/**
+ * Route Evidence Resolver
+ * 
+ * Resolves route claims against truthpack.routes.json
+ * Checks for ghost routes (UI references route not registered).
+ */
+
+"use strict";
+
+const { getRoutes } = require("../truthpack");
+const { canonicalizePath } = require("../../route-truth");
+const { shouldIgnore } = require("../utils/ignore-checker");
+
+/**
+ * Resolve route claim evidence
+ * @param {string} projectRoot - Project root directory
+ * @param {object} claim - Route claim
+ * @returns {object} Evidence result
+ */
+function resolve(projectRoot, claim) {
+  // Skip checking routes from ignored files (test files, fixtures, etc.)
+  if (claim.pointer && shouldIgnore(projectRoot, claim.pointer.split(":")[0])) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "ignored",
+        pointer: claim.pointer,
+        confidence: 1.0
+      }],
+      reason: `Route from ignored file (test/fixture)`
+    };
+  }
+  
+  // Normalize route path from claim
+  const routePath = canonicalizePath(claim.value);
+  
+  // Skip external API calls (routes that don't start with /api/ in Next.js context)
+  // Routes like /content/blog, /content/faq are external backend API calls, not Next.js routes
+  // Only validate Next.js API routes (those in apps/web-ui/src/app/api/)
+  if (!routePath.startsWith('/api/')) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "external_api",
+        pointer: claim.pointer,
+        confidence: 1.0
+      }],
+      reason: `Route ${routePath} is an external API call, not a Next.js route`
+    };
+  }
+  
+  // Skip wildcard patterns from template literals (they're approximations)
+  // These are detected from fetch(`${url}/api/...`) and are not exact routes
+  if (routePath.includes('*') || routePath === '/api/*') {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "pattern",
+        pointer: claim.pointer,
+        confidence: 0.8
+      }],
+      reason: `Route pattern ${routePath} is a template literal approximation, not an exact route`
+    };
+  }
+  
+  // Skip backend API routes (routes from template literals using apiUrl)
+  // These are calls to the backend API server, not Next.js API routes
+  if (claim.isBackendApi || (claim.reason && claim.reason.includes('backend API'))) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "backend_api",
+        pointer: claim.pointer,
+        confidence: 0.9
+      }],
+      reason: `Route ${routePath} is a backend API call (using apiUrl variable), not a Next.js route`
+    };
+  }
+  
+  // Skip routes that are clearly backend API routes (v1, v2, etc. are typically backend)
+  // Next.js API routes are usually simpler like /api/checkout, /api/health
+  if (routePath.match(/^\/api\/v\d+\//)) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "backend_api",
+        pointer: claim.pointer,
+        confidence: 0.85
+      }],
+      reason: `Route ${routePath} appears to be a backend API route (versioned), not a Next.js route`
+    };
+  }
+  
+  const routes = getRoutes(projectRoot);
+  
+  // Check if route exists in truthpack
+  if (routes && routes.length > 0) {
+    // First, check for exact match
+    const exactMatch = routes.find(route => {
+      const routePathNormalized = canonicalizePath(route.path || route);
+      return routePathNormalized === routePath;
+    });
+    
+    if (exactMatch) {
+      return {
+        result: "PROVEN",
+        sources: [{
+          type: "truthpack.routes",
+          pointer: claim.pointer,
+          confidence: 0.9
+        }],
+        reason: `Route ${routePath} found in truthpack (exact match)`
+      };
+    }
+    
+    // Then check parameterized routes - but be conservative
+    // Only match if it looks like a parameter value (numeric/UUID-like)
+    const paramMatch = routes.find(route => {
+      const routePathNormalized = canonicalizePath(route.path || route);
+      
+      if (!isParameterizedPath(routePathNormalized)) return false;
+      
+      const routeParts = routePathNormalized.split("/").filter(Boolean);
+      const claimParts = routePath.split("/").filter(Boolean);
+      
+      if (routeParts.length !== claimParts.length) return false;
+      
+      // Check if all non-parameter segments match exactly
+      for (let i = 0; i < routeParts.length; i++) {
+        const rSeg = routeParts[i];
+        const cSeg = claimParts[i];
+        
+        // If it's a parameter, check if the concrete value looks like a parameter
+        if (rSeg.startsWith(":")) {
+          // Only match if it looks like an ID (numeric or UUID-like)
+          // Don't match literal words like "create", "update", etc.
+          const looksLikeId = /^\d+$/.test(cSeg) || /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(cSeg);
+          if (!looksLikeId) return false;
+        } else if (rSeg !== cSeg) {
+          return false;
+        }
+      }
+      
+      return true;
+    });
+    
+    if (paramMatch) {
+      return {
+        result: "PROVEN",
+        sources: [{
+          type: "truthpack.routes",
+          pointer: claim.pointer,
+          confidence: 0.7
+        }],
+        reason: `Route ${routePath} matches parameterized route in truthpack`
+      };
+    }
+    
+    // No match found - ghost route
+    return {
+      result: "UNPROVEN",
+      sources: [],
+      reason: `Route ${routePath} not found in truthpack (ghost route)`
+    };
+  } else {
+    // No routes in truthpack - cannot prove
+    return {
+      result: "UNPROVEN",
+      sources: [],
+      reason: "No routes found in truthpack"
+    };
+  }
+}
+
+function isParameterizedPath(p) {
+  const s = String(p || "").trim();
+  return s.includes(":") || s.includes("*");
+}
+
+function matchPath(pattern, concrete) {
+  const pat = String(pattern || "").trim();
+  const con = String(concrete || "").trim();
+  
+  if (pat === con) return true;
+  
+  // Simple parameterized matching - only match if segments align
+  const patParts = pat.split("/").filter(Boolean);
+  const conParts = con.split("/").filter(Boolean);
+  
+  if (patParts.length !== conParts.length) return false;
+  
+  for (let i = 0; i < patParts.length; i++) {
+    const pSeg = patParts[i];
+    const cSeg = conParts[i];
+    
+    // Parameter placeholder (:id, :slug) matches any segment
+    if (pSeg.startsWith(":")) {
+      continue;
+    }
+    // Wildcard (*slug) matches remainder
+    if (pSeg.startsWith("*")) {
+      return true;
+    }
+    // Exact match required for literal segments
+    if (pSeg !== cSeg) return false;
+  }
+  
+  return true;
+}
+
+module.exports = {
+  resolve
+};

package/bin/runners/lib/agent-firewall/evidence/side-effect-evidence.js

@@ -0,0 +1,145 @@
+/**
+ * Side Effect Evidence Resolver
+ * 
+ * Detects unverified side effects (DB writes, email, payments).
+ * Checks for test coverage or reality proof.
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+/**
+ * Resolve side effect claim evidence
+ * @param {string} projectRoot - Project root directory
+ * @param {object} claim - Side effect claim
+ * @returns {object} Evidence result
+ */
+function resolve(projectRoot, claim) {
+  const claimFile = claim.file || "";
+  const claimValue = claim.value.toLowerCase();
+  
+  // Detect side effect types
+  const hasDbWrite = /\b(create|update|delete|insert|save|write)\b/i.test(claimValue) ||
+                     /\b(prisma|sequelize|mongoose|db\.|database\.)\b/i.test(claimValue);
+  
+  const hasEmail = /\b(email|sendMail|nodemailer|sendgrid|mailgun)\b/i.test(claimValue);
+  
+  const hasPayment = /\b(stripe|payment|charge|checkout|billing)\b/i.test(claimValue);
+  
+  if (!hasDbWrite && !hasEmail && !hasPayment) {
+    // Not a side effect
+    return {
+      result: "PROVEN",
+      sources: [],
+      reason: "No side effect detected"
+    };
+  }
+  
+  // Check for test coverage
+  const testFile = findTestFile(projectRoot, claimFile);
+  if (testFile && fs.existsSync(testFile)) {
+    const testContent = fs.readFileSync(testFile, "utf8");
+    // Check if test covers the side effect
+    if (testContent.includes(claimValue.slice(0, 20)) || 
+        testContent.includes("mock") || 
+        testContent.includes("test")) {
+      return {
+        result: "PROVEN",
+        sources: [{
+          type: "repo.search",
+          pointer: testFile,
+          confidence: 0.7
+        }],
+        reason: "Test file found for side effect"
+      };
+    }
+  }
+  
+  // Check for reality proof (reality report)
+  const realityReport = findRealityReport(projectRoot, claimFile);
+  if (realityReport) {
+    return {
+      result: "PROVEN",
+      sources: [{
+        type: "repo.search",
+        pointer: realityReport,
+        confidence: 0.8
+      }],
+      reason: "Reality proof found for side effect"
+    };
+  }
+  
+  // Side effect detected but no verification found
+  return {
+    result: "UNPROVEN",
+    sources: [],
+    reason: `Side effect detected (${hasDbWrite ? 'DB' : ''}${hasEmail ? 'Email' : ''}${hasPayment ? 'Payment' : ''}) but no test coverage or reality proof found`
+  };
+}
+
+/**
+ * Find test file for a source file
+ * @param {string} projectRoot - Project root directory
+ * @param {string} sourceFile - Source file path
+ * @returns {string|null} Test file path or null
+ */
+function findTestFile(projectRoot, sourceFile) {
+  if (!sourceFile) return null;
+  
+  // Try common test file patterns
+  const baseName = path.basename(sourceFile, path.extname(sourceFile));
+  const dir = path.dirname(sourceFile);
+  
+  const patterns = [
+    `${dir}/${baseName}.test.ts`,
+    `${dir}/${baseName}.test.tsx`,
+    `${dir}/${baseName}.test.js`,
+    `${dir}/__tests__/${baseName}.test.ts`,
+    `${dir}/__tests__/${baseName}.test.tsx`,
+    `${dir.replace(/\/src\//, '/tests/')}/${baseName}.test.ts`
+  ];
+  
+  for (const pattern of patterns) {
+    const testPath = path.join(projectRoot, pattern);
+    if (fs.existsSync(testPath)) {
+      return pattern;
+    }
+  }
+  
+  return null;
+}
+
+/**
+ * Find reality report for a file
+ * @param {string} projectRoot - Project root directory
+ * @param {string} sourceFile - Source file path
+ * @returns {string|null} Reality report path or null
+ */
+function findRealityReport(projectRoot, sourceFile) {
+  const realityDir = path.join(projectRoot, ".vibecheck", "reality");
+  
+  if (!fs.existsSync(realityDir)) {
+    return null;
+  }
+  
+  // Look for recent reality reports
+  const reports = fs.readdirSync(realityDir)
+    .filter(file => file.endsWith('.json'))
+    .map(file => path.join(realityDir, file))
+    .filter(file => {
+      try {
+        const report = JSON.parse(fs.readFileSync(file, "utf8"));
+        return report.files && report.files.includes(sourceFile);
+      } catch {
+        return false;
+      }
+    });
+  
+  return reports.length > 0 ? reports[0] : null;
+}
+
+module.exports = {
+  resolve
+};

package/bin/runners/lib/agent-firewall/fs-hook/daemon.js

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+/**
+ * File System Hook Daemon
+ * 
+ * Runs as a background process to intercept file writes.
+ * Start with: node bin/runners/lib/agent-firewall/fs-hook/daemon.js
+ */
+
+const path = require("path");
+const { startFileSystemHook } = require("./installer");
+
+const projectRoot = process.cwd();
+
+console.log("🛡️  Starting Agent Firewall File System Hook...\n");
+console.log(`   Project: ${projectRoot}\n`);
+
+startFileSystemHook(projectRoot);
+
+console.log("✅ File System Hook running (press Ctrl+C to stop)\n");

package/bin/runners/lib/agent-firewall/fs-hook/installer.js

@@ -0,0 +1,87 @@
+/**
+ * File System Hook Installer
+ * 
+ * Installs and manages the file system hook daemon.
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const { FileSystemHook } = require("./watcher");
+
+let globalHook = null;
+
+/**
+ * Install file system hook
+ * @param {string} projectRoot - Project root directory
+ * @returns {object} Installation result
+ */
+function installFileSystemHook(projectRoot) {
+  const hookScript = path.join(__dirname, "daemon.js");
+  const packageJson = path.join(projectRoot, "package.json");
+
+  // Add script to package.json
+  if (fs.existsSync(packageJson)) {
+    const pkg = JSON.parse(fs.readFileSync(packageJson, "utf8"));
+    if (!pkg.scripts) {
+      pkg.scripts = {};
+    }
+    pkg.scripts["firewall:fs-hook"] = `node ${path.relative(projectRoot, hookScript)}`;
+    fs.writeFileSync(packageJson, JSON.stringify(pkg, null, 2));
+  }
+
+  // Create marker file
+  const markerFile = path.join(projectRoot, ".vibecheck", "fs-hook-enabled");
+  fs.writeFileSync(markerFile, JSON.stringify({
+    enabled: true,
+    installedAt: new Date().toISOString()
+  }, null, 2));
+
+  return {
+    success: true,
+    message: "File system hook installed. Run 'npm run firewall:fs-hook' to start."
+  };
+}
+
+/**
+ * Start file system hook daemon
+ * @param {string} projectRoot - Project root directory
+ */
+function startFileSystemHook(projectRoot) {
+  if (globalHook) {
+    console.log("⚠️  File system hook already running");
+    return;
+  }
+
+  globalHook = new FileSystemHook(projectRoot);
+  globalHook.start().catch(console.error);
+
+  // Keep process alive
+  process.on("SIGINT", () => {
+    if (globalHook) {
+      globalHook.stop();
+    }
+    process.exit(0);
+  });
+
+  // Keep alive
+  setInterval(() => {}, 1000);
+}
+
+/**
+ * Stop file system hook
+ */
+function stopFileSystemHook() {
+  if (globalHook) {
+    globalHook.stop();
+    globalHook = null;
+  }
+}
+
+module.exports = {
+  installFileSystemHook,
+  startFileSystemHook,
+  stopFileSystemHook,
+  FileSystemHook
+};