vibecheck-ai 2.0.1 → 5.0.0

package/bin/runners/lib/agent-firewall/risk/scorer.js

@@ -0,0 +1,328 @@
+/**
+ * Risk Scoring Engine
+ * 
+ * Calculates numerical risk scores for proposed changes.
+ * Uses configurable vectors and thresholds to determine risk levels.
+ */
+
+"use strict";
+
+const { RISK_VECTORS, RISK_LEVELS, getRiskLevel } = require("./vectors");
+const { loadThresholds, getDecision } = require("./thresholds");
+const { classifyFileDomain } = require("../reality/state");
+
+/**
+ * @typedef {Object} RiskScore
+ * @property {number} total - Total risk score
+ * @property {string} level - Risk level (LOW, MEDIUM, HIGH, CRITICAL)
+ * @property {Object} vectors - Individual vector scores
+ * @property {string[]} reasons - Human-readable risk reasons
+ * @property {Object} decision - Decision based on thresholds
+ */
+
+/**
+ * Build context object for risk calculation
+ * @param {Object} params - Score parameters
+ * @returns {Object} Risk calculation context
+ */
+function buildContext(params) {
+  const {
+    files = [],
+    operations = [],
+    claims = [],
+    evidence = [],
+    intent = "",
+    assumptions = [],
+    proposalConfidence = 1,
+    policy = {},
+  } = params;
+  
+  // Extract domains from files
+  const domains = new Set();
+  for (const file of files) {
+    const path = file.path || file;
+    const domain = classifyFileDomain(path);
+    domains.add(domain);
+  }
+  
+  // Identify unresolved assumptions
+  const unresolvedAssumptions = [];
+  for (const assumption of assumptions) {
+    const evidenceForAssumption = evidence.find(e => 
+      e.claim?.key === assumption.key || 
+      e.claim?.type === assumption.type
+    );
+    
+    if (!evidenceForAssumption || evidenceForAssumption.status === "UNPROVEN") {
+      unresolvedAssumptions.push(assumption);
+    }
+  }
+  
+  // Detect new items
+  const newEnvVars = claims
+    .filter(c => c.type === "env" && !c.exists)
+    .map(c => c.key || c.value);
+  
+  const newRoutes = claims
+    .filter(c => c.type === "route" && !c.exists)
+    .map(c => c.path || c.value);
+  
+  const newDependencies = claims
+    .filter(c => c.type === "dependency" && !c.exists)
+    .map(c => c.name || c.value);
+  
+  return {
+    files,
+    operations,
+    claims,
+    evidence,
+    intent,
+    assumptions,
+    proposalConfidence,
+    domains: Array.from(domains),
+    unresolvedAssumptions,
+    newEnvVars,
+    newRoutes,
+    newDependencies,
+    policy,
+  };
+}
+
+/**
+ * Calculate risk score for a change
+ * @param {Object} params - Score parameters
+ * @returns {RiskScore} Risk score result
+ */
+function calculateRiskScore(params) {
+  const context = buildContext(params);
+  const policy = params.policy || {};
+  const thresholds = loadThresholds(policy);
+  
+  // Calculate individual vector scores
+  const vectorScores = {};
+  const reasons = [];
+  let totalScore = 0;
+  
+  for (const [key, vector] of Object.entries(RISK_VECTORS)) {
+    try {
+      // Get weight from policy or use default
+      const weight = policy.risk?.vectorWeights?.[vector.id] ?? vector.baseWeight;
+      
+      // Skip disabled vectors
+      if (weight === 0) continue;
+      
+      // Calculate raw score
+      const rawScore = vector.calculate(context);
+      const weightedScore = Math.round(rawScore * weight);
+      
+      vectorScores[vector.id] = {
+        raw: rawScore,
+        weighted: weightedScore,
+        weight,
+        name: vector.name,
+        description: vector.description,
+      };
+      
+      totalScore += weightedScore;
+      
+      // Add reason if score is significant
+      if (weightedScore > 0) {
+        const threshold = thresholds.vectors?.[vector.id];
+        if (threshold) {
+          if (weightedScore >= threshold.block) {
+            reasons.push(`${vector.name}: ${weightedScore} (CRITICAL - exceeds block threshold)`);
+          } else if (weightedScore >= threshold.warn) {
+            reasons.push(`${vector.name}: ${weightedScore} (WARNING - exceeds warn threshold)`);
+          } else if (weightedScore >= 10) {
+            reasons.push(`${vector.name}: ${weightedScore}`);
+          }
+        } else if (weightedScore >= 15) {
+          reasons.push(`${vector.name}: ${weightedScore}`);
+        }
+      }
+    } catch (error) {
+      // Log but continue with other vectors
+      console.warn(`Error calculating ${vector.id} risk: ${error.message}`);
+    }
+  }
+  
+  // Get risk level
+  const riskLevel = getRiskLevel(totalScore);
+  
+  // Get decision based on thresholds
+  const decision = getDecision(totalScore, thresholds, context.domains);
+  
+  // Build result
+  const result = {
+    total: totalScore,
+    level: riskLevel.label,
+    levelColor: riskLevel.color,
+    vectors: vectorScores,
+    reasons: reasons.length > 0 ? reasons : [`Total risk score: ${totalScore}`],
+    decision,
+    context: {
+      fileCount: context.files.length,
+      domains: context.domains,
+      unresolvedAssumptions: context.unresolvedAssumptions.length,
+      newEnvVars: context.newEnvVars.length,
+      newRoutes: context.newRoutes.length,
+    },
+    thresholds: {
+      autoAllow: thresholds.autoAllow,
+      requireConfirm: thresholds.requireConfirm,
+      autoBlock: thresholds.autoBlock,
+    },
+  };
+  
+  return result;
+}
+
+/**
+ * Quick risk assessment without full calculation
+ * @param {Object} params - Basic parameters
+ * @returns {Object} Quick assessment
+ */
+function quickAssess(params) {
+  const { files = [], operations = [], domains = [] } = params;
+  
+  // Quick checks
+  const hasDeletes = operations.some(op => op.type === "delete");
+  const hasMigrations = files.some(f => (f.path || f).includes("migration"));
+  const touchesAuth = domains.includes("auth") || files.some(f => (f.path || f).includes("auth"));
+  const touchesPayments = domains.includes("payments") || files.some(f => 
+    (f.path || f).includes("payment") || (f.path || f).includes("stripe")
+  );
+  
+  // Estimate risk level
+  let estimatedLevel = "LOW";
+  const flags = [];
+  
+  if (hasDeletes) {
+    flags.push("Contains deletions");
+    estimatedLevel = "MEDIUM";
+  }
+  
+  if (hasMigrations) {
+    flags.push("Contains migrations");
+    estimatedLevel = "HIGH";
+  }
+  
+  if (touchesAuth) {
+    flags.push("Touches auth");
+    estimatedLevel = estimatedLevel === "LOW" ? "MEDIUM" : estimatedLevel;
+  }
+  
+  if (touchesPayments) {
+    flags.push("Touches payments");
+    estimatedLevel = "HIGH";
+  }
+  
+  if (files.length > 10) {
+    flags.push("Large change (>10 files)");
+    estimatedLevel = estimatedLevel === "LOW" ? "MEDIUM" : estimatedLevel;
+  }
+  
+  if (files.length > 20) {
+    estimatedLevel = "HIGH";
+  }
+  
+  return {
+    estimatedLevel,
+    flags,
+    requiresFullAssessment: flags.length > 0 || files.length > 5,
+  };
+}
+
+/**
+ * Get risk breakdown by domain
+ * @param {RiskScore} riskScore - Calculated risk score
+ * @returns {Object} Domain breakdown
+ */
+function getDomainBreakdown(riskScore) {
+  const breakdown = {};
+  
+  for (const domain of riskScore.context?.domains || []) {
+    breakdown[domain] = {
+      files: 0,
+      contribution: 0,
+    };
+  }
+  
+  // Estimate contribution based on domain vector
+  const domainVector = riskScore.vectors?.domain;
+  if (domainVector && riskScore.context?.domains) {
+    const totalDomains = riskScore.context.domains.length;
+    if (totalDomains > 0) {
+      const avgContribution = domainVector.weighted / totalDomains;
+      for (const domain of riskScore.context.domains) {
+        breakdown[domain].contribution = Math.round(avgContribution);
+      }
+    }
+  }
+  
+  return breakdown;
+}
+
+/**
+ * Format risk score for display
+ * @param {RiskScore} riskScore - Risk score
+ * @returns {string} Formatted string
+ */
+function formatRiskScore(riskScore) {
+  const lines = [
+    `Risk Score: ${riskScore.total} (${riskScore.level})`,
+    `Decision: ${riskScore.decision.decision}`,
+    "",
+    "Breakdown:",
+  ];
+  
+  for (const [id, vector] of Object.entries(riskScore.vectors)) {
+    if (vector.weighted > 0) {
+      lines.push(`  ${vector.name}: ${vector.weighted}`);
+    }
+  }
+  
+  if (riskScore.reasons.length > 0) {
+    lines.push("", "Risk Factors:");
+    for (const reason of riskScore.reasons) {
+      lines.push(`  - ${reason}`);
+    }
+  }
+  
+  return lines.join("\n");
+}
+
+/**
+ * Compare two risk scores
+ * @param {RiskScore} a - First score
+ * @param {RiskScore} b - Second score
+ * @returns {Object} Comparison result
+ */
+function compareScores(a, b) {
+  return {
+    difference: a.total - b.total,
+    percentChange: b.total > 0 ? ((a.total - b.total) / b.total) * 100 : 0,
+    levelChanged: a.level !== b.level,
+    oldLevel: b.level,
+    newLevel: a.level,
+    vectorChanges: Object.keys(a.vectors).reduce((acc, key) => {
+      const oldVal = b.vectors[key]?.weighted || 0;
+      const newVal = a.vectors[key]?.weighted || 0;
+      if (oldVal !== newVal) {
+        acc[key] = { old: oldVal, new: newVal, change: newVal - oldVal };
+      }
+      return acc;
+    }, {}),
+  };
+}
+
+module.exports = {
+  calculateRiskScore,
+  quickAssess,
+  buildContext,
+  getDomainBreakdown,
+  formatRiskScore,
+  compareScores,
+  RISK_VECTORS,
+  RISK_LEVELS,
+};

package/bin/runners/lib/agent-firewall/risk/thresholds.js

@@ -0,0 +1,322 @@
+/**
+ * Risk Thresholds
+ * 
+ * Configurable thresholds for risk-based decisions.
+ * These can be overridden in policy configuration.
+ */
+
+"use strict";
+
+/**
+ * Default threshold configuration
+ * 
+ * Tuned to reduce false positives while maintaining security.
+ * Single-file UI/component changes should typically auto-allow.
+ * Multi-file changes to core/auth/payments require confirmation.
+ * Only block truly dangerous patterns (migrations, mass deletes, etc.)
+ */
+const DEFAULT_THRESHOLDS = {
+  /**
+   * Score thresholds for automatic decisions
+   * Raised significantly to focus only on real issues (hallucinations, drift)
+   * Normal development changes should almost never trigger
+   */
+  autoAllow: 50,        // Auto-allow if score <= this (raised from 30)
+  requireConfirm: 85,   // Require confirmation if score > this (raised from 70)
+  autoBlock: 120,       // Auto-block if score >= this (raised from 100)
+  
+  /**
+   * Vector-specific thresholds
+   */
+  vectors: {
+    surface_area: {
+      warn: 10,
+      block: 25,
+    },
+    blast_radius: {
+      warn: 30,
+      block: 60,
+    },
+    irreversibility: {
+      warn: 25,
+      block: 50,
+    },
+    confidence: {
+      warn: 20,
+      block: 60,
+    },
+    novelty: {
+      warn: 20,
+      block: 40,
+    },
+    domain: {
+      warn: 30,
+      block: 60,
+    },
+    side_effects: {
+      warn: 20,
+      block: 50,
+    },
+  },
+  
+  /**
+   * Domain-specific thresholds
+   * Multipliers reduced to prevent over-penalization of normal changes
+   */
+  domains: {
+    auth: {
+      multiplier: 1.2,         // Reduced from 1.5 - auth changes are common
+      requireConfirm: 50,      // Raised from 30
+      autoBlock: 90,           // Raised from 60
+    },
+    payments: {
+      multiplier: 1.3,         // Reduced from 1.8 - payments needs care but not blocking
+      requireConfirm: 45,      // Raised from 25
+      autoBlock: 85,           // Raised from 50
+    },
+    database: {
+      multiplier: 1.1,         // Reduced from 1.3 - DB changes are normal
+      requireConfirm: 55,      // Raised from 40
+      autoBlock: 95,           // Raised from 70
+    },
+    security: {
+      multiplier: 1.2,         // Reduced from 1.6
+      requireConfirm: 50,      // Raised from 25
+      autoBlock: 90,           // Raised from 55
+    },
+    core: {
+      multiplier: 1.1,         // Reduced from 1.2
+      requireConfirm: 60,      // Raised from 45
+      autoBlock: 95,           // Raised from 75
+    },
+    middleware: {
+      multiplier: 1.0,         // Reduced from 1.1 - middleware is usually safe
+      requireConfirm: 65,      // Raised from 50
+      autoBlock: 100,          // Raised from 80
+    },
+    ui: {
+      multiplier: 0.7,         // Reduced from 0.8 - UI is very safe
+      requireConfirm: 80,      // Raised from 60
+      autoBlock: 120,          // Raised from 90 - UI should almost never block
+    },
+    test: {
+      multiplier: 0.3,         // Reduced from 0.5 - tests are safest
+      requireConfirm: 100,     // Raised from 70
+      autoBlock: 150,          // Raised from 95 - tests should never block
+    },
+  },
+  
+  /**
+   * File count limits
+   */
+  fileLimits: {
+    warn: 5,
+    block: 15,
+    hardLimit: 50,
+  },
+  
+  /**
+   * Line count limits
+   */
+  lineLimits: {
+    warn: 200,
+    block: 500,
+    hardLimit: 2000,
+  },
+};
+
+/**
+ * Profile presets
+ */
+const THRESHOLD_PROFILES = {
+  /**
+   * Strict profile - very conservative
+   */
+  strict: {
+    autoAllow: 10,
+    requireConfirm: 30,
+    autoBlock: 60,
+    fileLimits: {
+      warn: 3,
+      block: 8,
+      hardLimit: 20,
+    },
+    lineLimits: {
+      warn: 100,
+      block: 300,
+      hardLimit: 1000,
+    },
+  },
+  
+  /**
+   * Balanced profile - default
+   */
+  balanced: {
+    ...DEFAULT_THRESHOLDS,
+  },
+  
+  /**
+   * Permissive profile - more lenient
+   */
+  permissive: {
+    autoAllow: 25,
+    requireConfirm: 70,
+    autoBlock: 95,
+    fileLimits: {
+      warn: 10,
+      block: 25,
+      hardLimit: 100,
+    },
+    lineLimits: {
+      warn: 500,
+      block: 1000,
+      hardLimit: 5000,
+    },
+  },
+  
+  /**
+   * Repo-lock profile - most conservative
+   */
+  "repo-lock": {
+    autoAllow: 5,
+    requireConfirm: 15,
+    autoBlock: 40,
+    fileLimits: {
+      warn: 2,
+      block: 5,
+      hardLimit: 10,
+    },
+    lineLimits: {
+      warn: 50,
+      block: 150,
+      hardLimit: 500,
+    },
+  },
+};
+
+/**
+ * Load thresholds from policy
+ * @param {Object} policy - Policy configuration
+ * @returns {Object} Merged threshold configuration
+ */
+function loadThresholds(policy = {}) {
+  // Start with default
+  let thresholds = { ...DEFAULT_THRESHOLDS };
+  
+  // Apply profile if specified
+  const profile = policy.profile || "balanced";
+  if (THRESHOLD_PROFILES[profile]) {
+    thresholds = mergeDeep(thresholds, THRESHOLD_PROFILES[profile]);
+  }
+  
+  // Apply custom thresholds from policy
+  if (policy.thresholds) {
+    thresholds = mergeDeep(thresholds, policy.thresholds);
+  }
+  
+  // Apply risk configuration
+  if (policy.risk) {
+    if (policy.risk.autoAllow !== undefined) thresholds.autoAllow = policy.risk.autoAllow;
+    if (policy.risk.requireConfirm !== undefined) thresholds.requireConfirm = policy.risk.requireConfirm;
+    if (policy.risk.autoBlock !== undefined) thresholds.autoBlock = policy.risk.autoBlock;
+  }
+  
+  return thresholds;
+}
+
+/**
+ * Deep merge objects
+ */
+function mergeDeep(target, source) {
+  const output = { ...target };
+  
+  for (const key of Object.keys(source)) {
+    if (source[key] && typeof source[key] === "object" && !Array.isArray(source[key])) {
+      output[key] = mergeDeep(output[key] || {}, source[key]);
+    } else {
+      output[key] = source[key];
+    }
+  }
+  
+  return output;
+}
+
+/**
+ * Get decision based on score and thresholds
+ * @param {number} score - Risk score
+ * @param {Object} thresholds - Threshold configuration
+ * @param {string[]} domains - Affected domains
+ * @returns {Object} Decision object
+ */
+function getDecision(score, thresholds, domains = []) {
+  // Check for domain-specific overrides
+  let effectiveThresholds = { ...thresholds };
+  let maxMultiplier = 1;
+  
+  for (const domain of domains) {
+    const domainConfig = thresholds.domains?.[domain];
+    if (domainConfig) {
+      if (domainConfig.multiplier > maxMultiplier) {
+        maxMultiplier = domainConfig.multiplier;
+      }
+      // Use the most restrictive domain threshold
+      if (domainConfig.autoBlock < effectiveThresholds.autoBlock) {
+        effectiveThresholds.autoBlock = domainConfig.autoBlock;
+      }
+      if (domainConfig.requireConfirm < effectiveThresholds.requireConfirm) {
+        effectiveThresholds.requireConfirm = domainConfig.requireConfirm;
+      }
+    }
+  }
+  
+  // Apply domain multiplier to score
+  const effectiveScore = Math.round(score * maxMultiplier);
+  
+  // Determine decision
+  if (effectiveScore >= effectiveThresholds.autoBlock) {
+    return {
+      decision: "BLOCK",
+      reason: `Risk score ${effectiveScore} exceeds auto-block threshold ${effectiveThresholds.autoBlock}`,
+      score: effectiveScore,
+      multiplier: maxMultiplier,
+      thresholdUsed: effectiveThresholds.autoBlock,
+    };
+  }
+  
+  if (effectiveScore > effectiveThresholds.requireConfirm) {
+    return {
+      decision: "REQUIRE_CONFIRMATION",
+      reason: `Risk score ${effectiveScore} exceeds confirmation threshold ${effectiveThresholds.requireConfirm}`,
+      score: effectiveScore,
+      multiplier: maxMultiplier,
+      thresholdUsed: effectiveThresholds.requireConfirm,
+    };
+  }
+  
+  if (effectiveScore <= effectiveThresholds.autoAllow) {
+    return {
+      decision: "ALLOW",
+      reason: `Risk score ${effectiveScore} within auto-allow threshold ${effectiveThresholds.autoAllow}`,
+      score: effectiveScore,
+      multiplier: maxMultiplier,
+      thresholdUsed: effectiveThresholds.autoAllow,
+    };
+  }
+  
+  // Default to allow with warning for scores in between
+  return {
+    decision: "ALLOW_WITH_WARNING",
+    reason: `Risk score ${effectiveScore} is elevated but within limits`,
+    score: effectiveScore,
+    multiplier: maxMultiplier,
+    thresholdUsed: effectiveThresholds.requireConfirm,
+  };
+}
+
+module.exports = {
+  DEFAULT_THRESHOLDS,
+  THRESHOLD_PROFILES,
+  loadThresholds,
+  getDecision,
+  mergeDeep,
+};