vibecheck-ai 2.0.1 → 5.0.0

package/bin/runners/lib/agent-firewall/ai/false-positive-analyzer.js

@@ -0,0 +1,474 @@
+/**
+ * AI-Powered False Positive Analyzer
+ * 
+ * Uses AI to analyze code context and determine if a violation is a false positive.
+ * Helps reduce false positives by understanding code intent and patterns.
+ */
+
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+
+// Cache for AI responses to avoid repeated calls
+const aiCache = new Map();
+const CACHE_TTL = 24 * 60 * 60 * 1000; // 24 hours
+
+/**
+ * Analyze if a violation is likely a false positive using AI
+ * @param {object} params
+ * @param {object} params.violation - Violation object
+ * @param {object} params.claim - Original claim
+ * @param {string} params.filePath - File path where violation occurred
+ * @param {string} params.projectRoot - Project root directory
+ * @param {object} params.policy - Policy configuration
+ * @returns {Promise<object>} Analysis result with isFalsePositive boolean and confidence
+ */
+async function analyzeFalsePositive({ violation, claim, filePath, projectRoot, policy }) {
+  const ruleConfig = policy.rules?.ai_false_positive_detection;
+  
+  // Check if AI analysis is enabled
+  if (!ruleConfig || !ruleConfig.enabled) {
+    return { isFalsePositive: false, confidence: 0, reason: "AI analysis disabled" };
+  }
+  
+  // Check cache first
+  const cacheKey = `${violation.rule}|${claim.value}|${filePath}`;
+  const cached = aiCache.get(cacheKey);
+  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
+    return cached.result;
+  }
+  
+  try {
+    // Read the file to get context
+    const fullPath = path.join(projectRoot, filePath);
+    if (!fs.existsSync(fullPath)) {
+      return { isFalsePositive: false, confidence: 0, reason: "File not found" };
+    }
+    
+    const fileContent = fs.readFileSync(fullPath, "utf8");
+    const lines = fileContent.split("\n");
+    
+    // Extract context around the violation (10 lines before and after)
+    const pointer = claim.pointer || violation.claim?.pointer;
+    let startLine = 1;
+    let endLine = lines.length;
+    
+    if (pointer) {
+      const match = pointer.match(/:(\d+)-(\d+)/);
+      if (match) {
+        const lineNum = parseInt(match[1], 10);
+        startLine = Math.max(1, lineNum - 10);
+        endLine = Math.min(lines.length, lineNum + 10);
+      }
+    }
+    
+    const context = lines.slice(startLine - 1, endLine).join("\n");
+    const lineNumbers = Array.from({ length: endLine - startLine + 1 }, (_, i) => startLine + i);
+    const numberedContext = lines.slice(startLine - 1, endLine)
+      .map((line, i) => `${lineNumbers[i]}: ${line}`)
+      .join("\n");
+    
+    // Build AI prompt
+    const prompt = buildAnalysisPrompt({
+      violation,
+      claim,
+      filePath,
+      context: numberedContext,
+      ruleType: violation.rule
+    });
+    
+    // Try LLM first if enabled, otherwise use heuristics
+    let analysis = null;
+    if (ruleConfig.useLLM) {
+      analysis = await analyzeWithLLM(prompt, ruleConfig);
+    }
+    
+    // Fall back to heuristics if LLM not available or failed
+    if (!analysis) {
+      analysis = await analyzeWithHeuristics({
+        violation,
+        claim,
+        filePath,
+        context,
+        numberedContext
+      });
+    }
+    
+    // Cache result
+    aiCache.set(cacheKey, {
+      timestamp: Date.now(),
+      result: analysis
+    });
+    
+    return analysis;
+  } catch (error) {
+    // If AI analysis fails, default to not a false positive
+    return {
+      isFalsePositive: false,
+      confidence: 0,
+      reason: `AI analysis failed: ${error.message}`
+    };
+  }
+}
+
+/**
+ * Build prompt for AI analysis
+ */
+function buildAnalysisPrompt({ violation, claim, filePath, context, ruleType }) {
+  return `Analyze this code violation to determine if it's a false positive.
+
+Rule: ${ruleType}
+Claim Type: ${claim.type}
+Claim Value: ${claim.value}
+File: ${filePath}
+
+Code Context:
+\`\`\`
+${context}
+\`\`\`
+
+Question: Is this a false positive? Consider:
+1. Is this an import path being mistaken for a route? (e.g., "from './api/content'")
+2. Is this an external API call that shouldn't be validated? (e.g., "/content/blog" going to backend)
+3. Is this test/fixture code that should be ignored?
+4. Is this a legitimate pattern that the rule doesn't understand?
+
+Respond with JSON:
+{
+  "isFalsePositive": boolean,
+  "confidence": 0.0-1.0,
+  "reason": "explanation",
+  "suggestedFix": "what to do if false positive"
+}`;
+}
+
+/**
+ * Analyze using heuristics (can be replaced with actual LLM call)
+ * This provides immediate value while LLM integration can be added later
+ */
+async function analyzeWithHeuristics({ violation, claim, filePath, context, numberedContext }) {
+  const claimValue = String(claim.value || "").trim();
+  const filePathLower = filePath.toLowerCase();
+  const contextLower = context.toLowerCase();
+  
+  // Heuristic 1: Import paths
+  if (claimValue.includes("./") || claimValue.includes("../")) {
+    // Check if it's in an import/require statement
+    const importPatterns = [
+      /from\s+['"]\./,
+      /require\(['"]\./,
+      /import\s+.*from\s+['"]\./
+    ];
+    
+    for (const pattern of importPatterns) {
+      if (pattern.test(context)) {
+        return {
+          isFalsePositive: true,
+          confidence: 0.95,
+          reason: "This appears to be an import path, not a route",
+          suggestedFix: "Update route detection to exclude import statements"
+        };
+      }
+    }
+  }
+  
+  // Heuristic 2: External API calls (not starting with /api/)
+  if (violation.rule === "ghost_route" && claimValue.startsWith("/")) {
+    if (!claimValue.startsWith("/api/")) {
+      return {
+        isFalsePositive: true,
+        confidence: 0.9,
+        reason: "This is an external API call to the backend, not a Next.js route",
+        suggestedFix: "Only validate Next.js API routes (starting with /api/)"
+      };
+    }
+  }
+  
+  // Heuristic 3: Test files
+  if (filePathLower.includes("test") || 
+      filePathLower.includes("spec") || 
+      filePathLower.includes("fixture") ||
+      filePathLower.includes("mock")) {
+    return {
+      isFalsePositive: true,
+      confidence: 0.85,
+      reason: "This is in a test/fixture file",
+      suggestedFix: "Add test files to .vibecheckignore"
+    };
+  }
+  
+  // Heuristic 4: Comments or strings
+  if (claimValue.includes("//") || 
+      claimValue.includes("/*") ||
+      contextLower.includes(`"${claimValue}"`) ||
+      contextLower.includes(`'${claimValue}'`)) {
+    // Check if it's in a comment
+    const lines = numberedContext.split("\n");
+    for (const line of lines) {
+      if (line.includes(claimValue)) {
+        const beforeValue = line.substring(0, line.indexOf(claimValue));
+        if (beforeValue.includes("//") || beforeValue.includes("/*")) {
+          return {
+            isFalsePositive: true,
+            confidence: 0.8,
+            reason: "This appears to be in a comment",
+            suggestedFix: "Skip route detection in comments"
+          };
+        }
+      }
+    }
+  }
+  
+  // Heuristic 5: Environment variable names (for ghost_env)
+  if (violation.rule === "ghost_env") {
+    // Check if it's a standard/system env var that doesn't need declaration
+    const systemEnvVars = [
+      "NODE_ENV", "PATH", "HOME", "USER", "SHELL", "TMPDIR",
+      "CI", "GITHUB_ACTIONS", "GITLAB_CI", "CIRCLECI", "BUILDKITE",
+      "COLORTERM", "TERM", "LANG", "LC_ALL"
+    ];
+    
+    if (systemEnvVars.includes(claimValue)) {
+      return {
+        isFalsePositive: true,
+        confidence: 0.9,
+        reason: "This is a standard system environment variable",
+        suggestedFix: "Add system env vars to allowlist"
+      };
+    }
+  }
+  
+  // Heuristic 6: Success messages in UI components (for fake_success_ui)
+  if (violation.rule === "fake_success_ui") {
+    // Check if it's just displaying a version number or static text
+    if (claimValue.match(/v?\d+\.\d+\.\d+/) || // Version numbers
+        claimValue.length < 10) { // Very short messages
+      return {
+        isFalsePositive: true,
+        confidence: 0.7,
+        reason: "This appears to be a version number or static text, not a success message",
+        suggestedFix: "Improve success message detection to exclude version numbers"
+      };
+    }
+  }
+  
+  // Default: not a false positive
+  return {
+    isFalsePositive: false,
+    confidence: 0.5,
+    reason: "No clear indicators of false positive",
+    suggestedFix: null
+  };
+}
+
+/**
+ * Call actual LLM for analysis (when AI is available)
+ * Uses OpenAI or Anthropic API directly
+ */
+async function analyzeWithLLM(prompt, config = {}) {
+  const { useLLM = false, provider = "openai" } = config;
+  
+  if (!useLLM) {
+    return null;
+  }
+  
+  const hasOpenAI = !!process.env.OPENAI_API_KEY;
+  const hasAnthropic = !!process.env.ANTHROPIC_API_KEY;
+  
+  if (!hasOpenAI && !hasAnthropic) {
+    return null;
+  }
+  
+  try {
+    // Try OpenAI first (default)
+    if ((provider === "openai" || !hasAnthropic) && hasOpenAI) {
+      return await callOpenAI(prompt);
+    }
+    
+    // Try Anthropic
+    if ((provider === "anthropic" || !hasOpenAI) && hasAnthropic) {
+      return await callAnthropic(prompt);
+    }
+  } catch (error) {
+    // If LLM call fails, fall back to heuristics
+    console.warn(`[AI] LLM analysis failed: ${error.message}`);
+    return null;
+  }
+  
+  return null;
+}
+
+/**
+ * Call OpenAI API
+ */
+async function callOpenAI(prompt) {
+  const https = require("https");
+  const http = require("http");
+  
+  const apiKey = process.env.OPENAI_API_KEY;
+  if (!apiKey) return null;
+  
+  const requestBody = JSON.stringify({
+    model: "gpt-4o-mini", // Fast and cheap model
+    messages: [
+      {
+        role: "system",
+        content: "You are a code analysis assistant. Analyze if a code violation is a false positive. Respond with JSON only."
+      },
+      {
+        role: "user",
+        content: prompt
+      }
+    ],
+    response_format: { type: "json_object" },
+    temperature: 0.3,
+    max_tokens: 500
+  });
+  
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: "api.openai.com",
+      path: "/v1/chat/completions",
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Authorization": `Bearer ${apiKey}`,
+        "Content-Length": Buffer.byteLength(requestBody)
+      },
+      timeout: 5000
+    };
+    
+    const req = https.request(options, (res) => {
+      let data = "";
+      
+      res.on("data", (chunk) => {
+        data += chunk;
+      });
+      
+      res.on("end", () => {
+        try {
+          const response = JSON.parse(data);
+          if (response.error) {
+            reject(new Error(response.error.message));
+            return;
+          }
+          
+          const content = response.choices?.[0]?.message?.content;
+          if (!content) {
+            resolve(null);
+            return;
+          }
+          
+          const result = JSON.parse(content);
+          resolve(result);
+        } catch (error) {
+          reject(error);
+        }
+      });
+    });
+    
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("Request timeout"));
+    });
+    
+    req.write(requestBody);
+    req.end();
+  });
+}
+
+/**
+ * Call Anthropic API
+ */
+async function callAnthropic(prompt) {
+  const https = require("https");
+  
+  const apiKey = process.env.ANTHROPIC_API_KEY;
+  if (!apiKey) return null;
+  
+  const requestBody = JSON.stringify({
+    model: "claude-3-haiku-20240307", // Fast and cheap model
+    max_tokens: 500,
+    messages: [
+      {
+        role: "user",
+        content: prompt
+      }
+    ],
+    system: "You are a code analysis assistant. Analyze if a code violation is a false positive. Respond with JSON only."
+  });
+  
+  return new Promise((resolve, reject) => {
+    const options = {
+      hostname: "api.anthropic.com",
+      path: "/v1/messages",
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "x-api-key": apiKey,
+        "anthropic-version": "2023-06-01",
+        "Content-Length": Buffer.byteLength(requestBody)
+      },
+      timeout: 5000
+    };
+    
+    const req = https.request(options, (res) => {
+      let data = "";
+      
+      res.on("data", (chunk) => {
+        data += chunk;
+      });
+      
+      res.on("end", () => {
+        try {
+          const response = JSON.parse(data);
+          if (response.error) {
+            reject(new Error(response.error.message));
+            return;
+          }
+          
+          const content = response.content?.[0]?.text;
+          if (!content) {
+            resolve(null);
+            return;
+          }
+          
+          // Extract JSON from response (might be wrapped in markdown)
+          const jsonMatch = content.match(/\{[\s\S]*\}/);
+          if (!jsonMatch) {
+            resolve(null);
+            return;
+          }
+          
+          const result = JSON.parse(jsonMatch[0]);
+          resolve(result);
+        } catch (error) {
+          reject(error);
+        }
+      });
+    });
+    
+    req.on("error", reject);
+    req.on("timeout", () => {
+      req.destroy();
+      reject(new Error("Request timeout"));
+    });
+    
+    req.write(requestBody);
+    req.end();
+  });
+}
+
+/**
+ * Clear AI cache (useful for testing)
+ */
+function clearCache() {
+  aiCache.clear();
+}
+
+module.exports = {
+  analyzeFalsePositive,
+  clearCache
+};