ship-safe 2.0.0 → 3.1.0

package/cli/commands/scan.js

@@ -2,18 +2,21 @@
  * Scan Command
  * ============
  *
- * Scans a directory for leaked secrets using pattern matching.
+ * Scans a directory for leaked secrets using pattern matching + entropy scoring.
  *
  * USAGE:
- *   ship-safe scan [path]     Scan specified path (default: current directory)
- *   ship-safe scan . -v       Verbose mode (show files being scanned)
- *   ship-safe scan . --json   Output as JSON (for CI integration)
+ *   ship-safe scan [path]            Scan specified path (default: current directory)
+ *   ship-safe scan . -v              Verbose mode (show files being scanned)
+ *   ship-safe scan . --json          Output as JSON (for CI integration)
+ *   ship-safe scan . --include-tests Also scan test files (excluded by default)
+ *
+ * SUPPRESSING FALSE POSITIVES:
+ *   Add  # ship-safe-ignore  as a comment on the same line to suppress a finding.
+ *   Create a .ship-safeignore file (same syntax as .gitignore) to exclude paths.
  *
  * EXIT CODES:
  *   0 - No secrets found
  *   1 - Secrets found (or error)
- *
- * This allows CI pipelines to fail builds when secrets are detected.
  */
 
 import fs from 'fs';
@@ -25,10 +28,54 @@ import {
   SECRET_PATTERNS,
   SKIP_DIRS,
   SKIP_EXTENSIONS,
+  TEST_FILE_PATTERNS,
   MAX_FILE_SIZE
 } from '../utils/patterns.js';
+import { isHighEntropyMatch, getConfidence } from '../utils/entropy.js';
 import * as output from '../utils/output.js';
 
+// =============================================================================
+// CUSTOM PATTERNS (.ship-safe.json)
+// =============================================================================
+
+/**
+ * Load custom patterns from .ship-safe.json in the project root.
+ *
+ * Format:
+ *   {
+ *     "patterns": [
+ *       {
+ *         "name": "My Internal Key",
+ *         "pattern": "MYAPP_[A-Z0-9]{32}",
+ *         "severity": "high",
+ *         "description": "Internal API key for myapp services."
+ *       }
+ *     ]
+ *   }
+ */
+function loadCustomPatterns(rootPath) {
+  const configPath = path.join(rootPath, '.ship-safe.json');
+  if (!fs.existsSync(configPath)) return [];
+
+  try {
+    const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+    if (!Array.isArray(config.patterns)) return [];
+
+    return config.patterns
+      .filter(p => p.name && p.pattern)
+      .map(p => ({
+        name: `[custom] ${p.name}`,
+        pattern: new RegExp(p.pattern, 'g'),
+        severity: p.severity || 'high',
+        description: p.description || `Custom pattern: ${p.name}`,
+        custom: true,
+      }));
+  } catch (err) {
+    output.warning(`.ship-safe.json parse error: ${err.message}`);
+    return [];
+  }
+}
+
 // =============================================================================
 // MAIN SCAN FUNCTION
 // =============================================================================
@@ -42,6 +89,17 @@ export async function scanCommand(targetPath = '.', options = {}) {
     process.exit(1);
   }
 
+  // Load .ship-safeignore patterns
+  const ignorePatterns = loadIgnoreFile(absolutePath);
+
+  // Load custom patterns from .ship-safe.json
+  const customPatterns = loadCustomPatterns(absolutePath);
+  const allPatterns = [...SECRET_PATTERNS, ...customPatterns];
+
+  if (customPatterns.length > 0 && options.verbose) {
+    output.info(`Loaded ${customPatterns.length} custom pattern(s) from .ship-safe.json`);
+  }
+
   // Start spinner
   const spinner = ora({
     text: 'Scanning for secrets...',
@@ -50,7 +108,7 @@ export async function scanCommand(targetPath = '.', options = {}) {
 
   try {
     // Find all files
-    const files = await findFiles(absolutePath, options.verbose);
+    const files = await findFiles(absolutePath, ignorePatterns, options);
     spinner.text = `Scanning ${files.length} files...`;
 
     // Scan each file
@@ -58,7 +116,7 @@ export async function scanCommand(targetPath = '.', options = {}) {
     let scannedCount = 0;
 
     for (const file of files) {
-      const findings = await scanFile(file);
+      const findings = await scanFile(file, allPatterns);
       if (findings.length > 0) {
         results.push({ file, findings });
       }
@@ -72,7 +130,9 @@ export async function scanCommand(targetPath = '.', options = {}) {
     spinner.stop();
 
     // Output results
-    if (options.json) {
+    if (options.sarif) {
+      outputSARIF(results, absolutePath);
+    } else if (options.json) {
       outputJSON(results, files.length);
     } else {
       outputPretty(results, files.length, absolutePath);
@@ -89,48 +149,90 @@ export async function scanCommand(targetPath = '.', options = {}) {
   }
 }
 
+// =============================================================================
+// .SHIP-SAFEIGNORE LOADING
+// =============================================================================
+
+/**
+ * Load ignore patterns from .ship-safeignore file.
+ * Same syntax as .gitignore — glob patterns, one per line, # for comments.
+ */
+function loadIgnoreFile(rootPath) {
+  const ignorePath = path.join(rootPath, '.ship-safeignore');
+
+  if (!fs.existsSync(ignorePath)) return [];
+
+  try {
+    return fs.readFileSync(ignorePath, 'utf-8')
+      .split('\n')
+      .map(line => line.trim())
+      .filter(line => line && !line.startsWith('#'));
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Check if a file path matches any ignore pattern.
+ * Supports: exact paths, glob patterns, and directory prefixes.
+ */
+function isIgnoredByFile(filePath, rootPath, ignorePatterns) {
+  if (ignorePatterns.length === 0) return false;
+
+  const relPath = path.relative(rootPath, filePath).replace(/\\/g, '/');
+
+  return ignorePatterns.some(pattern => {
+    // Directory prefix match: "tests/" ignores everything under tests/
+    if (pattern.endsWith('/')) {
+      return relPath.startsWith(pattern) || relPath.includes('/' + pattern);
+    }
+    // Simple glob: "**/fixtures/**" or "src/secrets.js"
+    const escaped = pattern
+      .replace(/[.+^${}()|[\]\\]/g, '\\$&')
+      .replace(/\*/g, '[^/]*')
+      .replace(/\?/g, '[^/]');
+    return new RegExp(`(^|/)${escaped}($|/)`).test(relPath);
+  });
+}
+
 // =============================================================================
 // FILE DISCOVERY
 // =============================================================================
 
-async function findFiles(rootPath, verbose = false) {
+async function findFiles(rootPath, ignorePatterns, options = {}) {
   // Build ignore patterns from SKIP_DIRS
-  const ignorePatterns = Array.from(SKIP_DIRS).map(dir => `**/${dir}/**`);
+  const globIgnore = Array.from(SKIP_DIRS).map(dir => `**/${dir}/**`);
 
   // Find all files
   const files = await glob('**/*', {
     cwd: rootPath,
     absolute: true,
     nodir: true,
-    ignore: ignorePatterns,
-    dot: true  // Include dotfiles (but not .git which is ignored)
+    ignore: globIgnore,
+    dot: true
   });
 
-  // Filter by extension and size
   const filtered = [];
 
   for (const file of files) {
     // Skip by extension
     const ext = path.extname(file).toLowerCase();
-    if (SKIP_EXTENSIONS.has(ext)) {
-      continue;
-    }
+    if (SKIP_EXTENSIONS.has(ext)) continue;
 
     // Handle compound extensions like .min.js
     const basename = path.basename(file);
-    if (basename.endsWith('.min.js') || basename.endsWith('.min.css')) {
-      continue;
-    }
+    if (basename.endsWith('.min.js') || basename.endsWith('.min.css')) continue;
+
+    // Skip test files by default (--include-tests to override)
+    if (!options.includeTests && isTestFile(file)) continue;
+
+    // Skip files matching .ship-safeignore
+    if (isIgnoredByFile(file, rootPath, ignorePatterns)) continue;
 
     // Skip by size
     try {
       const stats = fs.statSync(file);
-      if (stats.size > MAX_FILE_SIZE) {
-        if (verbose) {
-          console.log(chalk.gray(`  Skipping (too large): ${file}`));
-        }
-        continue;
-      }
+      if (stats.size > MAX_FILE_SIZE) continue;
     } catch {
       continue;
     }
@@ -141,39 +243,53 @@ async function findFiles(rootPath, verbose = false) {
   return filtered;
 }
 
+function isTestFile(filePath) {
+  return TEST_FILE_PATTERNS.some(pattern => pattern.test(filePath));
+}
+
 // =============================================================================
 // FILE SCANNING
 // =============================================================================
 
-async function scanFile(filePath) {
+async function scanFile(filePath, patterns = SECRET_PATTERNS) {
   const findings = [];
 
   try {
     const content = fs.readFileSync(filePath, 'utf-8');
     const lines = content.split('\n');
 
-    // Check each pattern against each line
     for (let lineNum = 0; lineNum < lines.length; lineNum++) {
       const line = lines[lineNum];
 
-      for (const pattern of SECRET_PATTERNS) {
+      // Inline suppression: # ship-safe-ignore on the same line
+      if (/ship-safe-ignore/i.test(line)) continue;
+
+      for (const pattern of patterns) {
         // Reset regex state (important for global regexes)
         pattern.pattern.lastIndex = 0;
 
         let match;
         while ((match = pattern.pattern.exec(line)) !== null) {
+          // For generic patterns, apply entropy check to filter placeholders
+          if (pattern.requiresEntropyCheck && !isHighEntropyMatch(match[0])) {
+            continue;
+          }
+
+          const confidence = getConfidence(pattern, match[0]);
+
           findings.push({
             line: lineNum + 1,
             column: match.index + 1,
             matched: match[0],
             patternName: pattern.name,
             severity: pattern.severity,
+            confidence,
             description: pattern.description
           });
         }
       }
     }
-  } catch (err) {
+  } catch {
     // Skip files that can't be read (binary, permissions, etc.)
   }
 
@@ -185,7 +301,6 @@ async function scanFile(filePath) {
 // =============================================================================
 
 function outputPretty(results, filesScanned, rootPath) {
-  // Calculate stats
   const stats = {
     total: 0,
     critical: 0,
@@ -197,20 +312,19 @@ function outputPretty(results, filesScanned, rootPath) {
   for (const { findings } of results) {
     for (const f of findings) {
       stats.total++;
-      stats[f.severity]++;
+      stats[f.severity] = (stats[f.severity] || 0) + 1;
     }
   }
 
-  // Print header
   output.header('Scan Results');
 
   if (results.length === 0) {
     output.success('No secrets detected in your codebase!');
     console.log();
-    console.log(chalk.gray('Note: This scanner uses pattern matching and may miss some secrets.'));
-    console.log(chalk.gray('Consider also using: gitleaks, trufflehog, or detect-secrets'));
+    console.log(chalk.gray('Note: Uses pattern matching + entropy scoring. Test files excluded by default.'));
+    console.log(chalk.gray('Tip:  Run with --include-tests to also scan test files.'));
+    console.log(chalk.gray('Tip:  Add a .ship-safeignore file to exclude paths.'));
   } else {
-    // Print findings grouped by file
     for (const { file, findings } of results) {
       const relPath = path.relative(rootPath, file);
 
@@ -221,16 +335,20 @@ function outputPretty(results, filesScanned, rootPath) {
           f.patternName,
           f.severity,
           f.matched,
-          f.description
+          f.description,
+          f.confidence
         );
       }
     }
 
-    // Print recommendations
+    // Remind about suppressions
+    console.log();
+    console.log(chalk.gray('Suppress a finding: add  # ship-safe-ignore  as a comment on that line'));
+    console.log(chalk.gray('Exclude a path:     add it to .ship-safeignore'));
+
     output.recommendations();
   }
 
-  // Print summary
   output.summary(stats);
 }
 
@@ -250,6 +368,7 @@ function outputJSON(results, filesScanned) {
         line: f.line,
         column: f.column,
         severity: f.severity,
+        confidence: f.confidence,
         type: f.patternName,
         matched: output.maskSecret(f.matched),
         description: f.description
@@ -259,3 +378,76 @@ function outputJSON(results, filesScanned) {
 
   console.log(JSON.stringify(jsonOutput, null, 2));
 }
+
+// =============================================================================
+// SARIF OUTPUT (GitHub Code Scanning compatible)
+// =============================================================================
+
+/**
+ * Output findings in SARIF 2.1.0 format.
+ * Feed this into GitHub's Security tab:
+ *   npx ship-safe scan . --sarif > results.sarif
+ *
+ * Then upload via:
+ *   github/codeql-action/upload-sarif@v3
+ */
+function outputSARIF(results, rootPath) {
+  const rules = {};
+
+  // Build rules from findings
+  for (const { findings } of results) {
+    for (const f of findings) {
+      if (!rules[f.patternName]) {
+        rules[f.patternName] = {
+          id: f.patternName.replace(/\s+/g, '-').toLowerCase(),
+          name: f.patternName,
+          shortDescription: { text: f.patternName },
+          fullDescription: { text: f.description },
+          defaultConfiguration: {
+            level: f.severity === 'critical' ? 'error'
+              : f.severity === 'high' ? 'error'
+              : f.severity === 'medium' ? 'warning'
+              : 'note'
+          },
+          helpUri: 'https://github.com/asamassekou10/ship-safe',
+        };
+      }
+    }
+  }
+
+  const sarif = {
+    version: '2.1.0',
+    $schema: 'https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json',
+    runs: [{
+      tool: {
+        driver: {
+          name: 'ship-safe',
+          version: '2.1.0',
+          informationUri: 'https://github.com/asamassekou10/ship-safe',
+          rules: Object.values(rules),
+        }
+      },
+      results: results.flatMap(({ file, findings }) =>
+        findings.map(f => ({
+          ruleId: f.patternName.replace(/\s+/g, '-').toLowerCase(),
+          level: f.severity === 'critical' || f.severity === 'high' ? 'error' : 'warning',
+          message: { text: f.description },
+          locations: [{
+            physicalLocation: {
+              artifactLocation: {
+                uri: path.relative(rootPath, file).replace(/\\/g, '/'),
+                uriBaseId: '%SRCROOT%'
+              },
+              region: {
+                startLine: f.line,
+                startColumn: f.column,
+              }
+            }
+          }]
+        }))
+      )
+    }]
+  };
+
+  console.log(JSON.stringify(sarif, null, 2));
+}

package/cli/utils/entropy.js

@@ -0,0 +1,126 @@
+/**
+ * Shannon Entropy Scoring
+ * =======================
+ *
+ * Used to reduce false positives in secret detection.
+ *
+ * CONCEPT:
+ * Real secrets (API keys, tokens) are randomly generated and have HIGH entropy.
+ * Placeholder values like "your-api-key-here" or "example123" have LOW entropy
+ * because they follow predictable patterns or use common words.
+ *
+ * Shannon entropy measures the "randomness" of a string on a scale of 0-8.
+ * - 0:   Completely uniform  ("aaaaaaaaaaaaaaaa")
+ * - 2-3: Low entropy         ("your-api-key-here", "example_value")
+ * - 3.5+: High entropy       ("xK9mP2nQ8vL4jR7s") - likely a real secret
+ * - 5+:   Very high entropy  (random bytes, base64)
+ *
+ * We only apply entropy checks to "generic" patterns that lack specific prefixes.
+ * Patterns with known prefixes (sk-ant-, ghp_, AKIA...) are already precise enough.
+ */
+
+// =============================================================================
+// ENTROPY CALCULATION
+// =============================================================================
+
+/**
+ * Calculate Shannon entropy of a string.
+ * Returns a number between 0 and log2(charset size).
+ */
+export function shannonEntropy(str) {
+  if (!str || str.length === 0) return 0;
+
+  const freq = {};
+  for (const char of str) {
+    freq[char] = (freq[char] || 0) + 1;
+  }
+
+  return Object.values(freq).reduce((sum, count) => {
+    const p = count / str.length;
+    return sum - p * Math.log2(p);
+  }, 0);
+}
+
+// Minimum entropy to consider a match a real secret
+// Real secrets: >3.5 | Placeholders: <3.0 | Safe buffer: 3.5
+export const ENTROPY_THRESHOLD = 3.5;
+
+// Strings shorter than this are unreliable for entropy analysis
+const MIN_ENTROPY_LENGTH = 16;
+
+// =============================================================================
+// VALUE EXTRACTION
+// =============================================================================
+
+/**
+ * Extract the actual secret value from a matched string.
+ *
+ * Patterns often match the full assignment, e.g.:
+ *   apiKey = "abc123xyz..."
+ *
+ * We want to score just the value part, not the variable name,
+ * because variable names are low-entropy and would skew the score.
+ */
+function extractSecretValue(matched) {
+  // Match: = "value" or : "value" or = value
+  const assignmentMatch = matched.match(/[:=]\s*["']?([a-zA-Z0-9_\-+/=.]{12,})["']?\s*$/);
+  if (assignmentMatch) return assignmentMatch[1];
+
+  // Match: Bearer <token>
+  const bearerMatch = matched.match(/Bearer\s+([a-zA-Z0-9_\-+/=.]{12,})/i);
+  if (bearerMatch) return bearerMatch[1];
+
+  // Match: quoted value anywhere
+  const quotedMatch = matched.match(/["']([a-zA-Z0-9_\-+/=.]{12,})["']/);
+  if (quotedMatch) return quotedMatch[1];
+
+  return matched;
+}
+
+// =============================================================================
+// PUBLIC API
+// =============================================================================
+
+/**
+ * Determine if a regex match looks like a real secret based on entropy.
+ *
+ * Returns true  → keep the finding (high entropy or can't determine)
+ * Returns false → filter it out (low entropy, likely a placeholder)
+ */
+export function isHighEntropyMatch(matched) {
+  const value = extractSecretValue(matched);
+
+  // If we can't extract a meaningful value, keep the finding
+  if (!value || value.length < MIN_ENTROPY_LENGTH) return true;
+
+  // Common placeholder patterns - fast path rejection
+  const PLACEHOLDER_PATTERNS = [
+    /^(your[-_]?|my[-_]?|example[-_]?|test[-_]?|dummy[-_]?|fake[-_]?|sample[-_]?)/i,
+    /^(xxx+|yyy+|zzz+|aaa+|000+)/i,
+    /^(insert|replace|changeme|placeholder|todo|fixme)/i,
+    /([-_]here|[-_]goes|[-_]key|[-_]token|[-_]secret)$/i,
+    /^[a-z]+[-_][a-z]+[-_][a-z]+$/, // looks like-a-passphrase not a key
+  ];
+
+  if (PLACEHOLDER_PATTERNS.some(p => p.test(value))) return false;
+
+  const entropy = shannonEntropy(value);
+  return entropy >= ENTROPY_THRESHOLD;
+}
+
+/**
+ * Get a human-readable confidence label for a finding.
+ */
+export function getConfidence(pattern, matched) {
+  // Strict prefix patterns (e.g. sk-ant-, ghp_, AKIA) are always high confidence
+  if (!pattern.requiresEntropyCheck) return 'high';
+
+  const value = extractSecretValue(matched);
+  if (!value || value.length < MIN_ENTROPY_LENGTH) return 'medium';
+
+  const entropy = shannonEntropy(value);
+
+  if (entropy >= 4.5) return 'high';
+  if (entropy >= ENTROPY_THRESHOLD) return 'medium';
+  return 'low'; // Should have been filtered, but just in case
+}

package/cli/utils/output.js

@@ -78,17 +78,26 @@ export function info(text) {
   console.log(chalk.blue('\u2139 ') + text);
 }
 
+const confidenceColors = {
+  high:   chalk.red,
+  medium: chalk.yellow,
+  low:    chalk.gray,
+};
+
 /**
  * Print a finding (secret detected)
  */
-export function finding(file, line, patternName, severity, matched, description) {
+export function finding(file, line, patternName, severity, matched, description, confidence) {
   const color = severityColors[severity] || chalk.white;
   const icon = severityIcons[severity] || '';
+  const confColor = confidenceColors[confidence] || chalk.gray;
+  const confLabel = confidence ? `  ${chalk.gray('Confidence:')} ${confColor(confidence)}` : '';
 
   console.log();
   console.log(chalk.white.bold(`${file}:${line}`));
   console.log(`  ${icon}${color(`[${severity.toUpperCase()}]`)} ${chalk.white(patternName)}`);
   console.log(`  ${chalk.gray('Found:')} ${chalk.yellow(maskSecret(matched))}`);
+  if (confLabel) console.log(confLabel);
   console.log(`  ${chalk.gray('Why:')} ${description}`);
 }
 

package/cli/utils/patterns.js

@@ -484,48 +484,56 @@ export const SECRET_PATTERNS = [
   },
 
   // =========================================================================
-  // MEDIUM: Generic patterns (may have false positives)
+  // MEDIUM: Generic patterns (entropy-checked to reduce false positives)
+  // requiresEntropyCheck: true → value is scored before reporting
   // =========================================================================
   {
     name: 'Generic API Key Assignment',
     pattern: /["']?(?:api[_-]?key|apikey)["']?\s*[:=]\s*["']([a-zA-Z0-9_\-]{20,})["']/gi,
     severity: 'medium',
+    requiresEntropyCheck: true,
     description: 'Hardcoded API keys should be moved to environment variables.'
   },
   {
     name: 'Generic Secret Assignment',
     pattern: /["']?(?:secret|secret[_-]?key)["']?\s*[:=]\s*["']([a-zA-Z0-9_\-]{20,})["']/gi,
     severity: 'medium',
+    requiresEntropyCheck: true,
     description: 'Hardcoded secrets should be moved to environment variables.'
   },
   {
     name: 'Password Assignment',
     pattern: /["']?password["']?\s*[:=]\s*["']([^"']{8,})["']/gi,
     severity: 'medium',
+    requiresEntropyCheck: true,
     description: 'Hardcoded passwords are a critical vulnerability.'
   },
   {
     name: 'Database URL with Credentials',
     pattern: /(mongodb|postgres|postgresql|mysql|redis):\/\/[^:]+:[^@]+@[^\s"']+/gi,
     severity: 'medium',
+    requiresEntropyCheck: true,
     description: 'Database URLs with embedded passwords expose your database.'
   },
   {
     name: 'Bearer Token in Code',
     pattern: /["']Bearer\s+[a-zA-Z0-9_\-\.=]{20,}["']/gi,
     severity: 'medium',
+    requiresEntropyCheck: true,
     description: 'Hardcoded bearer tokens should not be in source code.'
   },
   {
     name: 'Basic Auth Header',
     pattern: /["']Basic\s+[A-Za-z0-9+/=]{20,}["']/gi,
     severity: 'medium',
+    requiresEntropyCheck: true,
     description: 'Basic auth headers contain base64-encoded credentials.'
   },
   {
     name: 'Private Key in Environment Variable',
     pattern: /PRIVATE[_-]?KEY["']?\s*[:=]\s*["']([^"']+)["']/gi,
     severity: 'high',
+    requiresEntropyCheck: true,
     description: 'Private keys should be loaded from files, not hardcoded.'
   }
 ];
@@ -584,3 +592,26 @@ export const SKIP_EXTENSIONS = new Set([
 
 // Maximum file size to scan (1MB)
 export const MAX_FILE_SIZE = 1_000_000;
+
+// =============================================================================
+// TEST FILE PATTERNS (skipped by default, override with --include-tests)
+// =============================================================================
+// Test fixtures are the #1 source of false positives. They contain fake
+// credentials, mock data, and example values that look like real secrets.
+
+export const TEST_FILE_PATTERNS = [
+  /\.test\.[jt]sx?$/,
+  /\.spec\.[jt]sx?$/,
+  /\.test\.py$/,
+  /test_[^/]+\.py$/,
+  /__tests__[/\\]/,
+  /[/\\]tests?[/\\]/,
+  /[/\\]test[/\\]/,
+  /[/\\]fixtures?[/\\]/,
+  /[/\\]mocks?[/\\]/,
+  /[/\\]__mocks__[/\\]/,
+  /[/\\]stubs?[/\\]/,
+  /[/\\]fakes?[/\\]/,
+  /\.stories\.[jt]sx?$/,   // Storybook story files
+  /\.mock\.[jt]sx?$/,
+];