thuban 0.3.1 → 0.3.3

package/packages/scanner/secret-scanner.js

@@ -0,0 +1,612 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+const EventEmitter = require('events');
+const { execFileSync } = require('child_process');
+
+class SecretScanner extends EventEmitter {
+  constructor(config = {}) {
+    super();
+
+    this.config = {
+      rootPath: config.rootPath || process.cwd(),
+      ignorePatterns: config.ignorePatterns || [
+        'node_modules/**',
+        '.git/**',
+        'dist/**',
+        '.next/**',
+        'coverage/**'
+      ],
+      maxFileSize: config.maxFileSize || 1024 * 1024,
+      entropyThreshold: config.entropyThreshold || 4.2,
+      minEntropyLength: config.minEntropyLength || 20,
+      includeGitHistory: config.includeGitHistory !== false,
+      ...config
+    };
+
+    this.scanExtensions = new Set([
+      '.env',
+      '.js',
+      '.jsx',
+      '.ts',
+      '.tsx',
+      '.py',
+      '.json',
+      '.yaml',
+      '.yml',
+      '.toml',
+      '.ini',
+      '.conf',
+      '.config',
+      '.properties',
+      '.sh',
+      '.bash',
+      '.zsh'
+    ]);
+
+    this.ignoredDirectories = [
+      'node_modules',
+      '.git',
+      'dist',
+      '.next',
+      'coverage',
+      '.venv',
+      'venv',
+      '__pycache__'
+    ];
+
+    this.safeLinePatterns = [
+      /process\.env\./i,
+      /process\.env\[/i,
+      /os\.environ/i,
+      /sample/i,
+      /placeholder/i,
+      /dummy/i,
+      /mock/i,
+      /example/i,
+      /replace[_-\s]?me/i,
+      /replace[_-\s]?with/i,
+      /local-dev-token/i,
+      /changeme/i,
+      /your[_-\s]?key/i,
+      /your[_-\s]?(secret|token|password)/i,
+      /<secret>/i,
+      /<token>/i,
+      /x{8,}/i
+    ];
+
+    this.secretPatterns = [
+      {
+        id: 'SECRET001',
+        type: 'Stripe Secret Key',
+        severity: 'critical',
+        recommendation: 'Move this key to a cloud secret manager and load it via environment variables.',
+        pattern: /\bsk_(live|test)_[0-9a-zA-Z]{16,}\b/gi
+      },
+      {
+        id: 'SECRET002',
+        type: 'Stripe Publishable Key',
+        severity: 'high',
+        recommendation: 'Verify this publishable key is intended to be public; if not, move it to managed secrets.',
+        pattern: /\bpk_(live|test)_[0-9a-zA-Z]{16,}\b/gi
+      },
+      {
+        id: 'SECRET003',
+        type: 'AWS Access Key',
+        severity: 'critical',
+        recommendation: 'Rotate the AWS credential immediately and store replacements in a cloud secret manager.',
+        pattern: /\b(AKIA|ASIA|AGPA|AIDA|AROA|AIPA)[A-Z0-9]{16}\b/g
+      },
+      {
+        id: 'SECRET004',
+        type: 'GitHub Token',
+        severity: 'critical',
+        recommendation: 'Revoke the GitHub token and replace it with a managed secret or GitHub App credential.',
+        pattern: /\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{20,}\b/g
+      },
+      {
+        id: 'SECRET005',
+        type: 'Slack Token',
+        severity: 'critical',
+        recommendation: 'Rotate the Slack token and move it into a cloud secret manager.',
+        pattern: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g
+      },
+      {
+        id: 'SECRET006',
+        type: 'Twilio Credential',
+        severity: 'critical',
+        recommendation: 'Rotate the Twilio credential and store it in managed secrets.',
+        pattern: /\b(SK|AC)[0-9a-fA-F]{32}\b/g
+      },
+      {
+        id: 'SECRET007',
+        type: 'Google API Key',
+        severity: 'critical',
+        recommendation: 'Restrict and rotate the Google API key, then move it to a cloud secret manager.',
+        pattern: /\bAIza[0-9A-Za-z\-_]{35}\b/g
+      },
+      {
+        id: 'SECRET007B',
+        type: 'OpenAI Project Key',
+        severity: 'critical',
+        recommendation: 'Rotate the OpenAI project key and move it to a cloud secret manager.',
+        pattern: /\bsk-proj-[A-Za-z0-9\-_]{20,}\b/g
+      },
+      {
+        id: 'SECRET007C',
+        type: 'Cartesia API Key',
+        severity: 'critical',
+        recommendation: 'Rotate the Cartesia API key and move it to a cloud secret manager.',
+        pattern: /\bsk_car_[A-Za-z0-9]{16,}\b/g
+      },
+      {
+        id: 'SECRET008',
+        type: 'JWT Token',
+        severity: 'high',
+        recommendation: 'Do not hardcode JWTs; issue them dynamically and store signing secrets in managed secrets.',
+        pattern: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9._-]{8,}\.[A-Za-z0-9._-]{8,}\b/g
+      },
+      {
+        id: 'SECRET009',
+        type: 'Bearer Token',
+        severity: 'high',
+        recommendation: 'Move bearer tokens to a cloud secret manager and inject them at runtime.',
+        pattern: /\bBearer\s+[A-Za-z0-9\-._~+/]+=*\b/g
+      },
+      {
+        id: 'SECRET010',
+        type: 'Private Key',
+        severity: 'critical',
+        recommendation: 'Remove the private key from source control, rotate it, and store it in a secret manager.',
+        pattern: /-----BEGIN (RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY-----/g
+      },
+      {
+        id: 'SECRET011',
+        type: 'Database Connection String with Password',
+        severity: 'critical',
+        recommendation: 'Move the connection string to managed secrets and avoid embedding passwords in code or config.',
+        pattern: /\b(?:postgres(?:ql)?|mysql|mssql|mongodb(?:\+srv)?|redis):\/\/[^:\s]+:[^@\s]+@[^'"`\s]+\b/gi
+      },
+      {
+        id: 'SECRET012',
+        type: 'Webhook Secret',
+        severity: 'high',
+        recommendation: 'Store webhook secrets in a cloud secret manager and inject them via environment variables.',
+        pattern: /\b(?:whsec_[A-Za-z0-9]+|webhook[_-]?secret[_:=\s'"]+[A-Za-z0-9_\-]{12,})\b/gi
+      },
+      {
+        id: 'SECRET013',
+        type: 'Hardcoded Password or Secret Assignment',
+        severity: 'high',
+        recommendation: 'Replace hardcoded credentials with environment variables backed by a cloud secret manager.',
+        pattern: /\b(?:password|passwd|pwd|secret|client_secret|api[_-]?secret|signing[_-]?secret)\b\s*[:=]\s*['"`][^'"`\s][^'"`]{5,}['"`]/gi
+      },
+      {
+        id: 'SECRET014',
+        type: 'Hardcoded API Key Assignment',
+        severity: 'high',
+        recommendation: 'Move API keys to a cloud secret manager and reference them through environment variables.',
+        pattern: /\b(?:api[_-]?key|access[_-]?token|auth[_-]?token|token)\b\s*[:=]\s*['"`][A-Za-z0-9_\-./+=]{12,}['"`]/gi
+      },
+      {
+        id: 'SECRET015',
+        type: '.env File Present',
+        severity: 'medium',
+        recommendation: 'Ensure .env files are excluded from git and move production secrets to a cloud secret manager.',
+        filePattern: /(^|[\\/])\.env(\.[^\\/]+)?$/i
+      },
+      {
+        id: 'SECRET016',
+        type: 'LiveKit API Secret',
+        severity: 'critical',
+        recommendation: 'Rotate the LiveKit API secret and store it in managed secrets.',
+        pattern: /\b(?:LIVEKIT_API_SECRET|livekit[_-]?api[_-]?secret)\b\s*[:=]\s*['"`]?([A-Za-z0-9]{24,})['"`]?/gi
+      }
+    ];
+  }
+
+  async scanAll() {
+    const startTime = Date.now();
+    const files = await this._discoverFiles(this.config.rootPath);
+    const issues = [];
+
+    for (const filePath of files) {
+      issues.push(...await this.scanFile(filePath));
+    }
+
+    if (this.config.includeGitHistory) {
+      issues.push(...this.scanGitHistory());
+    }
+
+    return {
+      filesScanned: files.length,
+      issues,
+      summary: this._summarizeIssues(issues),
+      duration: Date.now() - startTime
+    };
+  }
+
+  async scanFiles(filePaths) {
+    const issues = [];
+    for (const filePath of filePaths) {
+      issues.push(...await this.scanFile(filePath));
+    }
+
+    if (this.config.includeGitHistory) {
+      issues.push(...this.scanGitHistory());
+    }
+
+    return {
+      filesScanned: filePaths.length,
+      issues,
+      summary: this._summarizeIssues(issues)
+    };
+  }
+
+  async scanFile(filePath) {
+    const issues = [];
+    const relativePath = path.relative(this.config.rootPath, filePath).replace(/\\/g, '/');
+
+    try {
+      const stats = fs.statSync(filePath);
+      if (stats.size > this.config.maxFileSize) {
+        return issues;
+      }
+
+      const fileName = path.basename(filePath);
+      const content = fs.readFileSync(filePath, 'utf8');
+      const lines = content.split(/\r?\n/);
+
+      for (const pattern of this.secretPatterns) {
+        if (pattern.filePattern && pattern.filePattern.test(filePath)) {
+          if (this._shouldSkipLine(fileName, relativePath, pattern)) {
+            continue;
+          }
+
+          if (!this._isGitIgnored(relativePath)) {
+            issues.push(this._createIssue({
+              file: filePath,
+              line: 1,
+              type: pattern.type,
+              severity: pattern.severity,
+              recommendation: 'Add this file to .gitignore and move any production secrets to a cloud secret manager.',
+              id: pattern.id,
+              match: fileName,
+              category: 'secret'
+            }));
+          }
+          continue;
+        }
+
+        if (!pattern.pattern) {
+          continue;
+        }
+
+        for (let index = 0; index < lines.length; index++) {
+          const line = lines[index];
+          if (this._shouldSkipLine(line, relativePath, pattern)) {
+            continue;
+          }
+
+          pattern.pattern.lastIndex = 0;
+          let match;
+          while ((match = pattern.pattern.exec(line)) !== null) {
+            const matchedValue = match[1] || match[0];
+            if (this._isLikelyPlaceholder(matchedValue)) {
+              continue;
+            }
+
+            issues.push(this._createIssue({
+              file: filePath,
+              line: index + 1,
+              column: match.index,
+              type: pattern.type,
+              severity: pattern.severity,
+              recommendation: pattern.recommendation,
+              id: pattern.id,
+              match: match[0],
+              category: 'secret'
+            }));
+          }
+        }
+      }
+
+      issues.push(...this._scanEntropy(filePath, lines, relativePath));
+      return this._dedupeIssues(issues);
+    } catch (error) {
+      return [{
+        file: filePath,
+        line: 1,
+        category: 'secret',
+        severity: 'medium',
+        id: 'SECRET_SCAN_ERROR',
+        type: 'Secret Scan Error',
+        recommendation: 'Verify file permissions and encoding, then rerun the scan.',
+        message: `Failed to scan file for secrets: ${error.message}`
+      }];
+    }
+  }
+
+  scanGitHistory() {
+    try {
+      const output = execFileSync(
+        'git',
+        ['log', '-p', '--all', '--full-history', '--', '.'],
+        { cwd: this.config.rootPath, encoding: 'utf8', maxBuffer: 20 * 1024 * 1024 }
+      );
+
+      const issues = [];
+      const lines = output.split(/\r?\n/);
+      let currentCommit = null;
+      let currentFile = null;
+
+      for (const line of lines) {
+        if (line.startsWith('commit ')) {
+          currentCommit = line.replace('commit ', '').trim();
+          continue;
+        }
+
+        if (line.startsWith('+++ b/')) {
+          currentFile = line.replace('+++ b/', '').trim();
+          continue;
+        }
+
+        if (!line.startsWith('+') || line.startsWith('+++')) {
+          continue;
+        }
+
+        const addedLine = line.slice(1);
+        if (this._shouldSkipLine(addedLine, currentFile || '')) {
+          continue;
+        }
+
+        for (const pattern of this.secretPatterns) {
+          if (!pattern.pattern) {
+            continue;
+          }
+
+          pattern.pattern.lastIndex = 0;
+          let match;
+          while ((match = pattern.pattern.exec(addedLine)) !== null) {
+            const matchedValue = match[1] || match[0];
+            if (this._isLikelyPlaceholder(matchedValue)) {
+              continue;
+            }
+
+            issues.push({
+              file: currentFile || 'unknown',
+              line: null,
+              category: 'secret-history',
+              severity: pattern.severity === 'medium' ? 'high' : pattern.severity,
+              id: `${pattern.id}_HISTORY`,
+              type: `${pattern.type} in Git History`,
+              recommendation: 'Purge the secret from git history, rotate it, and move it to a cloud secret manager.',
+              message: `Potential secret remains in git history (${currentCommit || 'unknown commit'})`,
+              commit: currentCommit,
+              code: addedLine.trim().slice(0, 140)
+            });
+          }
+        }
+      }
+
+      return this._dedupeIssues(issues);
+    } catch (error) {
+      return [];
+    }
+  }
+
+  _scanEntropy(filePath, lines, relativePath) {
+    const issues = [];
+    const candidatePattern = /['"`]([A-Za-z0-9+/=_\-]{20,})['"`]/g;
+
+    for (let index = 0; index < lines.length; index++) {
+      const line = lines[index];
+      if (this._shouldSkipLine(line, relativePath)) {
+        continue;
+      }
+
+      candidatePattern.lastIndex = 0;
+      let match;
+      while ((match = candidatePattern.exec(line)) !== null) {
+        const candidate = match[1];
+        if (candidate.length < this.config.minEntropyLength) {
+          continue;
+        }
+
+        if (!/[A-Z]/.test(candidate) || !/[a-z]/.test(candidate) || !/[0-9]/.test(candidate)) {
+          continue;
+        }
+
+        const entropy = this._calculateEntropy(candidate);
+        if (entropy >= this.config.entropyThreshold) {
+          issues.push(this._createIssue({
+            file: filePath,
+            line: index + 1,
+            column: match.index,
+            type: 'High Entropy Secret Candidate',
+            severity: 'medium',
+            recommendation: 'Review this value, move it to a cloud secret manager if it is a credential, and reference it via environment variables.',
+            id: 'SECRET_ENTROPY',
+            match: candidate,
+            category: 'secret',
+            extraMessage: `High-entropy string detected (entropy ${entropy.toFixed(2)})`
+          }));
+        }
+      }
+    }
+
+    return issues;
+  }
+
+  _createIssue({ file, line, column, type, severity, recommendation, id, match, category, extraMessage }) {
+    return {
+      file,
+      line,
+      column,
+      category,
+      severity,
+      id,
+      type,
+      recommendation,
+      message: extraMessage || `${type} detected`,
+      code: typeof match === 'string' ? match.slice(0, 140) : ''
+    };
+  }
+
+  async _discoverFiles(dir, files = []) {
+    let entries = [];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return files;
+    }
+
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      const relativePath = path.relative(this.config.rootPath, fullPath).replace(/\\/g, '/');
+
+      if (this._shouldIgnore(relativePath)) {
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        await this._discoverFiles(fullPath, files);
+        continue;
+      }
+
+      if (!entry.isFile()) {
+        continue;
+      }
+
+      const ext = path.extname(entry.name).toLowerCase();
+      if (this.scanExtensions.has(ext) || entry.name.startsWith('.env')) {
+        files.push(fullPath);
+      }
+    }
+
+    return files;
+  }
+
+  _shouldIgnore(relativePath) {
+    const normalized = relativePath.replace(/\\/g, '/');
+    const alwaysIgnore = this.ignoredDirectories;
+
+    for (const dir of alwaysIgnore) {
+      if (normalized === dir || normalized.startsWith(`${dir}/`) || normalized.includes(`/${dir}/`)) {
+        return true;
+      }
+    }
+
+    return this.config.ignorePatterns.some((pattern) => {
+      const normalizedPattern = pattern.replace(/\\/g, '/');
+      const regexPattern = normalizedPattern
+        .replace(/\./g, '\\.')
+        .replace(/\*\*/g, '.*')
+        .replace(/\*/g, '[^/]*');
+      return new RegExp(`^${regexPattern}$`).test(normalized) || normalized.startsWith(normalizedPattern.replace('/**', ''));
+    });
+  }
+
+  _shouldSkipLine(line, relativePath, pattern = null) {
+    if (!line || !line.trim()) {
+      return true;
+    }
+
+    const normalizedPath = (relativePath || '').replace(/\\/g, '/').toLowerCase();
+    if (
+      normalizedPath.includes('__tests__') ||
+      normalizedPath.includes('/test/') ||
+      normalizedPath.includes('/tests/') ||
+      normalizedPath.endsWith('.spec.js') ||
+      normalizedPath.endsWith('.test.js') ||
+      normalizedPath.endsWith('.spec.ts') ||
+      normalizedPath.endsWith('.test.ts') ||
+      normalizedPath.endsWith('.example') ||
+      normalizedPath.endsWith('.env.example')
+    ) {
+      return true;
+    }
+
+    if (pattern && pattern.id === 'SECRET015') {
+      return false;
+    }
+
+    return this.safeLinePatterns.some((safePattern) => safePattern.test(line));
+  }
+
+  _calculateEntropy(value) {
+    const counts = new Map();
+    for (const char of value) {
+      counts.set(char, (counts.get(char) || 0) + 1);
+    }
+
+    let entropy = 0;
+    for (const count of counts.values()) {
+      const probability = count / value.length;
+      entropy -= probability * Math.log2(probability);
+    }
+    return entropy;
+  }
+
+  _isLikelyPlaceholder(value) {
+    if (!value) {
+      return false;
+    }
+
+    return [
+      /replace[_-\s]?me/i,
+      /replace[_-\s]?with/i,
+      /example/i,
+      /dummy/i,
+      /placeholder/i,
+      /changeme/i,
+      /your[_-\s]?(key|secret|token|password)/i,
+      /local-dev-token/i,
+      /x{8,}/i
+    ].some((pattern) => pattern.test(value));
+  }
+
+  _isGitIgnored(relativePath) {
+    try {
+      execFileSync('git', ['check-ignore', relativePath], {
+        cwd: this.config.rootPath,
+        stdio: 'ignore'
+      });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  _dedupeIssues(issues) {
+    const seen = new Set();
+    return issues.filter((issue) => {
+      const hash = crypto
+        .createHash('sha1')
+        .update(JSON.stringify([issue.file, issue.line, issue.id, issue.code, issue.commit || '']))
+        .digest('hex');
+      if (seen.has(hash)) {
+        return false;
+      }
+      seen.add(hash);
+      return true;
+    });
+  }
+
+  _summarizeIssues(issues) {
+    return {
+      total: issues.length,
+      bySeverity: {
+        critical: issues.filter((issue) => issue.severity === 'critical').length,
+        high: issues.filter((issue) => issue.severity === 'high').length,
+        medium: issues.filter((issue) => issue.severity === 'medium').length
+      },
+      byType: issues.reduce((accumulator, issue) => {
+        accumulator[issue.type] = (accumulator[issue.type] || 0) + 1;
+        return accumulator;
+      }, {})
+    };
+  }
+}
+
+module.exports = SecretScanner;