gitrupt 0.1.0__py3-none-any.whl

gitrupt/scanners/secrets.py

@@ -0,0 +1,558 @@
+"""
+Secret scanner.
+
+Detects secrets in staged file content using:
+    - Curated legacy patterns (SECRET_PATTERNS) — provider-specific, high-specificity
+    - Private-key block detection (PRIVATE_KEY_RULE)
+    - Provider-specific rules (PROVIDER_RULES) — AI, payments, comms, cloud, tooling
+    - Shannon-entropy analysis with context-aware scoring
+
+Works on raw text content — language-agnostic.
+
+Secrets are NEVER logged in full. All evidence is redacted.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+import re
+from dataclasses import dataclass
+
+from gitrupt.models import Finding, ScanTarget, Severity
+from gitrupt.scanners.base import Scanner
+from gitrupt.scanners.entropy import find_entropy_secrets
+from gitrupt.scanners.secret_rules import (
+    CONTEXT_WINDOW,
+    PRIVATE_KEY_RULE,
+    PROVIDER_RULES,
+    SecretRule,
+)
+
+logger = logging.getLogger(__name__)
+
+# Maximum file size to scan for secrets (bytes). Large files are skipped.
+MAX_CONTENT_SIZE = 5 * 1024 * 1024  # 5 MB
+
+# Legacy entropy thresholds (kept for backward compatibility with earlier tests)
+ENTROPY_THRESHOLD_HIGH = 4.5
+ENTROPY_THRESHOLD_CRITICAL = 5.0
+
+
+@dataclass(frozen=True)
+class SecretPattern:
+    """A pattern for detecting a specific type of secret."""
+
+    rule_id: str
+    name: str
+    pattern: re.Pattern[str]
+    severity: Severity
+    confidence: float
+    description: str
+    recommendation: str
+    # Group index that contains the actual secret value (for redaction).
+    # 0 means use the whole match.
+    secret_group: int = 0
+
+
+def _compile(pattern: str) -> re.Pattern[str]:
+    return re.compile(pattern, re.IGNORECASE | re.MULTILINE)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Curated legacy patterns
+# ─────────────────────────────────────────────────────────────────────────────
+
+SECRET_PATTERNS: list[SecretPattern] = [
+    # ── Private Keys ─────────────────────────────────────────────────────────
+    SecretPattern(
+        rule_id="private-key-pem",
+        name="PEM Private Key",
+        pattern=_compile(r"-----BEGIN\s+(?:RSA|EC|DSA|OPENSSH|PGP|ENCRYPTED)\s+PRIVATE KEY-----"),
+        severity=Severity.CRITICAL,
+        confidence=0.99,
+        description="PEM-encoded private key detected in file content",
+        recommendation="Remove the private key and rotate/revoke it immediately. Store keys in a secure vault.",
+        secret_group=0,
+    ),
+    SecretPattern(
+        rule_id="private-key-generic",
+        name="Private Key (Generic)",
+        pattern=_compile(r"-----BEGIN\s+(?:[A-Z0-9\s_-]+)?PRIVATE KEY-----"),
+        severity=Severity.CRITICAL,
+        confidence=0.99,
+        description="PEM-encoded private key block detected",
+        recommendation="Remove the private key from this file and use a secrets manager.",
+        secret_group=0,
+    ),
+
+    # ── AWS Credentials ──────────────────────────────────────────────────────
+    SecretPattern(
+        rule_id="aws-access-key-id",
+        name="AWS Access Key ID",
+        pattern=_compile(r"\b(AKIA[0-9A-Z]{16})\b"),
+        severity=Severity.CRITICAL,
+        confidence=0.97,
+        description="AWS Access Key ID pattern detected",
+        recommendation="Revoke this AWS key immediately. Use IAM roles or environment variables.",
+        secret_group=1,
+    ),
+    SecretPattern(
+        rule_id="aws-secret-access-key",
+        name="AWS Secret Access Key",
+        pattern=_compile(
+            r"(?:aws[_\-\s]?secret[_\-\s]?(?:access[_\-\s]?)?key|aws[_\-\s]?key)"
+            r"\s*[=:\"'`]\s*([A-Za-z0-9/+=]{40})\b"
+        ),
+        severity=Severity.CRITICAL,
+        confidence=0.92,
+        description="Possible AWS Secret Access Key detected",
+        recommendation="Revoke this AWS key immediately. Use IAM roles or environment variables.",
+        secret_group=1,
+    ),
+
+    # ── GitHub Tokens ────────────────────────────────────────────────────────
+    SecretPattern(
+        rule_id="github-pat-classic",
+        name="GitHub Personal Access Token (Classic)",
+        pattern=_compile(r"\b(ghp_[A-Za-z0-9]{36})\b"),
+        severity=Severity.CRITICAL,
+        confidence=0.99,
+        description="GitHub Personal Access Token (classic) detected",
+        recommendation="Revoke this token at github.com/settings/tokens",
+        secret_group=1,
+    ),
+    SecretPattern(
+        rule_id="github-pat-fine-grained",
+        name="GitHub Fine-Grained PAT",
+        pattern=_compile(r"\b(github_pat_[A-Za-z0-9_]{82})\b"),
+        severity=Severity.CRITICAL,
+        confidence=0.99,
+        description="GitHub fine-grained personal access token detected",
+        recommendation="Revoke this token at github.com/settings/tokens",
+        secret_group=1,
+    ),
+    SecretPattern(
+        rule_id="github-oauth-token",
+        name="GitHub OAuth Token",
+        pattern=_compile(r"\b(gho_[A-Za-z0-9]{36})\b"),
+        severity=Severity.CRITICAL,
+        confidence=0.99,
+        description="GitHub OAuth token detected",
+        recommendation="Revoke this token immediately",
+        secret_group=1,
+    ),
+    SecretPattern(
+        rule_id="github-actions-token",
+        name="GitHub Actions Token",
+        pattern=_compile(r"\b(ghs_[A-Za-z0-9]{36})\b"),
+        severity=Severity.HIGH,
+        confidence=0.99,
+        description="GitHub Actions token detected",
+        recommendation="This token is short-lived but should not be committed",
+        secret_group=1,
+    ),
+
+    # ── Generic API Keys & Tokens ────────────────────────────────────────────
+    SecretPattern(
+        rule_id="generic-api-key",
+        name="Generic API Key",
+        pattern=_compile(
+            r"(?:api[_\-\s]?key|apikey|api[_\-\s]?secret|access[_\-\s]?token)"
+            r"\s*[=:\"'`]\s*([A-Za-z0-9_\-]{20,64})"
+        ),
+        severity=Severity.HIGH,
+        confidence=0.75,
+        description="Possible API key or token assignment detected",
+        recommendation="Move this value to environment variables or a secrets manager.",
+        secret_group=1,
+    ),
+
+    # ── Database URLs ────────────────────────────────────────────────────────
+    SecretPattern(
+        rule_id="database-url-with-password",
+        name="Database URL with Password",
+        pattern=_compile(
+            r"(?:postgresql|postgres|mysql|mongodb|redis|amqp|mssql|oracle)"
+            r"://[^:@\s]+:([^@\s]{6,}@[^\s\"'`]+)"
+        ),
+        severity=Severity.CRITICAL,
+        confidence=0.92,
+        description="Database connection URL with embedded password detected",
+        recommendation="Use environment variables for database credentials. Never hardcode passwords in URLs.",
+        secret_group=1,
+    ),
+
+    # ── JWT Tokens ───────────────────────────────────────────────────────────
+    SecretPattern(
+        rule_id="jwt-token",
+        name="JWT Token",
+        pattern=_compile(r"\b(eyJ[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,}\.[A-Za-z0-9_\-]{10,})\b"),
+        severity=Severity.HIGH,
+        confidence=0.88,
+        description="JSON Web Token detected",
+        recommendation="JWT tokens may contain sensitive claims. Do not commit tokens to version control.",
+        secret_group=1,
+    ),
+
+    # ── Google Credentials ───────────────────────────────────────────────────
+    SecretPattern(
+        rule_id="google-api-key",
+        name="Google API Key",
+        pattern=_compile(r"\b(AIza[0-9A-Za-z_\-]{35})\b"),
+        severity=Severity.CRITICAL,
+        confidence=0.96,
+        description="Google API key detected",
+        recommendation="Restrict this API key in the Google Cloud Console and revoke/rotate it.",
+        secret_group=1,
+    ),
+    SecretPattern(
+        rule_id="google-oauth-client-secret",
+        name="Google OAuth Client Secret",
+        pattern=_compile(r"\"client_secret\"\s*:\s*\"([A-Za-z0-9_\-]{24,})\""),
+        severity=Severity.CRITICAL,
+        confidence=0.85,
+        description="Google OAuth client secret detected",
+        recommendation="Revoke and rotate the OAuth client secret.",
+        secret_group=1,
+    ),
+
+    # ── Slack ────────────────────────────────────────────────────────────────
+    SecretPattern(
+        rule_id="slack-token",
+        name="Slack Token",
+        pattern=_compile(r"\b(xox[boaprs]-[0-9A-Za-z\-]{10,})\b"),
+        severity=Severity.CRITICAL,
+        confidence=0.97,
+        description="Slack API token detected",
+        recommendation="Revoke this token at api.slack.com/apps",
+        secret_group=1,
+    ),
+    SecretPattern(
+        rule_id="slack-webhook",
+        name="Slack Webhook URL",
+        pattern=_compile(r"https://hooks\.slack\.com/services/[A-Z0-9]+/[A-Z0-9]+/[A-Za-z0-9]+"),
+        severity=Severity.HIGH,
+        confidence=0.97,
+        description="Slack incoming webhook URL detected",
+        recommendation="Regenerate this webhook URL",
+        secret_group=0,
+    ),
+
+    # ── Stripe ───────────────────────────────────────────────────────────────
+    SecretPattern(
+        rule_id="stripe-secret-key",
+        name="Stripe Secret Key",
+        pattern=_compile(r"\b(sk_(?:live|test)_[0-9A-Za-z]{24,})\b"),
+        severity=Severity.CRITICAL,
+        confidence=0.99,
+        description="Stripe secret API key detected",
+        recommendation="Revoke this key at dashboard.stripe.com/apikeys",
+        secret_group=1,
+    ),
+
+    # ── Generic password assignments ────────────────────────────────────────
+    SecretPattern(
+        rule_id="hardcoded-password",
+        name="Hardcoded Password",
+        pattern=_compile(
+            r"(?:password|passwd|pwd)\s*[=:\"'`]\s*[\"']([^\"'\s]{8,})[\"']"
+        ),
+        severity=Severity.HIGH,
+        confidence=0.72,
+        description="Possible hardcoded password detected",
+        recommendation="Replace hardcoded passwords with environment variables or a secrets manager.",
+        secret_group=1,
+    ),
+]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Scanner
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+class SecretScanner(Scanner):
+    """
+    Scans staged file content for secrets and credentials.
+
+    Four detection passes per added line:
+        1. SECRET_PATTERNS       — curated legacy patterns (specific, low FP)
+        2. PRIVATE_KEY_RULE      — PEM private-key blocks
+        3. PROVIDER_RULES        — AI providers, payments, comms, cloud, tooling
+        4. Entropy               — Shannon entropy with context-aware scoring
+
+    All evidence is redacted before it ever reaches a Finding.
+    """
+
+    @property
+    def name(self) -> str:
+        return "secrets"
+
+    @property
+    def description(self) -> str:
+        return "Detects secrets, credentials, and API keys in file content"
+
+    def scan(self, target: ScanTarget) -> list[Finding]:
+        findings: list[Finding] = []
+
+        for staged_file in target.staged_files:
+            if staged_file.status == "D":
+                continue
+            if staged_file.is_binary:
+                continue
+            if staged_file.size_bytes > MAX_CONTENT_SIZE:
+                logger.warning(
+                    "Skipping secret scan of %s: file too large (%d bytes)",
+                    staged_file.path,
+                    staged_file.size_bytes,
+                )
+                continue
+
+            file_findings = self._scan_diff_for_file(
+                target.staged_diff, staged_file.path
+            )
+            findings.extend(file_findings)
+
+        return findings
+
+    # ── Internals ────────────────────────────────────────────────────────────
+
+    def _scan_diff_for_file(self, diff: str, file_path: str) -> list[Finding]:
+        """Extract added lines from the diff for a specific file and scan them."""
+        added_lines = _extract_added_lines(diff, file_path)
+        if not added_lines:
+            return []
+
+        findings: list[Finding] = []
+        # Dedup key is (rule_id, file_path) — one finding per rule per file,
+        # to avoid flooding when a config file contains many similar values.
+        seen: set[tuple[str, str]] = set()
+
+        for line_number, line_content in added_lines:
+
+            # ── Pass 1: curated legacy patterns ─────────────────────────────
+            for sp in SECRET_PATTERNS:
+                match = sp.pattern.search(line_content)
+                if not match:
+                    continue
+
+                key = (sp.rule_id, file_path)
+                if key in seen:
+                    continue
+
+                try:
+                    secret_value = (
+                        match.group(sp.secret_group)
+                        if sp.secret_group > 0
+                        else match.group(0)
+                    )
+                except IndexError:
+                    secret_value = match.group(0)
+
+                seen.add(key)
+                findings.append(
+                    Finding(
+                        scanner=self.name,
+                        rule_id=sp.rule_id,
+                        severity=sp.severity,
+                        confidence=sp.confidence,
+                        file=file_path,
+                        line=line_number,
+                        message=f"{sp.name} detected in {file_path}",
+                        description=sp.description,
+                        evidence=_redact(secret_value),
+                        recommendation=sp.recommendation,
+                        can_override=False,
+                    )
+                )
+
+            # ── Pass 2: private-key block ───────────────────────────────────
+            pk_match = PRIVATE_KEY_RULE.pattern.search(line_content)
+            if pk_match:
+                key = (PRIVATE_KEY_RULE.rule_id, file_path)
+                if key not in seen:
+                    seen.add(key)
+                    findings.append(
+                        _rule_to_finding(
+                            PRIVATE_KEY_RULE,
+                            file_path,
+                            line_number,
+                            line_content,
+                            match=pk_match,
+                        )
+                    )
+
+            # ── Pass 3: provider-specific rules ─────────────────────────────
+            for rule in PROVIDER_RULES:
+                match = rule.pattern.search(line_content)
+                if not match:
+                    continue
+                if not _context_ok(line_content, match, rule):
+                    continue
+
+                key = (rule.rule_id, file_path)
+                if key in seen:
+                    continue
+
+                seen.add(key)
+                findings.append(
+                    _rule_to_finding(
+                        rule,
+                        file_path,
+                        line_number,
+                        line_content,
+                        match=match,
+                    )
+                )
+
+            # ── Pass 4: entropy ─────────────────────────────────────────────
+            for hit in find_entropy_secrets(line_content):
+                key = ("generic-high-entropy", file_path)
+                if key in seen:
+                    continue
+
+                seen.add(key)
+                findings.append(
+                    Finding(
+                        scanner=self.name,
+                        rule_id="generic-high-entropy",
+                        severity=Severity.MEDIUM,
+                        confidence=(
+                            0.7 if hit.reason == "context-assisted" else 0.55
+                        ),
+                        file=file_path,
+                        line=line_number,
+                        message="High-entropy string — possible secret",
+                        description=(
+                            f"Entropy {hit.entropy:.2f} bits/char ({hit.reason}). "
+                            "May be an unrecognised API key or token."
+                        ),
+                        evidence=_redact(hit.value),
+                        recommendation=(
+                            "Verify whether this is a secret. If it is, move it "
+                            "to an environment variable and add the shape to the "
+                            ".gitrupt.yml allowlist."
+                        ),
+                        can_override=True,
+                    )
+                )
+
+        return findings
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Helpers
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def _context_ok(line: str, match: re.Match[str], rule: SecretRule) -> bool:
+    """
+    True if the rule has no context requirement, or if one of its context
+    words appears within CONTEXT_WINDOW characters of the match.
+    """
+    if not rule.context_words:
+        return True
+    start = max(0, match.start() - CONTEXT_WINDOW)
+    end = min(len(line), match.end() + CONTEXT_WINDOW)
+    window = line[start:end].lower()
+    return any(w.lower() in window for w in rule.context_words)
+
+
+def _rule_to_finding(
+    rule: SecretRule,
+    file_path: str,
+    line_number: int,
+    line_content: str,
+    match: re.Match[str] | None = None,
+) -> Finding:
+    """Convert a SecretRule match into a Finding with redacted evidence."""
+    if match is None:
+        match = rule.pattern.search(line_content)
+    raw_value = match.group(0) if match else line_content
+    return Finding(
+        scanner="secrets",
+        rule_id=rule.rule_id,
+        severity=rule.severity,
+        confidence=rule.confidence,
+        file=file_path,
+        line=line_number,
+        message=rule.message,
+        description=rule.description,
+        evidence=_redact(raw_value),
+        recommendation=rule.recommendation,
+        can_override=False,
+    )
+
+
+def _extract_added_lines(diff: str, file_path: str) -> list[tuple[int, str]]:
+    """
+    Extract lines that were added (starting with '+') for a specific file
+    from a unified diff.
+
+    Returns a list of (line_number, content) tuples.
+    """
+    if not diff:
+        return []
+
+    lines: list[tuple[int, str]] = []
+    in_target_file = False
+    current_line = 0
+
+    for raw_line in diff.splitlines():
+        if raw_line.startswith("diff --git "):
+            in_target_file = (
+                f"b/{file_path}" in raw_line or raw_line.endswith(file_path)
+            )
+            current_line = 0
+            continue
+
+        if not in_target_file:
+            continue
+
+        if raw_line.startswith("@@"):
+            match = re.search(r"\+(\d+)(?:,\d+)?", raw_line)
+            if match:
+                current_line = int(match.group(1)) - 1
+            continue
+
+        if raw_line.startswith("+") and not raw_line.startswith("+++"):
+            current_line += 1
+            lines.append((current_line, raw_line[1:]))
+        elif raw_line.startswith(" "):
+            current_line += 1
+
+    return lines
+
+
+def _redact(secret: str) -> str:
+    """
+    Redact a secret value for safe display.
+
+    Shows the first few characters followed by bullets.
+    Never shows the full value.
+    """
+    if not secret:
+        return "••••••••"
+
+    visible = min(4, len(secret) // 4)
+    hidden = len(secret) - visible
+    return secret[:visible] + "•" * min(hidden, 20)
+
+
+def shannon_entropy(data: str) -> float:
+    """Calculate the Shannon entropy of a string."""
+    if not data:
+        return 0.0
+
+    freq: dict[str, int] = {}
+    for char in data:
+        freq[char] = freq.get(char, 0) + 1
+
+    entropy = 0.0
+    length = len(data)
+    for count in freq.values():
+        probability = count / length
+        entropy -= probability * math.log2(probability)
+
+    return entropy