pr-analyzer-cli 0.1.0__py3-none-any.whl

pr_analyzer/security/scanner.py

@@ -0,0 +1,272 @@
+import re
+from collections import Counter
+from collections.abc import Collection
+from typing import Any
+
+import structlog
+
+from pr_analyzer.analyzer.ast_analyzer import CODE_EXTENSIONS, file_extension
+from pr_analyzer.analyzer.line_lexer import split_line
+from pr_analyzer.diff import iter_added_lines
+from pr_analyzer.models import FileChange, Suggestion
+from pr_analyzer.security.dep_auditor import DEPENDENCY_AUDIT_RULE, DepAuditScanner
+from pr_analyzer.security.semgrep_runner import SEMGREP_RULE, SemgrepRunner
+
+log = structlog.get_logger(__name__)
+
+# Every pattern runs on one added line at a time, so a match can't span lines and its line
+# number is exact. Secrets are checked in every file, comments included. Each "id" is the stable
+# name .pr-analyzer.toml uses to switch the pattern off.
+SECRET_PATTERNS: list[dict[str, Any]] = [
+    {
+        "id": "aws-access-key",
+        "regex": re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
+        "severity": "CRITICAL",
+        "category": "security",
+        "title": "AWS access key exposed",
+        "description": "An AWS access key was found hardcoded. Revoke it immediately and use environment variables.",
+    },
+    {
+        "id": "hardcoded-api-key",
+        "regex": re.compile(r"(?i)(api[_-]?key|apikey|api[_-]?secret)\s*[:=]\s*[\"']([A-Za-z0-9_\-]{16,})[\"']"),
+        "value_group": 2,
+        "severity": "CRITICAL",
+        "category": "security",
+        "title": "Hardcoded API key",
+        "description": "An API key was found hardcoded. Move it to environment variables.",
+    },
+    {
+        "id": "private-key",
+        "regex": re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----"),
+        "severity": "CRITICAL",
+        "category": "security",
+        "title": "Private key exposed",
+        "description": "A private PEM key block was found. Remove it immediately and rotate the key.",
+    },
+    {
+        "id": "github-token",
+        "regex": re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{22,})\b"),
+        "severity": "CRITICAL",
+        "category": "security",
+        "title": "GitHub personal access token exposed",
+        "description": "A GitHub token was found hardcoded. Revoke it immediately at github.com/settings/tokens.",
+    },
+    {
+        "id": "jwt-token",
+        "regex": re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}"),
+        "severity": "HIGH",
+        "category": "security",
+        "title": "JWT token hardcoded",
+        "description": "A JWT token was found hardcoded. Tokens should never be committed to source control.",
+    },
+    {
+        "id": "hardcoded-password",
+        "regex": re.compile(r"(?i)(?:password|passwd|pwd)\s*[:=]\s*[\"']([^\"']{4,})[\"']"),
+        "value_group": 1,
+        "severity": "CRITICAL",
+        "category": "security",
+        "title": "Hardcoded password",
+        "description": "A password was found hardcoded. Move it to environment variables.",
+    },
+    {
+        "id": "db-connection-string",
+        "regex": re.compile(
+            r"(?i)\b(?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|rediss?|amqps?)://"
+            r"[^\s:/@\"']+:[^\s@/\"']+@"
+        ),
+        "severity": "CRITICAL",
+        "category": "security",
+        "title": "Database connection string with credentials",
+        "description": "A database URL with embedded credentials was found. Use environment variables instead.",
+    },
+]
+
+# Checked only in source files, and only on code: a call named in a comment or written inside a
+# string literal isn't a call, though one in an f-string's {...} or a template literal's ${...}
+# is. String quotes stay visible (see line_lexer), so the SQL rule still sees a string being
+# formatted into a query.
+DANGEROUS_PATTERNS: list[dict[str, Any]] = [
+    {
+        "id": "sql-injection",
+        # an f-string, or a string literal followed by %, .format( or + ; parameterised calls such
+        # as execute("... %s", (value,)) are the safe form and are not flagged
+        "regex": re.compile(
+            r"\bexecute(?:many)?\s*\(\s*"
+            r"(?:[rRbBuU]?[fF][rR]?[\"']|[\"'][^\"']*[\"']\s*(?:%|\.format\s*\(|\+))"
+        ),
+        "severity": "HIGH",
+        "category": "security",
+        "title": "Possible SQL injection",
+        "description": "SQL query built with string formatting. Use parameterised queries instead.",
+    },
+    {
+        "id": "eval",
+        "regex": re.compile(r"(?<![\w.])eval\s*\("),   # not model.eval() or my_eval()
+        "severity": "HIGH",
+        "category": "security",
+        "title": "eval() usage detected",
+        "description": "eval() executes arbitrary code. Never use it on user input.",
+    },
+    {
+        "id": "pickle-load",
+        "regex": re.compile(r"\bpickle\.loads?\s*\("),
+        "severity": "HIGH",
+        "category": "security",
+        "title": "pickle.load() usage detected",
+        "description": "pickle.load() can execute arbitrary code when loading untrusted data. Use JSON instead.",
+    },
+    {
+        "id": "yaml-load",
+        "regex": re.compile(
+            r"\byaml\.(?:unsafe_load\s*\(|load\s*\((?![^)]*Loader\s*=\s*(?:yaml\.)?C?SafeLoader\b))"
+        ),
+        "severity": "HIGH",
+        "category": "security",
+        "title": "yaml.load() without SafeLoader",
+        "description": "yaml.load() is unsafe. Use yaml.safe_load() or pass Loader=yaml.SafeLoader.",
+    },
+    {
+        "id": "subprocess-shell",
+        "regex": re.compile(r"\bsubprocess\.(?:run|call|check_call|check_output|Popen)\s*\(.*\bshell\s*=\s*True"),
+        "severity": "HIGH",
+        "category": "security",
+        "title": "subprocess with shell=True",
+        "description": "shell=True with user input allows command injection. Pass a list of arguments instead.",
+    },
+    {
+        "id": "insecure-random",
+        # only when the value looks security-related; random is fine for jitter or shuffling
+        "regex": re.compile(
+            r"(?i)\b\w*(?:token|passw|secret|otp|nonce|salt|session|api_?key)\w*\s*=.*"
+            r"\brandom\.(?:random|randint|choice|choices|randrange|getrandbits)\s*\("
+        ),
+        "severity": "MEDIUM",
+        "category": "security",
+        "title": "Non-cryptographic random used",
+        "description": "random module is not cryptographically secure. Use secrets module for tokens and passwords.",
+    },
+    {
+        "id": "debug-mode",
+        "regex": re.compile(r"(?i)\bdebug\s*=\s*True\b"),
+        "severity": "MEDIUM",
+        "category": "security",
+        "title": "Debug mode enabled",
+        "description": "debug=True should never be committed. It exposes stack traces and sensitive data in production.",
+    },
+]
+
+# obvious placeholders in examples and docs are not leaked secrets
+PLACEHOLDER = re.compile(
+    r"(?i)^(?:x+|\*+|\.+|<[^>]*>|\$\{[^}]*\}|\{\{?[^}]*\}\}?|%\([^)]*\)s|"
+    r".*(?:changeme|change_me|example|placeholder|dummy|redacted|"
+    r"your[_-]?(?:api[_-]?)?(?:key|token|password|secret)).*)$"
+)
+
+RULE_IDS = tuple(p["id"] for p in SECRET_PATTERNS + DANGEROUS_PATTERNS) + (
+    SEMGREP_RULE, DEPENDENCY_AUDIT_RULE,
+)
+
+
+def deduplicate(suggestions: list[Suggestion]) -> list[Suggestion]:
+    seen = set()
+    unique = []
+    for s in suggestions:
+        key = (s.filename, s.line, s.title, s.description)
+        if key not in seen:
+            seen.add(key)
+            unique.append(s)
+    return unique
+
+
+class SecurityScanner:
+
+    def __init__(self, semgrep: SemgrepRunner | None = None, dep_auditor: DepAuditScanner | None = None):
+        self.semgrep = semgrep or SemgrepRunner()
+        self.dep_auditor = dep_auditor or DepAuditScanner()
+        self.warnings: list[str] = []
+
+    def scan(self, files: list[FileChange], disabled: Collection[str] = ()) -> list[Suggestion]:
+        """Check the files. `disabled` holds rule ids (see RULE_IDS) to skip."""
+        secrets = [p for p in SECRET_PATTERNS if p["id"] not in disabled]
+        dangerous = [p for p in DANGEROUS_PATTERNS if p["id"] not in disabled]
+        suggestions = []
+        for file in files:
+            if file.status == "removed" or not file.patch:
+                continue
+            # works from the patch alone, so files whose full content couldn't be fetched are
+            # still checked
+            suggestions.extend(self.scan_secrets(file, secrets))
+            suggestions.extend(self.scan_dangerous(file, dangerous))
+        suggestions.extend(self._run_tools(files, disabled))
+
+        unique = deduplicate(suggestions)
+        counts = Counter(s.severity for s in unique)
+        log.info(
+            "security scan complete",
+            total=len(unique),
+            critical=counts["CRITICAL"],
+            high=counts["HIGH"],
+            medium=counts["MEDIUM"],
+            low=counts["LOW"],
+        )
+        return unique
+
+    def _run_tools(self, files: list[FileChange], disabled: Collection[str]) -> list[Suggestion]:
+        # the external tools run once per PR, not once per file
+        suggestions: list[Suggestion] = []
+        self.warnings = []
+        if SEMGREP_RULE not in disabled:
+            suggestions.extend(self.semgrep.scan(files))
+            self.warnings += self.semgrep.warnings
+        if DEPENDENCY_AUDIT_RULE not in disabled:
+            suggestions.extend(self.dep_auditor.scan(files))
+            self.warnings += self.dep_auditor.warnings
+        return suggestions
+
+    def scan_secrets(
+        self, file: FileChange, patterns: list[dict[str, Any]] = SECRET_PATTERNS
+    ) -> list[Suggestion]:
+        if file.patch is None:
+            return []
+
+        suggestions = []
+        for line_no, text in iter_added_lines(file.patch):
+            for pattern in patterns:
+                match = pattern["regex"].search(text)
+                if match is None:
+                    continue
+                group = pattern.get("value_group")
+                if group is not None and PLACEHOLDER.match(match.group(group)):
+                    continue
+                suggestions.append(self._suggestion(file, pattern, line_no))
+
+        return suggestions
+
+    def scan_dangerous(
+        self, file: FileChange, patterns: list[dict[str, Any]] = DANGEROUS_PATTERNS
+    ) -> list[Suggestion]:
+        extension = file_extension(file.filename)
+        if file.patch is None or extension not in CODE_EXTENSIONS:
+            return []
+
+        suggestions = []
+        for line_no, text in iter_added_lines(file.patch):
+            code = split_line(text, extension).code
+            for pattern in patterns:
+                if pattern["regex"].search(code):
+                    suggestions.append(self._suggestion(file, pattern, line_no))
+
+        return suggestions
+
+    @staticmethod
+    def _suggestion(file: FileChange, pattern: dict, line_no: int) -> Suggestion:
+        return Suggestion(
+            filename=file.filename,
+            severity=pattern["severity"],
+            category=pattern["category"],
+            title=pattern["title"],
+            description=pattern["description"],
+            line=line_no,
+            source="security",
+            rule_id=pattern["id"],
+        )

pr_analyzer/security/semgrep_runner.py

@@ -0,0 +1,190 @@
+import json
+import os
+import subprocess
+import tempfile
+from urllib.parse import unquote
+
+import structlog
+
+from pr_analyzer.config import settings
+from pr_analyzer.diff import iter_added_lines
+from pr_analyzer.models import FileChange, Suggestion
+from pr_analyzer.security import tools
+
+log = structlog.get_logger(__name__)
+
+# the stage's id in .pr-analyzer.toml; Semgrep's own rule id is in each finding's title
+SEMGREP_RULE = "semgrep"
+
+
+class SemgrepRunner:
+
+    def __init__(self, config: str | None = None, timeout: int | None = None):
+        self.config = config or settings.semgrep_config
+        self.timeout = timeout or settings.semgrep_timeout_seconds
+        self.warnings: list[str] = []
+
+    def scan(self, files: list[FileChange]) -> list[Suggestion]:
+        self.warnings = []
+        targets = [
+            f for f in files
+            if f.status != "removed" and f.raw_content is not None and f.patch
+        ]
+        if not targets:
+            return []
+
+        executable = tools.find_tool("semgrep")
+        if executable is None:
+            self._warn("Semgrep scan skipped: semgrep is not installed "
+                       "(pip install 'pr-analyzer-cli[security]')")
+            return []
+
+        # one run over a copy of every changed file, laid out with its repo path so findings map
+        # straight back to the right file
+        with tempfile.TemporaryDirectory(prefix="pr-analyzer-semgrep-") as workdir:
+            files_by_path = {}
+            for file in targets:
+                relative = safe_relative_path(file.filename)
+                if relative is None:
+                    log.warning("semgrep_skipped_path", filename=file.filename)
+                    continue
+                destination = os.path.join(workdir, relative)
+                os.makedirs(os.path.dirname(destination), exist_ok=True)
+                with open(destination, "w", encoding="utf-8") as handle:
+                    handle.write(file.raw_content or "")
+                files_by_path[relative] = file
+
+            if not files_by_path:
+                return []
+            sarif = self.run_semgrep(executable, workdir)
+            suggestions = self.parse_sarif(sarif, files_by_path, root=workdir)
+
+        log.info("semgrep scan complete", total_suggestions=len(suggestions))
+        return suggestions
+
+    def run_semgrep(self, executable: str, target_dir: str) -> dict:
+        command = [
+            executable, "scan",
+            "--config", self.config,
+            "--sarif", "--quiet",
+            "--metrics", "off",
+            "--disable-version-check",
+            ".",
+        ]
+        try:
+            result = subprocess.run(
+                command,
+                cwd=target_dir,
+                capture_output=True,
+                text=True,
+                timeout=self.timeout,
+                env=tools.scrubbed_env(),
+            )
+        except subprocess.TimeoutExpired:
+            self._warn(f"Semgrep scan skipped: it took longer than {self.timeout}s")
+            return {}
+        except OSError as e:
+            self._warn(f"Semgrep scan skipped: could not run semgrep ({e})")
+            return {}
+
+        if result.returncode not in (0, 1):
+            self._warn(f"Semgrep scan failed (exit {result.returncode}): "
+                       f"{result.stderr.strip()[:200]}")
+            return {}
+        if not result.stdout.strip():
+            return {}
+        try:
+            return json.loads(result.stdout)
+        except json.JSONDecodeError:
+            self._warn("Semgrep scan failed: its output was not valid SARIF JSON")
+            return {}
+
+    def parse_sarif(
+        self, sarif: dict, files_by_path: dict[str, FileChange], root: str | None = None
+    ) -> list[Suggestion]:
+        runs = sarif.get("runs") or []
+        if not runs:
+            return []
+        run = runs[0]
+        levels = rule_levels(run)
+
+        added_lines: dict[str, set[int]] = {}
+        suggestions = []
+        for result in run.get("results") or []:
+            location = result_location(result)
+            if location is None:
+                continue
+            uri, start, end = location
+            file = files_by_path.get(sarif_path(uri, root))
+            if file is None:
+                continue
+
+            # report only what the PR added; issues already in the file aren't this PR's doing
+            if file.filename not in added_lines:
+                added_lines[file.filename] = {n for n, _ in iter_added_lines(file.patch or "")}
+            if not any(n in added_lines[file.filename] for n in range(start, end + 1)):
+                continue
+
+            rule_id = result.get("ruleId", "unknown")
+            level = result.get("level") or levels.get(rule_id, "warning")
+            message = (result.get("message") or {}).get("text", "")
+            suggestions.append(Suggestion(
+                filename=file.filename,
+                severity=self.map_severity(level),
+                category="security",
+                title=f"Semgrep: {rule_id.rsplit('.', 1)[-1]}",
+                description=f"{message} (rule: {rule_id})",
+                line=start,
+                source="security",
+                rule_id=SEMGREP_RULE,
+            ))
+        return suggestions
+
+    def map_severity(self, level: str) -> str:
+        mapping = {
+            "error": "HIGH",
+            "warning": "MEDIUM",
+            "note": "LOW",
+            "none": "LOW",
+        }
+        return mapping.get(level.lower(), "MEDIUM")
+
+    def _warn(self, message: str) -> None:
+        log.warning("semgrep", problem=message)
+        self.warnings.append(message)
+
+
+def rule_levels(run: dict) -> dict[str, str]:
+    # semgrep puts the severity on the rule (defaultConfiguration.level), not on each result
+    rules = (run.get("tool") or {}).get("driver", {}).get("rules") or []
+    return {
+        rule.get("id"): (rule.get("defaultConfiguration") or {}).get("level", "warning")
+        for rule in rules
+    }
+
+
+def result_location(result: dict) -> tuple[str, int, int] | None:
+    """(uri, first line, last line) of a SARIF result, or None if it names no line."""
+    location = ((result.get("locations") or [{}])[0]).get("physicalLocation") or {}
+    uri = (location.get("artifactLocation") or {}).get("uri", "")
+    region = location.get("region") or {}
+    start = region.get("startLine")
+    if not isinstance(start, int):
+        return None
+    end = region.get("endLine")
+    return uri, start, end if isinstance(end, int) else start
+
+
+def safe_relative_path(filename: str) -> str | None:
+    """The file's repo path, or None if it could escape the scan directory."""
+    normalized = os.path.normpath(filename.replace("\\", "/"))
+    if os.path.isabs(normalized) or normalized in (".", "..") or normalized.startswith(".." + os.sep):
+        return None
+    return normalized
+
+
+def sarif_path(uri: str, root: str | None) -> str:
+    path = unquote(uri[len("file://"):] if uri.startswith("file://") else uri)
+    if os.path.isabs(path) and root:
+        path = os.path.relpath(os.path.realpath(path), os.path.realpath(root))
+    return os.path.normpath(path)

pr_analyzer/security/tools.py

@@ -0,0 +1,22 @@
+"""Finding and preparing the external scanners (semgrep, pip-audit)."""
+import os
+import shutil
+import sys
+
+# credentials that must never reach tools which process untrusted PR content
+SENSITIVE_ENV_VARS = {"GITHUB_TOKEN", "GH_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_API_KEY"}
+
+
+def find_tool(name: str) -> str | None:
+    """Look next to the running interpreter first, so tools installed in the same virtualenv
+    are found even when that virtualenv isn't activated, then on PATH."""
+    search_path = os.pathsep.join([os.path.dirname(sys.executable), os.environ.get("PATH", "")])
+    return shutil.which(name, path=search_path)
+
+
+def scrubbed_env() -> dict[str, str]:
+    return {
+        key: value
+        for key, value in os.environ.items()
+        if key not in SENSITIVE_ENV_VARS and not key.endswith("_API_KEY")
+    }

pr_analyzer/suppression.py

@@ -0,0 +1,52 @@
+"""Inline suppression of findings.
+
+Every finding on a line whose text contains `pr-analyzer: ignore` (usually in a comment) is
+dropped, whichever check reported it. The pipeline names the ignored lines in a warning, so a
+reviewer can see what a PR chose to silence:
+
+    subprocess.run(cmd, shell=True)  # pr-analyzer: ignore - cmd is a fixed string
+"""
+import re
+from collections.abc import Iterable
+
+from pr_analyzer.diff import iter_new_lines
+from pr_analyzer.models import FileChange, Suggestion
+
+IGNORE_MARKER = re.compile(r"\bpr-analyzer:\s*ignore\b")
+# ignored lines named in the warning; any beyond this are only counted
+MAX_LISTED_IGNORED = 10
+
+
+def marked_lines(file: FileChange) -> set[int]:
+    """Line numbers in the new version of the file that carry the marker."""
+    numbered: Iterable[tuple[int, str]]
+    if file.raw_content is not None:
+        # split on "\n" only, so the numbering matches the diff's
+        numbered = enumerate(file.raw_content.split("\n"), start=1)
+    else:
+        # the full content isn't always available; the patch still holds every added line
+        numbered = ((line_no, text) for line_no, text, _ in iter_new_lines(file.patch or ""))
+    return {line_no for line_no, text in numbered if IGNORE_MARKER.search(text)}
+
+
+def split_ignored(
+    suggestions: list[Suggestion], files: list[FileChange]
+) -> tuple[list[Suggestion], list[Suggestion]]:
+    """Return (kept, ignored)."""
+    marked = {f.filename: marked_lines(f) for f in files}
+    kept: list[Suggestion] = []
+    ignored: list[Suggestion] = []
+    for s in suggestions:
+        if s.line is not None and s.line in marked.get(s.filename, set()):
+            ignored.append(s)
+        else:
+            kept.append(s)
+    return kept, ignored
+
+
+def ignored_warning(ignored: list[Suggestion]) -> str:
+    locations = sorted({(s.filename, s.line or 0) for s in ignored})
+    listed = ", ".join(f"{name}:{line}" for name, line in locations[:MAX_LISTED_IGNORED])
+    if len(locations) > MAX_LISTED_IGNORED:
+        listed += f" and {len(locations) - MAX_LISTED_IGNORED} more"
+    return f"Ignored {len(ignored)} finding(s) on lines marked 'pr-analyzer: ignore': {listed}"