universal-dev-security 1.0.0__py3-none-any.whl

universal_dev_security/scanners/base.py

@@ -0,0 +1,269 @@
+"""Common scanner adapter contract and safe process execution."""
+from __future__ import annotations
+
+import os
+import re
+import shutil
+import subprocess
+import sys
+import time
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from universal_dev_security.findings import Category, Finding, ScanResult, ScanStatus
+from universal_dev_security.projectfiles import DEFAULT_EXCLUDED_DIRS, ProjectInventory
+
+# Batch wrappers are executed through cmd.exe on Windows, where argument
+# quoting cannot be made fully safe (see CVE-2024-24576 class of issues).
+UNSAFE_WINDOWS_SUFFIXES = (".bat", ".cmd")
+VERSION_RE = re.compile(r"(\d+)\.(\d+)(?:\.(\d+))?")
+MAX_OUTPUT_BYTES = 200 * 1024 * 1024
+
+
+class ScannerExecutionError(RuntimeError):
+    """Raised for scanner failures. Messages never contain raw scanner output."""
+
+
+@dataclass
+class ScanOptions:
+    timeout_seconds: int = 600
+    allow_network: bool = False
+    semgrep_configs: list[str] = field(default_factory=list)
+    exclude_paths: list[str] = field(default_factory=list)
+    technologies: list[str] = field(default_factory=list)
+
+    @property
+    def excluded_dirs(self) -> list[str]:
+        return sorted(set(DEFAULT_EXCLUDED_DIRS) | {p.strip("/\\") for p in self.exclude_paths if p.strip("/\\")})
+
+
+def parse_version(text: str | None) -> tuple[int, int, int] | None:
+    if not text:
+        return None
+    match = VERSION_RE.search(text)
+    if not match:
+        return None
+    return int(match.group(1)), int(match.group(2)), int(match.group(3) or 0)
+
+
+def format_version(version: tuple[int, int, int] | None) -> str | None:
+    return ".".join(map(str, version)) if version else None
+
+
+def _is_safe_executable(path: str) -> bool:
+    if os.name == "nt" and path.lower().endswith(UNSAFE_WINDOWS_SUFFIXES):
+        return False
+    return Path(path).is_file()
+
+
+def run_tool(
+    args: list[str],
+    *,
+    cwd: Path,
+    timeout: int,
+    env_overrides: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[str]:
+    """Run a scanner safely: argument list, no shell, no stdin, bounded time.
+
+    Callers must never print or persist ``stdout``/``stderr`` verbatim because
+    scanners may echo secret material.
+    """
+    if not args or not all(isinstance(arg, str) for arg in args):
+        raise ScannerExecutionError("Invalid scanner command")
+    if any("\x00" in arg for arg in args):
+        raise ScannerExecutionError("Scanner arguments may not contain NUL bytes")
+    if not _is_safe_executable(args[0]):
+        raise ScannerExecutionError("Refusing to execute a missing or batch-file scanner executable")
+    env = dict(os.environ)
+    env.update(env_overrides or {})
+    creationflags = getattr(subprocess, "CREATE_NO_WINDOW", 0) if os.name == "nt" else 0
+    return subprocess.run(  # noqa: S603 - argument list, shell=False, validated executable
+        args,
+        cwd=str(cwd),
+        stdin=subprocess.DEVNULL,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        timeout=timeout,
+        check=False,
+        shell=False,
+        env=env,
+        creationflags=creationflags,
+    )
+
+
+def _candidate_dirs(tool: str) -> list[Path]:
+    """Well-known per-user install locations, checked after PATH."""
+    home = Path.home()
+    dirs = [Path(sys.executable).parent]  # same virtual environment as UDS (pip-installed semgrep)
+    if os.name == "nt":
+        local = os.environ.get("LOCALAPPDATA")
+        if local:
+            winget = Path(local) / "Microsoft" / "WinGet"
+            dirs.append(winget / "Links")
+            patterns = {
+                "gitleaks": "Gitleaks.Gitleaks_*",
+                "trivy": "AquaSecurity.Trivy_*",
+                "osv-scanner": "Google.OSVScanner_*",
+            }
+            if tool in patterns and (winget / "Packages").is_dir():
+                dirs.extend(sorted((winget / "Packages").glob(patterns[tool])))
+        dirs.append(home / "scoop" / "shims")
+    else:
+        dirs.extend([Path("/opt/homebrew/bin"), Path("/usr/local/bin"), home / ".local" / "bin"])
+    dirs.append(home / "go" / "bin")
+    return dirs
+
+
+def find_executable(tool: str, names: tuple[str, ...]) -> tuple[str | None, str | None]:
+    """Return (path, note). ``note`` explains refused candidates."""
+    override = os.environ.get(f"UDS_{tool.upper().replace('-', '_')}_PATH")
+    if override:
+        if _is_safe_executable(override):
+            return str(Path(override).resolve()), None
+        return None, f"UDS_{tool.upper().replace('-', '_')}_PATH does not point to a usable executable."
+    refused = None
+    for name in names:
+        found = shutil.which(name)
+        if found:
+            if _is_safe_executable(found):
+                return found, None
+            refused = found
+    suffixes = (".exe",) if os.name == "nt" else ("",)
+    for directory in _candidate_dirs(tool):
+        for name in names:
+            for suffix in suffixes:
+                candidate = directory / f"{name}{suffix}"
+                if candidate.is_file() and _is_safe_executable(str(candidate)):
+                    return str(candidate), None
+    if refused:
+        return None, "Only a batch-file wrapper was found; UDS refuses to run .bat/.cmd scanners. Install the native executable."
+    return None, None
+
+
+def relative_path(value: Any, root: Path) -> str:
+    """Normalize a scanner-reported path to a project-relative posix path."""
+    text = str(value or "").strip()
+    if not text:
+        return ""
+    candidate = Path(text)
+    if candidate.is_absolute():
+        try:
+            return candidate.resolve().relative_to(root.resolve()).as_posix()
+        except (ValueError, OSError):
+            return candidate.name  # never leak paths outside the project
+    normalized = text.replace("\\", "/")
+    while normalized.startswith("./"):
+        normalized = normalized[2:]
+    return normalized
+
+
+class ScannerAdapter(ABC):
+    name: str = ""
+    display_name: str = ""
+    category: Category = Category.CODE
+    executable_names: tuple[str, ...] = ()
+    min_version: tuple[int, int, int] = (0, 0, 0)
+    version_args: tuple[str, ...] = ("--version",)
+    # Privacy properties surfaced in reports and used by the scan policy.
+    transmits_project_metadata: bool = False
+    downloads_data: bool = False
+    network_note: str = ""
+    docs_url: str = ""
+    install_hints: dict[str, str] = {}
+
+    # -- discovery ------------------------------------------------------
+
+    def locate(self) -> tuple[str | None, str | None]:
+        return find_executable(self.name, self.executable_names)
+
+    def detect_version(self, executable: str, cwd: Path) -> str | None:
+        try:
+            completed = run_tool([executable, *self.version_args], cwd=cwd, timeout=60)
+        except (OSError, subprocess.SubprocessError, ScannerExecutionError):
+            return None
+        return format_version(parse_version(completed.stdout) or parse_version(completed.stderr))
+
+    def is_compatible(self, version: str | None) -> bool:
+        parsed = parse_version(version)
+        return parsed is not None and parsed >= self.min_version
+
+    def info(self, cwd: Path | None = None) -> dict[str, Any]:
+        executable, note = self.locate()
+        version = self.detect_version(executable, cwd or Path.cwd()) if executable else None
+        return {
+            "name": self.name,
+            "display_name": self.display_name,
+            "installed": executable is not None,
+            "executable": executable,
+            "version": version,
+            "minimum_version": format_version(self.min_version),
+            "compatible": self.is_compatible(version) if executable else None,
+            "note": note,
+            "transmits_project_metadata": self.transmits_project_metadata,
+            "downloads_data": self.downloads_data,
+            "network_note": self.network_note,
+            "docs_url": self.docs_url,
+            "install_hints": self.install_hints,
+        }
+
+    # -- execution ------------------------------------------------------
+
+    @abstractmethod
+    def applicable(self, inventory: ProjectInventory) -> tuple[bool, str]:
+        """Return (applicable, reason)."""
+
+    @abstractmethod
+    def execute(self, executable: str, version: str, target: Path, options: ScanOptions) -> list[Finding]:
+        """Run the scanner and return normalized findings, or raise ScannerExecutionError."""
+
+    def scan(self, target: Path, inventory: ProjectInventory, options: ScanOptions) -> ScanResult:
+        started = time.monotonic()
+        root = target.expanduser().resolve()
+
+        def result(status: ScanStatus, message: str, reason: str | None = None, version: str | None = None,
+                   findings: list[Finding] | None = None) -> ScanResult:
+            return ScanResult(
+                tool=self.name, status=status, findings=findings or [], version=version, message=message,
+                reason=reason, duration_seconds=round(time.monotonic() - started, 3),
+            )
+
+        applicable, why = self.applicable(inventory)
+        if not applicable:
+            return result(ScanStatus.NOT_APPLICABLE, why, "not_applicable")
+        executable, note = self.locate()
+        if not executable:
+            hint = note or f"{self.display_name} was not found. See {self.docs_url} for official installation."
+            return result(ScanStatus.NOT_INSTALLED, hint, "executable_not_found")
+        version = self.detect_version(executable, root)
+        if not self.is_compatible(version):
+            return result(
+                ScanStatus.SKIPPED,
+                f"{self.display_name} version {version or 'unknown'} is not supported; "
+                f"{format_version(self.min_version)} or newer is required.",
+                "unsupported_version", version,
+            )
+        if self.transmits_project_metadata and not options.allow_network:
+            return result(
+                ScanStatus.SKIPPED,
+                f"{self.display_name} was not run because it {self.network_note} "
+                "Enable it with `uds scan --allow-network` or `[scanners] allow_network = true`.",
+                "network_not_approved", version,
+            )
+        try:
+            findings = self.execute(executable, version or "", root, options)
+        except subprocess.TimeoutExpired:
+            return result(ScanStatus.ERROR, f"{self.display_name} timed out after {options.timeout_seconds}s.",
+                          "timeout", version)
+        except ScannerExecutionError as exc:
+            return result(ScanStatus.ERROR, str(exc), "execution_failed", version)
+        except OSError as exc:
+            return result(ScanStatus.ERROR, f"{self.display_name} could not be started ({type(exc).__name__}).",
+                          "execution_failed", version)
+        if findings:
+            return result(ScanStatus.FINDINGS, f"{len(findings)} finding(s) reported by {self.display_name}.",
+                          version=version, findings=findings)
+        return result(ScanStatus.CLEAN, f"No findings reported by {self.display_name}.", version=version)

universal_dev_security/scanners/gitleaks.py

@@ -0,0 +1,132 @@
+"""Gitleaks adapter: secret detection (https://github.com/gitleaks/gitleaks)."""
+from __future__ import annotations
+
+import json
+import re
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from universal_dev_security.findings import Category, Finding, Severity
+from universal_dev_security.projectfiles import ProjectInventory
+from universal_dev_security.scanners.base import (
+    ScannerAdapter,
+    ScannerExecutionError,
+    ScanOptions,
+    relative_path,
+    run_tool,
+)
+
+# Third-party and UDS-generated content only. Build output (dist/, build/) is
+# deliberately still scanned: bundles frequently leak credentials.
+SECRET_SCAN_EXCLUDES = (
+    ".venv", "venv", "node_modules", "bower_components", "site-packages", ".tox", ".nox", "__pycache__",
+    ".mypy_cache", ".pytest_cache", ".ruff_cache", ".terraform", ".git",
+    # Framework caches and dev-server builds: never deployed, extremely noisy (found when
+    # scanning a real Next.js project: 97% of findings). Deployable output such as
+    # .next/static and .next/server is still scanned.
+    ".next/cache", ".next/dev", ".turbo", ".parcel-cache", ".angular/cache",
+)
+UDS_GENERATED = (".uds/reports", ".uds/history", ".uds/backups", ".uds/cache")
+
+REMEDIATION = (
+    "Treat the credential as compromised: revoke or rotate it with the issuing service, remove it from the "
+    "code, load it from an environment variable or secret manager, and if it was committed, purge it from "
+    "Git history with an approved history-rewrite procedure."
+)
+
+
+def build_exclusion_config(extra_excludes: list[str]) -> str:
+    names = list(SECRET_SCAN_EXCLUDES) + [e.strip("/\\") for e in extra_excludes if e.strip("/\\")]
+    parts = [re.escape(name).replace("/", r"[\\/]") for name in names]
+    parts += [re.escape(p).replace("/", r"[\\/]") for p in UDS_GENERATED]
+    path_regex = r"(?:^|[\\/])(?:" + "|".join(parts) + r")(?:[\\/]|$)"
+    if "'''" in path_regex:  # cannot happen with re.escape output, but never emit broken TOML
+        raise ScannerExecutionError("Invalid exclusion pattern")
+    return (
+        'title = "UDS default exclusions"\n\n'
+        "[extend]\nuseDefault = true\n\n"
+        "[allowlist]\n"
+        'description = "Third-party and UDS-generated directories"\n'
+        f"paths = ['''{path_regex}''']\n"
+    )
+
+
+def normalize(raw: list[dict[str, Any]], root: Path) -> list[Finding]:
+    findings = []
+    for item in raw:
+        if not isinstance(item, dict):
+            continue
+        rule = str(item.get("RuleID") or "unknown")[:200]
+        # Only metadata is copied. Secret, Match, Line and Entropy are never read.
+        findings.append(Finding(
+            tool="gitleaks",
+            category=Category.SECRET,
+            rule_id=rule,
+            title=f"Potential secret: {rule}",
+            message=str(item.get("Description") or "")[:500],
+            file=relative_path(item.get("File"), root),
+            start_line=item.get("StartLine"),
+            end_line=item.get("EndLine"),
+            commit=str(item.get("Commit") or "")[:64] or None,
+            severity=Severity.UNCLASSIFIED,
+            severity_source="none",
+            remediation=REMEDIATION,
+            references=["https://github.com/gitleaks/gitleaks"],
+        ))
+    return findings
+
+
+class GitleaksAdapter(ScannerAdapter):
+    name = "gitleaks"
+    display_name = "Gitleaks"
+    category = Category.SECRET
+    executable_names = ("gitleaks",)
+    min_version = (8, 19, 0)  # `gitleaks dir` was introduced in 8.19.0
+    version_args = ("version",)
+    network_note = "runs fully offline."
+    docs_url = "https://github.com/gitleaks/gitleaks#installing"
+    install_hints = {
+        "windows": "winget install --id Gitleaks.Gitleaks -e",
+        "macos": "brew install gitleaks",
+        "linux": "Download a release from https://github.com/gitleaks/gitleaks/releases and verify its checksum",
+    }
+
+    def applicable(self, inventory: ProjectInventory) -> tuple[bool, str]:
+        return True, "Secret scanning applies to every project."
+
+    def build_command(self, executable: str, report: Path, config: Path | None) -> list[str]:
+        command = [executable, "dir", "--no-banner", "--redact", "--exit-code", "1",
+                   "--report-format", "json", "--report-path", str(report)]
+        if config is not None:
+            command += ["--config", str(config)]
+        return command + ["."]
+
+    def execute(self, executable: str, version: str, target: Path, options: ScanOptions) -> list[Finding]:
+        with tempfile.TemporaryDirectory(prefix="uds-gitleaks-") as temp_dir:
+            report = Path(temp_dir) / "gitleaks-report.json"
+            config: Path | None = None
+            # Respect a project-provided Gitleaks configuration instead of overriding it.
+            if not (target / ".gitleaks.toml").is_file():
+                config = Path(temp_dir) / "uds-gitleaks.toml"
+                config.write_text(build_exclusion_config(options.exclude_paths), encoding="utf-8")
+            completed = run_tool(self.build_command(executable, report, config), cwd=target,
+                                 timeout=options.timeout_seconds)
+            if completed.returncode not in (0, 1):
+                raise ScannerExecutionError(
+                    f"Gitleaks exited with code {completed.returncode}; raw output was suppressed."
+                )
+            if not report.is_file():
+                if completed.returncode == 0:
+                    return []
+                raise ScannerExecutionError("Gitleaks reported leaks but produced no report file.")
+            try:
+                data = json.loads(report.read_text(encoding="utf-8") or "[]")
+            except (OSError, ValueError, UnicodeDecodeError) as exc:
+                raise ScannerExecutionError("Gitleaks report could not be parsed; raw output was suppressed.") from exc
+        if not isinstance(data, list):
+            raise ScannerExecutionError("Gitleaks report has an unexpected format.")
+        findings = normalize(data, target)
+        if completed.returncode == 1 and not findings:
+            raise ScannerExecutionError("Gitleaks signalled leaks but the report contained none.")
+        return findings

universal_dev_security/scanners/osv.py

@@ -0,0 +1,156 @@
+"""OSV-Scanner adapter: dependency vulnerabilities (https://google.github.io/osv-scanner/)."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from universal_dev_security.detector import has_dependency_manifests
+from universal_dev_security.findings import Category, Finding, Severity, cvss_score_to_severity
+from universal_dev_security.projectfiles import ProjectInventory
+from universal_dev_security.scanners.base import (
+    ScannerAdapter,
+    ScannerExecutionError,
+    ScanOptions,
+    parse_version,
+    relative_path,
+    run_tool,
+)
+
+# GitHub Advisory Database severities, used only when no CVSS score is present.
+ADVISORY_SEVERITY = {
+    "CRITICAL": Severity.CRITICAL,
+    "HIGH": Severity.HIGH,
+    "MODERATE": Severity.MEDIUM,
+    "MEDIUM": Severity.MEDIUM,
+    "LOW": Severity.LOW,
+}
+EXIT_NO_PACKAGES = 128
+
+
+def _fixed_version(vuln: dict[str, Any], package_name: str) -> str | None:
+    affected = vuln.get("affected") if isinstance(vuln.get("affected"), list) else []
+    fixed: list[str] = []
+    for entry in affected:
+        if not isinstance(entry, dict):
+            continue
+        pkg = entry.get("package") if isinstance(entry.get("package"), dict) else {}
+        if pkg.get("name") and str(pkg.get("name")).lower() != package_name.lower():
+            continue
+        for rng in entry.get("ranges") or []:
+            for event in (rng.get("events") or []) if isinstance(rng, dict) else []:
+                if isinstance(event, dict) and event.get("fixed"):
+                    fixed.append(str(event["fixed"]))
+    return ", ".join(dict.fromkeys(fixed))[:200] or None
+
+
+def normalize(data: dict[str, Any], root: Path) -> list[Finding]:
+    results = data.get("results")
+    if results is None:
+        return []
+    if not isinstance(results, list):
+        raise ScannerExecutionError("OSV-Scanner output has an unexpected format.")
+    findings: list[Finding] = []
+    for result in results:
+        if not isinstance(result, dict):
+            continue
+        source = result.get("source") if isinstance(result.get("source"), dict) else {}
+        file = relative_path(source.get("path"), root)
+        for pkg_entry in result.get("packages") or []:
+            if not isinstance(pkg_entry, dict):
+                continue
+            package = pkg_entry.get("package") if isinstance(pkg_entry.get("package"), dict) else {}
+            name = str(package.get("name") or "unknown")[:200]
+            version = str(package.get("version") or "")[:100] or None
+            ecosystem = str(package.get("ecosystem") or "")[:50]
+            vulns = {str(v.get("id")): v for v in pkg_entry.get("vulnerabilities") or [] if isinstance(v, dict)}
+            groups = [g for g in pkg_entry.get("groups") or [] if isinstance(g, dict)]
+            if not groups:  # older output: one group per vulnerability
+                groups = [{"ids": [vid], "max_severity": ""} for vid in vulns]
+            for group in groups:
+                ids = [str(i) for i in group.get("ids") or []]
+                if not ids:
+                    continue
+                primary = next((i for i in ids if i.startswith("GHSA-")), ids[0])
+                vuln = vulns.get(primary) or vulns.get(ids[0]) or {}
+                severity = cvss_score_to_severity(group.get("max_severity"))
+                severity_source = "cvss-qualitative-scale" if severity is not Severity.UNCLASSIFIED else "none"
+                db_specific = vuln.get("database_specific") if isinstance(vuln.get("database_specific"), dict) else {}
+                if severity is Severity.UNCLASSIFIED:
+                    advisory = str(db_specific.get("severity") or "").upper()
+                    if advisory in ADVISORY_SEVERITY:
+                        severity, severity_source = ADVISORY_SEVERITY[advisory], "advisory-database"
+                aliases = [str(a) for a in (group.get("aliases") or vuln.get("aliases") or [])][:10]
+                references = [str(r.get("url")) for r in vuln.get("references") or []
+                              if isinstance(r, dict) and r.get("url")]
+                references.insert(0, f"https://osv.dev/vulnerability/{primary}")
+                fixed = _fixed_version(vuln, name)
+                findings.append(Finding(
+                    tool="osv-scanner",
+                    category=Category.DEPENDENCY,
+                    rule_id=primary[:100],
+                    title=str(vuln.get("summary") or f"Known vulnerability in {name}")[:300],
+                    message=f"{ecosystem} package {name} {version or ''} is affected by {', '.join(ids[:5])}"
+                            + (f" (aliases: {', '.join(aliases[:5])})" if aliases else "") + ".",
+                    file=file,
+                    package=name,
+                    installed_version=version,
+                    fixed_version=fixed,
+                    severity=severity,
+                    severity_source=severity_source,
+                    scanner_severity=str(group.get("max_severity") or db_specific.get("severity") or "") or None,
+                    references=references,
+                    remediation=(f"Upgrade {name} to {fixed} or later, then run the test suite."
+                                 if fixed else f"No fixed version is listed; assess exposure and consider replacing {name}."),
+                ))
+    return findings
+
+
+class OsvScannerAdapter(ScannerAdapter):
+    name = "osv-scanner"
+    display_name = "OSV-Scanner"
+    category = Category.DEPENDENCY
+    executable_names = ("osv-scanner",)
+    min_version = (1, 4, 0)
+    transmits_project_metadata = True
+    downloads_data = True
+    network_note = "sends package names and versions (not source code) to the osv.dev API."
+    docs_url = "https://google.github.io/osv-scanner/installation/"
+    install_hints = {
+        "windows": "winget install --id Google.OSVScanner -e  or download from https://github.com/google/osv-scanner/releases",
+        "macos": "brew install osv-scanner",
+        "linux": "Download a release from https://github.com/google/osv-scanner/releases and verify its checksum",
+    }
+
+    def applicable(self, inventory: ProjectInventory) -> tuple[bool, str]:
+        if has_dependency_manifests(inventory):
+            return True, "Dependency manifests or lockfiles detected."
+        return False, "No dependency manifests or lockfiles were found."
+
+    def build_command(self, executable: str, version: str, options: ScanOptions) -> list[str]:
+        parsed = parse_version(version) or (0, 0, 0)
+        command = [executable]
+        if parsed >= (2, 0, 0):
+            command += ["scan", "source"]
+        command += ["--format", "json", "--recursive"]
+        return command + ["."]
+
+    def execute(self, executable: str, version: str, target: Path, options: ScanOptions) -> list[Finding]:
+        completed = run_tool(self.build_command(executable, version, options), cwd=target,
+                             timeout=options.timeout_seconds)
+        if completed.returncode == EXIT_NO_PACKAGES:
+            return []
+        if completed.returncode not in (0, 1):
+            raise ScannerExecutionError(
+                f"OSV-Scanner exited with code {completed.returncode}; raw output was suppressed."
+            )
+        try:
+            data = json.loads(completed.stdout or "{}")
+        except ValueError as exc:
+            raise ScannerExecutionError("OSV-Scanner output could not be parsed; raw output was suppressed.") from exc
+        if not isinstance(data, dict):
+            raise ScannerExecutionError("OSV-Scanner output has an unexpected format.")
+        findings = normalize(data, target)
+        if completed.returncode == 1 and not findings:
+            raise ScannerExecutionError("OSV-Scanner reported vulnerabilities but none could be parsed.")
+        return findings