dockerls 1.0.0__py3-none-any.whl

dockerls/integrations/exploitdb/client.py

@@ -0,0 +1,271 @@
+"""Public exploits catalogued in Exploit-DB, matched strictly by CVE-ID.
+
+Exploit-DB has no official public API, and this project does not scrape:
+CISA KEV and EPSS are consumed from their published feeds, and a scraper
+breaks silently every time somebody edits the site's HTML. What Exploit-DB
+*does* publish is the same `files_exploits.csv` that `searchsploit` reads,
+kept in the project's own repository. One HTTP request, one CSV parse, no
+system dependency and nothing to install.
+
+**Matching is CVE-ID to CVE-ID, and nothing else.** The catalogue records a
+CVE only when the exploit's author declared one, so plenty of entries carry
+none -- roughly 42% of them. The tempting fallback is to search by product
+name and attach whatever comes back, and it manufactures false positives: a
+2015 exploit for an old release of curl says nothing about the curl CVE
+Trivy found in this image today, and reporting it as "exploit available"
+would be worse than reporting nothing. An unmatched CVE comes back FALSE
+(the catalogue answered and does not list it), never a guess.
+
+Failure degrades to UNKNOWN for every CVE in the scan, never to FALSE and
+never to an exception: an unreachable catalogue is an absent measurement,
+and the single rule this project is organised around is that an absent
+measurement is never spent as reassurance.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import csv
+import io
+from typing import TYPE_CHECKING, Any
+
+import httpx
+from loguru import logger
+
+from dockerls.infrastructure.network.host_guard import host_of_url
+
+if TYPE_CHECKING:
+    from dockerls.domain.interfaces.cache_store import CacheStoreInterface
+    from dockerls.infrastructure.network.host_guard import HostGuard
+
+#: The CSV the Exploit-DB project publishes and `searchsploit` consumes.
+#: The repository moved from GitHub to GitLab; the old
+#: `offensive-security/exploitdb` path is gone and answers 404.
+EXPLOITDB_CSV_URL = "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv"
+
+#: Where a reader goes to judge the exploit rather than trust our summary.
+EXPLOIT_URL_TEMPLATE = "https://www.exploit-db.com/exploits/{edb_id}"
+
+#: Refreshed daily. The catalogue grows by a handful of entries a day, so a
+#: fresher copy would cost 10 MB per run to learn almost nothing.
+CACHE_TTL_SECONDS = 24 * 60 * 60
+CACHE_KEY = "exploitdb:cve-index:v1"
+
+#: The file is ~10 MB and ~47k rows today. The ceiling is not a performance
+#: tuning knob: it bounds what an unexpected response (a redirect to
+#: something else, a compromised mirror, an error page) can make this
+#: process allocate. Generous enough to absorb years of growth.
+MAX_CSV_BYTES = 128 * 1024 * 1024
+
+#: Defence against a malformed or hostile row claiming thousands of CVEs.
+#: Real entries carry a handful.
+MAX_CODES_PER_ROW = 64
+
+
+class ExploitEntry:
+    """One catalogued exploit, reduced to what the report shows."""
+
+    __slots__ = ("edb_id", "verified")
+
+    def __init__(self, edb_id: str, verified: bool):
+        self.edb_id = edb_id
+        self.verified = verified
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging aid
+        return f"ExploitEntry({self.edb_id!r}, verified={self.verified})"
+
+
+class ExploitDBClient:
+    """Best-effort Exploit-DB lookups over the published CSV.
+
+    Mirrors `ThreatIntelClient` in shape -- an `available` tri-state, a
+    memo guarded by a lock so a 100-tag `recommend` run does not start 100
+    simultaneous downloads -- and adds the persistent cache that one lacks,
+    because 10 MB per invocation is a different proposition from 2 MB.
+    """
+
+    def __init__(
+        self,
+        timeout: int = 30,
+        cache: CacheStoreInterface | None = None,
+        guard: HostGuard | None = None,
+    ):
+        self._timeout = timeout
+        self._cache = cache
+        self._guard = guard
+        self._index: dict[str, list[ExploitEntry]] | None = None
+        # None before anything was asked, True once the catalogue answered,
+        # False after it failed. The caller needs this to tell "not listed"
+        # from "never looked", which is the whole point of the tri-state.
+        self._available: bool | None = None
+        self._lock = asyncio.Lock()
+
+    @property
+    def available(self) -> bool | None:
+        """True once the catalogue answered, False after it failed, None
+        before anything was asked."""
+        return self._available
+
+    async def exploits_for(self, cve_ids: list[str]) -> dict[str, list[ExploitEntry]]:
+        """Map each of `cve_ids` that has catalogued exploits to them.
+
+        A CVE absent from the result was *not* found in the catalogue, which
+        is a finding -- but only when `available` is True. With `available`
+        False nothing was consulted and every CVE must stay UNKNOWN.
+        """
+        if not cve_ids:
+            return {}
+        index = await self._load()
+        if not index:
+            return {}
+        found: dict[str, list[ExploitEntry]] = {}
+        for cve in cve_ids:
+            entries = index.get(cve.upper())
+            if entries:
+                found[cve.upper()] = entries
+        return found
+
+    async def _load(self) -> dict[str, list[ExploitEntry]]:
+        if self._index is not None:
+            return self._index
+        async with self._lock:
+            # Re-checked inside the lock: everything that queued behind the
+            # first downloader is served the memo instead of repeating it.
+            if self._index is not None:
+                return self._index
+            self._index = await self._load_uncached()
+        return self._index
+
+    async def _load_uncached(self) -> dict[str, list[ExploitEntry]]:
+        cached = await self._from_cache()
+        if cached is not None:
+            self._available = True
+            return cached
+
+        raw = await self._download()
+        if raw is None:
+            self._available = False
+            return {}
+
+        index = self._parse(raw)
+        # An empty index is not a successful lookup: the real catalogue
+        # carries tens of thousands of rows, so parsing to nothing means the
+        # response was not the file we asked for.
+        self._available = bool(index)
+        if index:
+            await self._store(index)
+        else:
+            logger.warning(
+                "Exploit-DB catalogue parsed to zero entries: exploit status stays UNKNOWN"
+            )
+        return index
+
+    async def _download(self) -> str | None:
+        # The network policy decides who this process may contact, exactly
+        # as it does for registries and scanners. A guard that refuses is a
+        # deliberate configuration, not an error to route around.
+        # `allows` julga um host, nunca uma URL: entregar a URL inteira faz o
+        # guard perguntar ao DNS por `https`, que não resolve, e recusar todo
+        # download como se fosse decisão de política.
+        if self._guard is not None and not self._guard.allows(host_of_url(EXPLOITDB_CSV_URL)):
+            logger.warning(
+                "Exploit-DB catalogue refused by the network policy: exploit status will be UNKNOWN"
+            )
+            return None
+        try:
+            async with (
+                httpx.AsyncClient(timeout=self._timeout, follow_redirects=True) as client,
+                client.stream("GET", EXPLOITDB_CSV_URL) as resp,
+            ):
+                resp.raise_for_status()
+                chunks: list[bytes] = []
+                total = 0
+                # Streamed with a running ceiling rather than read whole:
+                # `resp.text` on an unexpected response would allocate
+                # whatever the other end decided to send.
+                async for chunk in resp.aiter_bytes():
+                    total += len(chunk)
+                    if total > MAX_CSV_BYTES:
+                        logger.warning(
+                            f"Exploit-DB catalogue exceeded {MAX_CSV_BYTES} bytes; "
+                            "refusing it and leaving exploit status UNKNOWN"
+                        )
+                        return None
+                    chunks.append(chunk)
+            return b"".join(chunks).decode("utf-8", errors="replace")
+        except (httpx.HTTPError, UnicodeDecodeError) as e:
+            logger.warning(f"Exploit-DB catalogue unavailable: exploit status stays UNKNOWN ({e})")
+            return None
+
+    @staticmethod
+    def _parse(raw: str) -> dict[str, list[ExploitEntry]]:
+        """Index the CSV by CVE-ID.
+
+        `codes` holds identifiers separated by `;` and mixes schemes --
+        `CVE-2009-3699;OSVDB-58726` -- so only the CVE-shaped ones are kept.
+        Everything here treats the file as untrusted input: a row missing the
+        fields this needs is skipped rather than raising.
+
+        A `csv.Error` -- a NUL byte, an unterminated quote, a field past the
+        module limit -- discards the **whole** index instead of keeping what
+        was read so far, and that is deliberate rather than lazy: a partial
+        catalogue would answer FALSE for every CVE whose row came after the
+        break, turning rows that were never read into "no published exploit".
+        Discarding leaves `available` False and every CVE UNKNOWN, which is
+        what actually happened.
+        """
+        index: dict[str, list[ExploitEntry]] = {}
+        try:
+            reader = csv.DictReader(io.StringIO(raw))
+            for row in reader:
+                edb_id = (row.get("id") or "").strip()
+                if not edb_id or not edb_id.isdigit():
+                    # The EDB-ID becomes a URL in the report; anything but
+                    # digits does not belong in one.
+                    continue
+                codes = (row.get("codes") or "").strip()
+                if not codes:
+                    continue
+                verified = (row.get("verified") or "").strip() == "1"
+                entry = ExploitEntry(edb_id, verified)
+                for code in codes.split(";")[:MAX_CODES_PER_ROW]:
+                    cve = code.strip().upper()
+                    if not cve.startswith("CVE-"):
+                        continue
+                    index.setdefault(cve, []).append(entry)
+        except csv.Error as e:
+            logger.warning(f"Exploit-DB catalogue could not be parsed: {e}")
+            return {}
+        return index
+
+    async def _from_cache(self) -> dict[str, list[ExploitEntry]] | None:
+        if self._cache is None:
+            return None
+        try:
+            data = await self._cache.get(CACHE_KEY)
+        except Exception as e:  # pragma: no cover - an unreadable cache is a miss
+            logger.debug(f"Could not read the cached Exploit-DB index: {e}")
+            return None
+        if not isinstance(data, dict) or not data:
+            return None
+        try:
+            return {
+                str(cve): [ExploitEntry(str(e["id"]), bool(e["verified"])) for e in entries]
+                for cve, entries in data.items()
+            }
+        except (KeyError, TypeError, ValueError) as e:
+            # A cache written by another version is a miss, not a crash.
+            logger.debug(f"Discarding an unreadable Exploit-DB cache entry: {e}")
+            return None
+
+    async def _store(self, index: dict[str, list[ExploitEntry]]) -> None:
+        if self._cache is None:
+            return
+        payload: dict[str, Any] = {
+            cve: [{"id": e.edb_id, "verified": e.verified} for e in entries]
+            for cve, entries in index.items()
+        }
+        try:
+            await self._cache.set(CACHE_KEY, payload, ttl_seconds=CACHE_TTL_SECONDS)
+        except Exception as e:  # pragma: no cover - a cache that will not write is not fatal
+            logger.debug(f"Could not cache the Exploit-DB index: {e}")

dockerls/integrations/grype/scanner.py

@@ -0,0 +1,336 @@
+from __future__ import annotations
+
+import json
+import os
+import shutil
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+from loguru import logger
+
+from dockerls.domain.entities.scan_result import ScanErrorKind, ScanResult, ScanStatus
+from dockerls.domain.entities.vulnerability import Severity, Vulnerability
+from dockerls.domain.interfaces.scanner import ScannerInterface
+from dockerls.integrations.engine.batch import EngineBatchScanner
+from dockerls.integrations.scan_errors import classify_scanner_error
+from dockerls.integrations.scan_target import blocked_scan_result, blocked_target_reason
+from dockerls.utils.executables import ExecutableNotFoundError, resolve_executable
+from dockerls.utils.subprocess_runner import (
+    VERSION_TIMEOUT_SECONDS,
+    OutputTooLargeError,
+    run_capture,
+)
+from dockerls.utils.validation import sanitize_image_name
+
+if TYPE_CHECKING:
+    from dockerls.infrastructure.evidence import EvidenceStore
+    from dockerls.infrastructure.network.host_guard import HostGuard
+
+
+class GrypeScanner(ScannerInterface):
+    def __init__(
+        self,
+        timeout: int = 300,
+        evidence: EvidenceStore | None = None,
+        guard: HostGuard | None = None,
+        workers: int = 1,
+    ):
+        self._timeout = timeout
+        self._evidence = evidence
+        self._version: str | None = None
+        self._skip_db_update = False
+        self._batch: EngineBatchScanner | None = None
+        self._workers = max(1, workers)
+        # Grype pulls the image itself, so the reference has to clear the
+        # network policy here or it never clears it at all.
+        self._guard = guard
+
+    @property
+    def batch(self) -> EngineBatchScanner:
+        """O caminho em lote, quando a engine Go estiver disponível.
+
+        O Grype não tem `--cache-dir`: a base dele mora num diretório único
+        e o que desliga a atualização automática são variáveis de ambiente.
+        Por isso o lote vai sem diretórios de cache e com o mesmo par de
+        variáveis que o caminho individual usa -- e a engine sabe que, sem
+        o lock BoltDB do Trivy, não há motivo para serializar.
+        """
+        if self._batch is None:
+            self._batch = EngineBatchScanner(
+                scanner="grype",
+                timeout_seconds=float(self._timeout),
+                skip_db_update=self._skip_db_update,
+                workers=self._workers,
+                guard=self._guard,
+                evidence=self._evidence,
+                raw_dir=None,
+                # Só o par que interessa, e nunca `_scan_env()`: aquele
+                # devolve `os.environ.copy()` inteiro, e mandá-lo pela
+                # fronteira escreveria DOCKERHUB_TOKEN e companhia dentro
+                # de um documento JSON. A engine soma o que recebe ao
+                # ambiente que ela própria herdou.
+                env=self._batch_env(),
+            )
+        return self._batch
+
+    def _batch_env(self) -> dict[str, str]:
+        """As variáveis que o lote precisa, e só elas."""
+        if not self._skip_db_update:
+            return {}
+        return {"GRYPE_DB_AUTO_UPDATE": "false", "GRYPE_CHECK_FOR_APP_UPDATE": "false"}
+
+    async def is_available(self) -> bool:
+        return shutil.which("grype") is not None
+
+    async def version(self) -> str:
+        """The scanner's own version string, memoised for the run.
+
+        Recorded so an analysis can be reconstructed: a score produced by
+        grype 0.48 and one produced by 0.58 are different measurements of
+        the same image, and until this existed nothing wrote down which one
+        happened. It is also what keeps a cached result from being served
+        across a scanner upgrade.
+
+        Returns "" when the binary is missing or does not answer -- an
+        unknown version, which the cache key treats as its own value rather
+        than as "same as before".
+        """
+        if self._version is not None:
+            return self._version
+        self._version = await self._read_version()
+        return self._version
+
+    async def _read_version(self) -> str:
+        try:
+            returncode, stdout, _ = await run_capture(
+                [resolve_executable("grype"), "--version"], timeout=VERSION_TIMEOUT_SECONDS
+            )
+        except (TimeoutError, OSError, ExecutableNotFoundError) as e:
+            logger.debug(f"Could not read grype version: {e}")
+            return ""
+        if returncode != 0 or not stdout:
+            return ""
+        # `<tool> --version` prints a banner; the first line carries the
+        # version and the rest is database metadata that changes on its own
+        # schedule and would churn the cache key for no reason.
+        return stdout.decode(errors="replace").strip().splitlines()[0].strip()
+
+    def _scan_env(self) -> dict[str, str] | None:
+        """Environment for a scan invocation.
+
+        Left at None until the DB has been refreshed once, so a scanner used
+        without `refresh_db()` still behaves exactly as before.
+        """
+        if not self._skip_db_update:
+            return None
+        env = os.environ.copy()
+        env["GRYPE_DB_AUTO_UPDATE"] = "false"
+        env["GRYPE_CHECK_FOR_APP_UPDATE"] = "false"
+        return env
+
+    async def refresh_db(self) -> bool:
+        """Update the vulnerability DB once, up front.
+
+        Grype otherwise checks its DB freshness on *every* invocation, which
+        is a network round trip per scan -- the dominant cost when
+        cross-validating several images. After this succeeds, scans run with
+        GRYPE_DB_AUTO_UPDATE=false so they go straight to matching.
+        """
+        try:
+            returncode, _, stderr = await run_capture(
+                [resolve_executable("grype"), "db", "update"], timeout=self._timeout
+            )
+            if returncode != 0:
+                logger.warning(f"Grype DB refresh failed: {stderr.decode()[:200]}")
+                return False
+        except (TimeoutError, OSError, ExecutableNotFoundError) as e:
+            logger.warning(f"Grype DB refresh failed: {e}")
+            return False
+
+        self._skip_db_update = True
+        logger.info("Grype DB ready; per-scan auto-update disabled")
+        return True
+
+    async def scan(self, image_reference: str) -> ScanResult:
+        safe_ref = sanitize_image_name(image_reference)
+        blocked = blocked_target_reason(safe_ref, self._guard)
+        if blocked:
+            return blocked_scan_result(safe_ref, "grype", blocked)
+        logger.info(f"Scanning {safe_ref} with Grype")
+        timestamp = datetime.now(tz=UTC).isoformat()
+
+        try:
+            returncode, stdout, stderr = await run_capture(
+                [resolve_executable("grype"), safe_ref, "-o", "json", "--quiet"],
+                timeout=self._timeout,
+                env=self._scan_env(),
+            )
+
+            if returncode != 0:
+                err = stderr.decode(errors="replace")[:500]
+                logger.error(f"Grype returned code {returncode} for {safe_ref}: {err}")
+                return ScanResult(
+                    image_reference=safe_ref,
+                    scanner="grype",
+                    scan_timestamp=timestamp,
+                    status=ScanStatus.ERROR,
+                    error_message=err,
+                    error_kind=classify_scanner_error(err),
+                )
+
+            if not stdout:
+                return ScanResult(
+                    image_reference=safe_ref,
+                    scanner="grype",
+                    scan_timestamp=timestamp,
+                    status=ScanStatus.ERROR,
+                    error_message="Grype produced no output",
+                    error_kind=ScanErrorKind.INVALID_OUTPUT,
+                )
+
+            raw = stdout.decode()
+            data = json.loads(raw)
+            result = self._parse_results(safe_ref, data)
+            if self._evidence is not None:
+                result.evidence_path = await self._evidence.record_scan(safe_ref, "grype", raw)
+            return result
+
+        except OutputTooLargeError as e:
+            # Unbounded output is not a measurement: the document was never
+            # read in full, so there is nothing to parse and nothing to
+            # conclude. INVALID_OUTPUT is already a non-verified state.
+            logger.error(f"Grype output exceeded the size limit for {safe_ref}: {e}")
+            return ScanResult(
+                image_reference=safe_ref,
+                scanner="grype",
+                scan_timestamp=timestamp,
+                status=ScanStatus.ERROR,
+                error_message=str(e),
+                error_kind=ScanErrorKind.INVALID_OUTPUT,
+            )
+        except TimeoutError:
+            logger.error(f"Grype scan timed out for {safe_ref}")
+            return ScanResult(
+                image_reference=safe_ref,
+                scanner="grype",
+                scan_timestamp=timestamp,
+                status=ScanStatus.TIMEOUT,
+                error_message=f"Scan exceeded {self._timeout}s timeout",
+                error_kind=ScanErrorKind.TIMEOUT,
+            )
+        except json.JSONDecodeError as e:
+            logger.error(f"Grype produced unparseable JSON for {safe_ref}: {e}")
+            return ScanResult(
+                image_reference=safe_ref,
+                scanner="grype",
+                scan_timestamp=timestamp,
+                status=ScanStatus.ERROR,
+                error_message=str(e),
+                error_kind=ScanErrorKind.INVALID_OUTPUT,
+            )
+        except ExecutableNotFoundError as e:
+            logger.error(f"Grype scan failed for {safe_ref}: {e}")
+            return ScanResult(
+                image_reference=safe_ref,
+                scanner="grype",
+                scan_timestamp=timestamp,
+                status=ScanStatus.ERROR,
+                error_message=str(e),
+                error_kind=ScanErrorKind.SCANNER_MISSING,
+            )
+        except OSError as e:
+            logger.error(f"Grype scan failed for {safe_ref}: {e}")
+            return ScanResult(
+                image_reference=safe_ref,
+                scanner="grype",
+                scan_timestamp=timestamp,
+                status=ScanStatus.ERROR,
+                error_message=str(e),
+                error_kind=classify_scanner_error(str(e)),
+            )
+
+    def _parse_results(self, image_ref: str, data: dict[str, Any]) -> ScanResult:
+        vulns: list[Vulnerability] = []
+        for match in data.get("matches", []):
+            vd = match.get("vulnerability", {})
+            sev_str = vd.get("severity", "Unknown").upper()
+            if sev_str == "NEGLIGIBLE":
+                sev_str = "LOW"
+            try:
+                severity = Severity(sev_str)
+            except ValueError:
+                severity = Severity.UNKNOWN
+
+            artifact = match.get("artifact", {})
+            fixed_versions = vd.get("fix", {}).get("versions", [])
+            fixed_version = fixed_versions[0] if fixed_versions else ""
+
+            cvss_score, cvss_source = self._extract_cvss(vd.get("cvss", []))
+
+            vulns.append(
+                Vulnerability(
+                    cve_id=vd.get("id", ""),
+                    severity=severity,
+                    cvss_score=cvss_score,
+                    cvss_source=cvss_source,
+                    package_name=artifact.get("name", ""),
+                    installed_version=artifact.get("version", ""),
+                    fixed_version=fixed_version,
+                    description=vd.get("description", "")[:200],
+                    package_type=str(artifact.get("type") or ""),
+                    target=str(artifact.get("locations", [{}])[0].get("path", ""))
+                    if artifact.get("locations")
+                    else "",
+                )
+            )
+
+        family, version = self._parse_distro(data)
+        return ScanResult(
+            image_reference=image_ref,
+            scanner="grype",
+            vulnerabilities=vulns,
+            scan_timestamp=datetime.now(tz=UTC).isoformat(),
+            os_family=family,
+            os_version=version,
+        )
+
+    @staticmethod
+    def _parse_distro(data: dict[str, Any]) -> tuple[str, str]:
+        """The base distribution Grype identified, from its `distro` block.
+
+        Same fact Trivy reports under `Metadata.OS`, spelled differently.
+        Normalising it here means the migration analysis reads one field
+        regardless of which scanner produced the result.
+        """
+        distro = data.get("distro")
+        if not isinstance(distro, dict):
+            return "", ""
+        return str(distro.get("name") or ""), str(distro.get("version") or "")
+
+    @staticmethod
+    def _extract_cvss(entries: list[dict[str, Any]]) -> tuple[float, str]:
+        """Deterministic CVSS selection: NVD source > any other vendor
+        source > first available, instead of an arbitrary max() across
+        differently-scored advisories. Returns (score, source) so the report
+        can say which base produced the number."""
+        if not entries:
+            return 0.0, ""
+
+        def base_score(entry: dict[str, Any]) -> float:
+            # Grype emits `"metrics": null` for advisories with no CVSS
+            # vector, and a null/non-numeric score must read as "unscored",
+            # not blow up the whole parse of an otherwise good scan.
+            metrics = entry.get("metrics") or {}
+            if not isinstance(metrics, dict):
+                return 0.0
+            try:
+                return float(metrics.get("baseScore", 0.0))
+            except (TypeError, ValueError):
+                return 0.0
+
+        for entry in entries:
+            source = str(entry.get("source", ""))
+            if "nvd" in source.lower():
+                return base_score(entry), source or "nvd"
+
+        return base_score(entries[0]), str(entries[0].get("source", ""))