dockerls 1.0.0__py3-none-any.whl

dockerls/integrations/endoflife/checker.py

@@ -0,0 +1,216 @@
+from __future__ import annotations
+
+import asyncio
+from datetime import UTC, datetime
+from typing import Any, cast
+
+import httpx
+from loguru import logger
+
+from dockerls.domain.interfaces.eol_checker import EOLCheckerInterface
+from dockerls.domain.value_objects.tristate import Tristate
+from dockerls.utils.retry import (
+    DEFAULT_BACKOFF_BASE,
+    DEFAULT_MAX_ATTEMPTS,
+    retry_policy,
+)
+
+# Docker Hub image name -> endoflife.date product slug. Docker Hub names and
+# endoflife.date slugs frequently diverge (e.g. "node" vs "nodejs").
+DOCKER_TO_ENDOFLIFE: dict[str, str] = {
+    "node": "nodejs",
+    "python": "python",
+    "postgres": "postgresql",
+    "postgresql": "postgresql",
+    "mysql": "mysql",
+    "mariadb": "mariadb",
+    "redis": "redis",
+    "nginx": "nginx",
+    "httpd": "apache",
+    "php": "php",
+    "ruby": "ruby",
+    "golang": "go",
+    "go": "go",
+    "django": "django",
+    "rails": "rails",
+    "ubuntu": "ubuntu",
+    "debian": "debian",
+    "alpine": "alpine",
+    "elasticsearch": "elasticsearch",
+    "rabbitmq": "rabbitmq",
+    "mongo": "mongodb",
+    "mongodb": "mongodb",
+    "dotnet": "dotnet",
+    "grafana": "grafana",
+    "prometheus": "prometheus",
+    "kafka": "kafka",
+    "cassandra": "cassandra",
+    "erlang": "erlang",
+    "haproxy": "haproxy",
+    "jenkins": "jenkins",
+    "traefik": "traefik",
+}
+
+
+def _version_parts(version: str) -> tuple[int, ...]:
+    """Parse the leading numeric dot-separated segments of a version string."""
+    parts: list[int] = []
+    for segment in version.split("."):
+        if segment.isdigit():
+            parts.append(int(segment))
+        else:
+            break
+    return tuple(parts)
+
+
+def _cycle_matches(version: str, cycle: str) -> bool:
+    """True if `version` falls under endoflife.date `cycle`, using SemVer-aware
+    segment comparison (not naive substring prefix matching)."""
+    vparts = _version_parts(version)
+    cparts = _version_parts(cycle)
+    if not vparts or not cparts:
+        return version == cycle
+    if len(cparts) > len(vparts):
+        return False
+    return vparts[: len(cparts)] == cparts
+
+
+class EndOfLifeChecker(EOLCheckerInterface):
+    BASE_URL = "https://endoflife.date/api"
+
+    def __init__(
+        self,
+        timeout: int = 15,
+        max_attempts: int = DEFAULT_MAX_ATTEMPTS,
+        backoff_base: float = DEFAULT_BACKOFF_BASE,
+    ):
+        self._timeout = timeout
+        self._max_attempts = max_attempts
+        self._backoff_base = backoff_base
+        self._cache: dict[str, list[dict[str, Any]]] = {}
+        # One lock per product slug. The memo below only closes the window
+        # *after* a response lands, and `recommend` asks `is_eol` + `is_lts`
+        # for ~100 tags concurrently -- so a single-product run fired up to
+        # 200 identical requests before the first one returned. That is the
+        # rate limiting (and the resulting inconsistent EOL verdicts within
+        # one run) that the memo was added to stop.
+        self._locks: dict[str, asyncio.Lock] = {}
+
+    def _resolve_product(self, product: str) -> str:
+        return DOCKER_TO_ENDOFLIFE.get(product.lower(), product.lower())
+
+    async def _fetch_product(self, product: str) -> list[dict[str, Any]]:
+        slug = self._resolve_product(product)
+        cached = self._cache.get(slug)
+        if cached is not None:
+            return cached
+
+        lock = self._locks.setdefault(slug, asyncio.Lock())
+        async with lock:
+            cached = self._cache.get(slug)
+            if cached is not None:
+                return cached
+            policy = retry_policy(self._max_attempts, self._backoff_base)
+            result: list[dict[str, Any]] = await policy(self._fetch_product_once, product)
+            return result
+
+    async def _fetch_product_once(self, product: str) -> list[dict[str, Any]]:
+        slug = self._resolve_product(product)
+        if slug in self._cache:
+            return self._cache[slug]
+        async with httpx.AsyncClient(timeout=self._timeout) as client:
+            try:
+                resp = await client.get(f"{self.BASE_URL}/{slug}.json")
+                if resp.status_code == 200:
+                    data = cast("list[dict[str, Any]]", resp.json())
+                    self._cache[slug] = data
+                    return data
+                if resp.status_code == 404:
+                    # O produto não existe no catálogo -- uma resposta
+                    # definitiva, e cacheável. Sem isso, cada uma das ~100
+                    # tags de uma execução repetia a mesma consulta perdida
+                    # (duas, contando is_eol e is_lts). Além do desperdício,
+                    # o volume provocava rate limiting: parte das tags então
+                    # recebia dados e parte recebia lista vazia, e a mesma
+                    # execução emitia vereditos de EOL inconsistentes entre
+                    # tags do mesmo produto.
+                    self._cache[slug] = []
+                    return []
+                logger.debug(f"EOL check for {slug} returned HTTP {resp.status_code}")
+                return []
+            except httpx.HTTPError as e:
+                logger.debug(f"EOL check failed for {slug}: {e}")
+                return []
+
+    def _find_cycle(self, cycles: list[dict[str, Any]], version: str) -> dict[str, Any] | None:
+        best: dict[str, Any] | None = None
+        best_len = -1
+        for cycle in cycles:
+            cycle_ver = str(cycle.get("cycle", ""))
+            if _cycle_matches(version, cycle_ver):
+                clen = len(_version_parts(cycle_ver))
+                if clen > best_len:
+                    best = cycle
+                    best_len = clen
+        return best
+
+    async def is_eol(self, product: str, version: str) -> bool:
+        """Whether this release is past its end-of-life date.
+
+        Kept as the boolean the interface has always published. It answers
+        `False` for "supported" *and* for "nobody could tell", which is why
+        `eol_status` exists -- callers that can represent the difference
+        should ask that instead.
+        """
+        return (await self.eol_status(product, version)).is_true
+
+    async def eol_status(self, product: str, version: str) -> Tristate:
+        """Three-valued end-of-life: TRUE, FALSE, or UNKNOWN.
+
+        Every path that previously answered `False` had one of two very
+        different meanings behind it, and collapsing them meant an image
+        whose lifecycle nobody could look up was scored, and gated, exactly
+        like one confirmed to be inside its support window:
+
+        * no version could be extracted from the tag -- UNKNOWN;
+        * the product is not in the catalogue (404) -- UNKNOWN;
+        * the catalogue could not be reached -- UNKNOWN;
+        * no cycle matches this version -- UNKNOWN;
+        * the cycle exists and carries a date or a boolean -- TRUE/FALSE,
+          the only two answers actually supported by evidence.
+        """
+        if not version:
+            return Tristate.UNKNOWN
+        cycles = await self._fetch_product(product)
+        if not cycles:
+            # Empty covers "unknown product" and "request failed" alike;
+            # neither is a statement that the release is supported.
+            return Tristate.UNKNOWN
+        cycle = self._find_cycle(cycles, version)
+        if cycle is None:
+            return Tristate.UNKNOWN
+
+        eol = cycle.get("eol")
+        if isinstance(eol, bool):
+            return Tristate.of(eol)
+        if isinstance(eol, str):
+            try:
+                eol_date = datetime.strptime(eol, "%Y-%m-%d").replace(tzinfo=UTC)
+            except ValueError:
+                # A date the catalogue publishes but we cannot parse is a
+                # gap in our reading of it, not a supported release.
+                logger.debug(f"Unparseable EOL date for {product} {version}: {eol!r}")
+                return Tristate.UNKNOWN
+            return Tristate.of(datetime.now(tz=UTC) > eol_date)
+        # `eol: false` arrives as a bool; anything else (null, missing) means
+        # the catalogue has not set a date yet.
+        return Tristate.UNKNOWN
+
+    async def is_lts(self, product: str, version: str) -> bool:
+        if not version:
+            return False
+        cycles = await self._fetch_product(product)
+        cycle = self._find_cycle(cycles, version)
+        if cycle is None:
+            return False
+        return bool(cycle.get("lts", False))

dockerls/integrations/engine/batch.py

@@ -0,0 +1,197 @@
+"""Medir um lote com a engine Go, sem tirar nenhuma decisão do Python.
+
+Este módulo é a fronteira, e ela é estreita de propósito. A engine recebe
+referências **já sanitizadas e já aprovadas pela política de rede**, e
+devolve resultados. Tudo que decide alguma coisa continua deste lado:
+
+* `sanitize_image_name` e o `HostGuard` são aplicados aqui, antes de a
+  requisição existir. `trivy image X` e `grype X` abrem os próprios
+  sockets, e foi para cobrir esse buraco que `scan_target.py` existe -- um
+  lote que contornasse o guard abriria a mesma porta em nome de desempenho;
+* `redact()` continua aqui. A engine guarda o JSON cru em arquivo `0600` e
+  devolve o caminho; quem decide o que vai para a evidência definitiva é o
+  `EvidenceStore`;
+* nada de score, tier, EOL ou inteligência de ameaça atravessa.
+
+Serve Trivy e Grype: o que difere entre os dois -- argv, forma do JSON,
+diretório de cache versus variável de ambiente -- está inteiramente do lado
+Go. Aqui os dois são o mesmo lote.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from loguru import logger
+
+from dockerls.integrations.engine.client import EngineClient, EngineTarget
+from dockerls.integrations.engine.locator import find_engine, probe
+from dockerls.integrations.scan_target import blocked_scan_result, blocked_target_reason
+from dockerls.utils.executables import ExecutableNotFoundError, resolve_executable
+from dockerls.utils.validation import sanitize_image_name
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable, Mapping, Sequence
+
+    from dockerls.domain.entities.scan_result import ScanResult
+    from dockerls.infrastructure.evidence import EvidenceStore
+    from dockerls.infrastructure.network.host_guard import HostGuard
+    from dockerls.integrations.engine.client import EngineOutcome
+
+
+@dataclass(frozen=True)
+class BatchOutcome:
+    """O que um lote medido pela engine produziu."""
+
+    #: Um resultado por alvo, na ordem em que foram pedidos.
+    results: list[ScanResult]
+    #: Scans que de fato aconteceram -- irmãos do mesmo digest não contam.
+    scans_performed: int
+    #: Alvos servidos pelo digest de um irmão.
+    duplicates_collapsed: int
+    #: Quanto o lote levou, medido pela engine.
+    wall_seconds: float
+
+
+class EngineBatchScanner:
+    """Mede um lote inteiro com uma travessia de processo, não N.
+
+    O ganho é bom ser exato sobre qual é: Trivy e Grype já são Go, e cada
+    scan continua custando os mesmos 1,2-2,5s. O que muda é o entorno --
+    criar e colher N processos, revezar o diretório de cache, coordenar o
+    dedup por digest --, que sai de N travessias Python<->processo para uma.
+    """
+
+    def __init__(
+        self,
+        *,
+        scanner: str,
+        timeout_seconds: float,
+        skip_db_update: bool,
+        workers: int,
+        guard: HostGuard | None = None,
+        evidence: EvidenceStore | None = None,
+        raw_dir: Path | None = None,
+        cache_dirs: Callable[[], Awaitable[Sequence[Path]]] | None = None,
+        env: Mapping[str, str] | None = None,
+    ) -> None:
+        self._scanner = scanner
+        self._timeout_seconds = timeout_seconds
+        self._skip_db_update = skip_db_update
+        self._workers = workers
+        self._guard = guard
+        self._evidence = evidence
+        self._raw_dir = raw_dir
+        self._cache_dirs = cache_dirs
+        self._env = dict(env or {})
+        self._client: EngineClient | None = None
+        self._resolved = False
+
+    def _resolve(self) -> EngineClient | None:
+        """Localiza o binário uma vez por processo.
+
+        `probe()` custa milissegundos e `find_engine()` toca o disco; fazer
+        isso por lote seria pagar a descoberta repetidamente por uma
+        resposta que não muda dentro de um run.
+        """
+        if self._resolved:
+            return self._client
+        self._resolved = True
+
+        path = find_engine()
+        if not path or not probe(path):
+            return None
+        try:
+            scanner_path = resolve_executable(self._scanner)
+        except ExecutableNotFoundError:
+            # Sem o scanner não há o que a engine dirija, e o caminho Python
+            # dará a mesma resposta -- SCANNER_MISSING -- por si.
+            return None
+
+        logger.info(f"Using the Go engine at {path} for {self._scanner} batch scans")
+        self._client = EngineClient(
+            engine_path=path,
+            scanner=self._scanner,
+            scanner_path=scanner_path,
+            timeout_seconds=self._timeout_seconds,
+            skip_db_update=self._skip_db_update,
+            raw_dir=self._raw_dir,
+            env=self._env,
+        )
+        return self._client
+
+    async def scan_batch(self, targets: Sequence[tuple[str, str]]) -> BatchOutcome | None:
+        """Mede `(referência, chave_de_dedup)`, ou None para usar o Python."""
+        client = self._resolve()
+        if client is None:
+            return None
+
+        # A política primeiro, e do lado de cá. Um alvo recusado é um
+        # resultado ERROR/BLOCKED_BY_POLICY como sempre foi, e não entra na
+        # requisição: o binário Go nunca vê a referência que a política
+        # negou.
+        blocked: dict[int, ScanResult] = {}
+        allowed: list[tuple[int, EngineTarget]] = []
+        for index, (reference, dedup_key) in enumerate(targets):
+            safe_ref = sanitize_image_name(reference)
+            reason = blocked_target_reason(safe_ref, self._guard)
+            if reason:
+                blocked[index] = blocked_scan_result(safe_ref, self._scanner, reason)
+                continue
+            allowed.append((index, EngineTarget(reference=safe_ref, dedup_key=dedup_key)))
+
+        cache_dirs = list(await self._cache_dirs()) if self._cache_dirs is not None else []
+        outcome = await client.scan_batch(
+            [target for _, target in allowed],
+            workers=self._workers,
+            cache_dirs=cache_dirs,
+        )
+        if outcome is None:
+            return None
+
+        results: list[ScanResult] = [None] * len(targets)  # type: ignore[list-item]
+        for position, (index, _) in enumerate(allowed):
+            results[index] = outcome.results[position]
+        for index, result in blocked.items():
+            results[index] = result
+
+        await self._archive_evidence(outcome)
+        return BatchOutcome(
+            results=results,
+            scans_performed=outcome.scans_performed,
+            duplicates_collapsed=outcome.duplicates_collapsed,
+            wall_seconds=outcome.wall_seconds,
+        )
+
+    async def _archive_evidence(self, outcome: EngineOutcome) -> None:
+        """Lê o JSON cru que a engine guardou, redige e arquiva.
+
+        A redação continua aqui de propósito: `redact()` é um controle de
+        segurança, e uma segunda implementação dele em Go seria uma segunda
+        a divergir desta. A engine só guarda o arquivo; quem decide o que
+        vai para o disco definitivo é o EvidenceStore.
+        """
+        evidence = self._evidence
+        if evidence is None:
+            return
+        for result, raw_path in zip(outcome.results, outcome.raw_paths, strict=True):
+            if not raw_path:
+                continue
+            try:
+                raw = await asyncio.to_thread(Path(raw_path).read_text, encoding="utf-8")
+            except OSError as e:
+                logger.warning(f"Could not read engine evidence at {raw_path}: {e}")
+                continue
+            finally:
+                # O temporário sai do disco em qualquer caminho: ele é o
+                # documento não redigido, e deixá-lo para trás desfaria a
+                # redação que acabou de acontecer.
+                with contextlib.suppress(OSError):
+                    await asyncio.to_thread(Path(raw_path).unlink)
+            result.evidence_path = await evidence.record_scan(
+                result.image_reference, self._scanner, raw
+            )