dockerls 1.0.0__py3-none-any.whl

dockerls/integrations/dhi/repository.py

@@ -0,0 +1,238 @@
+"""Docker Hardened Images as an image source.
+
+DHI is different from every other source this tool searches, in a way that
+shapes the whole integration: its *catalogue* is public (a GitHub
+repository of build definitions) while its *registry* is not (dhi.io refuses
+anonymous pulls). Discovery therefore works for everyone; scanning works
+only for a machine holding DHI credentials.
+
+That split is not a limitation to work around -- it is the exact case the
+rest of this codebase was built to handle honestly. A DHI candidate is
+discovered from the catalogue and enters the same pipeline as everything
+else. If the registry will not serve it, the scan fails, the candidate is
+reported as UNVERIFIED, and it is never ranked, never scored, and never
+called production ready. What a vendor says about its own hardening does not
+substitute for a measurement, and this provider is where that rule is at its
+most tempting to break.
+
+The candidate a definition produces carries the definition's declared
+metadata alongside it, so the CLI can explain *why* a DHI image is worth
+considering (non-root by declaration, a supported lifecycle, a small package
+set) while keeping every one of those claims labelled as a claim.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from loguru import logger
+
+from dockerls.domain.entities.image import DockerImage
+from dockerls.domain.interfaces.image_repository import ImageRepositoryInterface
+
+if TYPE_CHECKING:
+    from dockerls.domain.entities.declared_metadata import DeclaredImageMetadata
+    from dockerls.integrations.dhi.catalog import DHICatalogClient
+
+DHI = "Docker Hardened Images"
+DHI_REGISTRY = "dhi.io"
+
+#: Query name -> catalogue directory, where DHI names an image differently
+#: from the Docker Hub convention a user types.
+ALIASES = {
+    "nodejs": "node",
+    "python3": "python",
+    "golang": "go",
+    "openjdk": "jdk",
+    "postgresql": "postgres",
+}
+
+#: Variant suffixes that are not the image most users want. `-dev` ships a
+#: shell, a package manager and a compiler by design; `sfw`/`ent` are the
+#: FIPS and enterprise builds, which are not pullable at all without the
+#: corresponding entitlement. They are still discoverable by name -- they are
+#: just not what a bare `recommend node` should fan out to.
+DEPRIORITISED_MARKERS = ("dev", "sfw", "swf", "ent")
+
+
+class DHIRepository(ImageRepositoryInterface):
+    """Discovers candidates from the DHI catalogue's build definitions."""
+
+    source = DHI
+    host = DHI_REGISTRY
+
+    def __init__(self, catalog: DHICatalogClient, definition_limit: int = 12):
+        self._catalog = catalog
+        self._definition_limit = max(1, definition_limit)
+
+    def repository_for(self, image_name: str) -> str | None:
+        """Map a user's query onto a catalogue directory name.
+
+        Returns None for anything that already names a registry or a
+        namespace: `recommend ghcr.io/org/app` must not fan out to DHI, and
+        a slash in the query means the user has already been specific.
+        """
+        name = image_name.strip().strip("/").lower()
+        if not name or "/" in name or ":" in name:
+            return None
+        return ALIASES.get(name, name)
+
+    async def search_tags(self, image_name: str, limit: int = 100) -> list[DockerImage]:
+        catalogue_name = self.repository_for(image_name)
+        if catalogue_name is None:
+            return []
+
+        variants = await self._catalog.variants(catalogue_name)
+        if not variants:
+            # Empty has three causes and only one of them is about the
+            # image. Saying which one happened is the difference between
+            # "DHI publishes no hardened build of this" and "nobody asked",
+            # and the second must not be reported as the first.
+            state = self._catalog.index_state
+            if state.is_conclusive:
+                logger.info(f"{DHI}: the catalogue has no {catalogue_name}")
+            else:
+                logger.warning(
+                    f"{DHI}: no candidates for {catalogue_name}, and the catalogue index "
+                    f"is {state} -- this is an absence of an answer, not a finding that "
+                    "no hardened build exists"
+                )
+            return []
+
+        candidates: list[DockerImage] = []
+        for path in self._selected_paths(variants):
+            declared = await self._catalog.definition(path)
+            if declared is None:
+                continue
+            image = self._build_image(declared)
+            if image is None:
+                continue
+            candidates.append(image)
+            if len(candidates) >= limit:
+                break
+
+        logger.info(
+            f"{DHI}: {len(candidates)} candidate(s) for {catalogue_name} "
+            f"(catalogue @{self._catalog.revision or 'unknown'})"
+        )
+        return candidates
+
+    def _selected_paths(self, variants: dict[str, list[str]]) -> list[str]:
+        """Bound and order the definitions this query will read.
+
+        A popular image has dozens of definitions across OS variants and
+        build flavours. Fetching all of them would be a request each, so the
+        list is ordered -- runtime flavours before `-dev`, newest OS variant
+        first, newest version first -- and cut at `definition_limit`. The
+        ordering is total and deterministic, so two runs against the same
+        catalogue revision read the same files.
+        """
+        paths = [path for variant_paths in variants.values() for path in variant_paths]
+        paths.sort(key=_definition_rank)
+        return paths[: self._definition_limit]
+
+    def _build_image(self, declared: DeclaredImageMetadata) -> DockerImage | None:
+        """Turn a definition into a candidate pinned to its primary tag.
+
+        A definition publishes a dozen aliases of the same image (`22`,
+        `22.23`, `22.23.2-debian13`, ...). They are the same bytes, so
+        emitting one candidate per alias would multiply the scan work by
+        twelve for no additional information. The most specific tag is
+        chosen, because it is the one that stays meaningful over time.
+        """
+        tag = _primary_tag(declared.tags)
+        repository = _repository_name(declared)
+        if not tag or not repository:
+            return None
+        return DockerImage(
+            name=repository,
+            tag=tag,
+            source=DHI,
+            os=declared.os_id or "linux",
+            available_architectures=[p.split("/")[-1] for p in declared.platforms],
+            declared=declared,
+        )
+
+    async def get_image_metadata(self, image_name: str, tag: str) -> DockerImage | None:
+        for image in await self.search_tags(image_name, limit=self._definition_limit):
+            if image.tag == tag:
+                return image
+        return None
+
+    async def tag_exists(self, image_name: str, tag: str) -> bool | None:
+        """Whether the *catalogue* declares this tag.
+
+        Deliberately not a registry check: dhi.io refuses anonymous
+        requests, so this provider cannot confirm that a tag is actually
+        published. Returning True here would mean "the definition says this
+        tag exists", which is not what the caller is asking, so a catalogue
+        hit answers None -- unknown -- and the tag stands or falls on
+        whether the scanner could pull it.
+        """
+        del image_name, tag
+        return None
+
+    async def close(self) -> None:
+        await self._catalog.close()
+
+
+def _repository_name(declared: DeclaredImageMetadata) -> str:
+    """The repository a definition publishes to, if it is a DHI repository.
+
+    The definition states it as `image: dhi.io/node`. It is *checked* here
+    rather than trusted: a definition that names some other host would
+    otherwise let catalogue content redirect a scan at an arbitrary
+    registry. Anything that is not a `dhi.io/...` path yields no candidate.
+    """
+    repository = declared.registry_repository.strip().lower()
+    prefix = f"{DHI_REGISTRY}/"
+    if not repository.startswith(prefix):
+        return ""
+    path = repository[len(prefix) :]
+    if not path or not _REPOSITORY_PATH.match(path):
+        return ""
+    return f"{DHI_REGISTRY}/{path}"
+
+
+_REPOSITORY_PATH = re.compile(r"^[a-z0-9][a-z0-9._/-]{0,127}$")
+
+
+def _primary_tag(tags: tuple[str, ...]) -> str:
+    """The most specific tag a definition publishes.
+
+    Specificity is measured by dotted components then by length, so
+    `22.23.2-debian13` beats `22`. A pinned tag is the one worth recording
+    in a recommendation: `22` moves with every patch release, and a
+    recommendation that moves is a recommendation that was never verified.
+    """
+    if not tags:
+        return ""
+    return max(tags, key=lambda tag: (tag.count("."), len(tag), tag))
+
+
+def _definition_rank(path: str) -> tuple[int, str]:
+    """Order definitions: plain runtime flavours first, newest variant first.
+
+    The variant directory (`debian-13`, `alpine-3.24`) is reversed so the
+    highest version sorts first, and any definition whose file name carries
+    a de-prioritised marker sinks below the plain ones.
+    """
+    parts = path.split("/")
+    variant = parts[2] if len(parts) > 3 else ""
+    filename = parts[-1].rsplit(".", 1)[0]
+    markers = set(filename.lower().replace(".", "-").split("-"))
+    deprioritised = 1 if markers & set(DEPRIORITISED_MARKERS) else 0
+    # Reverse-lexicographic on variant and file name, achieved by negating
+    # the ordering with a descending key built from the strings themselves.
+    return (deprioritised, _descending(f"{variant}/{filename}"))
+
+
+def _descending(value: str) -> str:
+    """Key that sorts `value` in reverse order under an ascending sort.
+
+    Inverting each code point keeps the comparison total and stable without
+    needing a second sort pass or a reverse=True that would also flip the
+    de-prioritisation flag.
+    """
+    return "".join(chr(0x10FFFF - ord(ch)) for ch in value)

dockerls/integrations/dockerhub/client.py

@@ -0,0 +1,318 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from datetime import datetime
+from typing import TYPE_CHECKING, Any
+
+import httpx
+from loguru import logger
+
+from dockerls.domain.entities.image import DockerImage
+from dockerls.domain.interfaces.image_repository import ImageRepositoryInterface
+from dockerls.infrastructure.network.guarded_client import guarded_async_client
+from dockerls.integrations.dockerhub.urls import build_tag_api_url
+from dockerls.utils.retry import (
+    DEFAULT_BACKOFF_BASE,
+    DEFAULT_MAX_ATTEMPTS,
+    retry_policy,
+)
+from dockerls.utils.validation import sanitize_image_name
+
+if TYPE_CHECKING:
+    from dockerls.domain.interfaces.cache_store import CacheStoreInterface
+    from dockerls.infrastructure.network.host_guard import HostGuard
+
+# Anonymous Docker Hub requests are rate limited, and a `recommend` run
+# checks one tag per candidate. Existence of a tag changes rarely, so a
+# local TTL cache keeps repeated runs well under the limit. Kept shorter
+# than the analysis cache: a tag disappearing matters sooner than a score
+# going slightly stale.
+TAG_EXISTS_TTL_SECONDS = 6 * 3600
+
+
+class DockerHubClient(ImageRepositoryInterface):
+    BASE_URL = "https://hub.docker.com/v2"
+
+    def __init__(
+        self,
+        # Defaults vazios: "sem credencial", não uma senha embutida.
+        username: str = "",  # nosec B107
+        token: str = "",  # nosec B107
+        timeout: int = 30,
+        cache: CacheStoreInterface | None = None,
+        max_attempts: int = DEFAULT_MAX_ATTEMPTS,
+        backoff_base: float = DEFAULT_BACKOFF_BASE,
+        tag_ttl_seconds: int = TAG_EXISTS_TTL_SECONDS,
+        guard: HostGuard | None = None,
+    ):
+        # hub.docker.com is a fixed host, but it answers with redirects and
+        # this client follows them. A redirect target is chosen by the far
+        # end, so it is policed per hop rather than trusted because the
+        # first hop was fine.
+        self._guard = guard
+        self._username = username
+        self._token = token
+        self._timeout = timeout
+        self._auth_token: str = ""
+        self._cache = cache
+        self._max_attempts = max_attempts
+        self._backoff_base = backoff_base
+        self._tag_ttl_seconds = tag_ttl_seconds
+        self._client: httpx.AsyncClient | None = None
+        self._client_lock = asyncio.Lock()
+        # In-flight `tag_exists` checks, keyed by cache key. `_verify_tags`
+        # runs up to ten of them concurrently and several candidates can
+        # share a tag, so without this a cold cache means every one of them
+        # goes to the network before the first result is written back --
+        # a textbook stampede against a rate-limited API.
+        self._tag_checks: dict[str, asyncio.Future[bool | None]] = {}
+
+    async def _get_client(self) -> httpx.AsyncClient:
+        """One client for this repository's lifetime.
+
+        A fresh `AsyncClient` per call meant a new connection pool and a new
+        TLS handshake for every page of a listing and every tag check, with
+        nothing kept alive between them. Sharing one gives HTTP keep-alive
+        across the whole run.
+        """
+        if self._client is None:
+            async with self._client_lock:
+                if self._client is None:
+                    self._client = guarded_async_client(
+                        self._guard,
+                        timeout=self._timeout,
+                        headers={"Accept": "application/json"},
+                        follow_redirects=True,
+                    )
+        return self._client
+
+    def _auth_headers(self) -> dict[str, str]:
+        """Per-request auth, so a token obtained by a later `authenticate()`
+        applies to a client that was already created."""
+        return {"Authorization": f"Bearer {self._auth_token}"} if self._auth_token else {}
+
+    async def close(self) -> None:
+        """Release the shared connection pool."""
+        client, self._client = self._client, None
+        if client is not None:
+            await client.aclose()
+
+    async def authenticate(self) -> bool:
+        if not self._username or not self._token:
+            return False
+        try:
+            async with httpx.AsyncClient(timeout=self._timeout) as client:
+                resp = await client.post(
+                    f"{self.BASE_URL}/users/login/",
+                    json={"username": self._username, "password": self._token},
+                )
+                if resp.status_code == 200:
+                    self._auth_token = resp.json().get("token", "")
+                    return bool(self._auth_token)
+        except (httpx.HTTPError, ValueError) as e:
+            # ValueError covers a 200 whose body is not JSON -- a captive
+            # portal or proxy error page. Failing to authenticate degrades to
+            # anonymous access; it must never abort the command.
+            logger.warning(f"Docker Hub auth failed: {e}")
+        return False
+
+    async def _get_json(self, client: httpx.AsyncClient, url: str) -> httpx.Response:
+        """Perform a single GET with retry scoped to *this* request only, so
+        a transient failure deep into a paginated fetch doesn't force the
+        whole listing to restart from page one. Honors Retry-After on 429.
+
+        The policy is built per call so `retry_max_attempts` and
+        `retry_backoff_base` reach it; as a decorator it was fixed at
+        import time and the settings could never apply.
+        """
+        policy = retry_policy(self._max_attempts, self._backoff_base)
+        resp: httpx.Response = await policy(self._get_once, client, url)
+        return resp
+
+    async def _get_once(self, client: httpx.AsyncClient, url: str) -> httpx.Response:
+        resp = await client.get(url, headers=self._auth_headers())
+        if resp.status_code == 429:
+            retry_after = resp.headers.get("Retry-After")
+            wait_s = float(retry_after) if retry_after and retry_after.isdigit() else 2.0
+            logger.warning(f"Rate limited by Docker Hub, waiting {wait_s}s")
+            await asyncio.sleep(wait_s)
+            resp.raise_for_status()
+        return resp
+
+    @staticmethod
+    def _parse_images(images: list[dict[str, Any]]) -> tuple[int, str, str, list[str]]:
+        """Return (size, digest, primary_architecture, all_architectures)."""
+        archs = [img.get("architecture", "unknown") for img in images]
+        for img in images:
+            if img.get("architecture") == "amd64":
+                return img.get("size", 0), img.get("digest", ""), "amd64", archs
+        if images:
+            first = images[0]
+            return (
+                first.get("size", 0),
+                first.get("digest", ""),
+                first.get("architecture", "unknown"),
+                archs,
+            )
+        return 0, "", "amd64", archs
+
+    async def search_tags(self, image_name: str, limit: int = 100) -> list[DockerImage]:
+        safe_name = sanitize_image_name(image_name)
+        namespace = "library" if "/" not in safe_name else safe_name.split("/")[0]
+        repo = safe_name if "/" not in safe_name else safe_name.split("/", 1)[1]
+
+        tags: list[DockerImage] = []
+        page_size = min(limit, 100)
+        url: str | None = (
+            f"{self.BASE_URL}/repositories/{namespace}/{repo}/tags/"
+            f"?page_size={page_size}&ordering=last_updated"
+        )
+
+        client = await self._get_client()
+        while url and len(tags) < limit:
+            try:
+                resp = await self._get_json(client, url)
+                if resp.status_code == 404:
+                    logger.warning(f"Image not found: {safe_name}")
+                    return []
+                resp.raise_for_status()
+                data = resp.json()
+
+                for tag_data in data.get("results", []):
+                    tag_name = tag_data.get("name", "")
+                    if not tag_name:
+                        continue
+
+                    last_updated = None
+                    lu_str = tag_data.get("last_updated")
+                    if lu_str:
+                        with contextlib.suppress(ValueError):
+                            last_updated = datetime.fromisoformat(lu_str.replace("Z", "+00:00"))
+
+                    size, digest, arch, archs = self._parse_images(tag_data.get("images", []))
+
+                    tags.append(
+                        DockerImage(
+                            name=safe_name,
+                            tag=tag_name,
+                            digest=digest,
+                            size_bytes=size,
+                            architecture=arch,
+                            available_architectures=archs,
+                            last_updated=last_updated,
+                            is_official=namespace == "library",
+                        )
+                    )
+
+                url = data.get("next")
+            except httpx.HTTPError as e:
+                # Network blips or non-429 API errors degrade to a
+                # partial result (whatever pages already fetched)
+                # instead of crashing the whole search.
+                logger.error(f"Docker Hub API error, returning partial results: {e}")
+                break
+
+        return tags[:limit]
+
+    async def tag_exists(self, image_name: str, tag: str) -> bool | None:
+        """Confirm `tag` really exists on Docker Hub.
+
+        Returns True/False on a definitive API answer, and None when the
+        answer is unknown -- the image is not hosted on Docker Hub, or the
+        network call failed. `None` must not be reported to the user as
+        "tag missing"; it means "not verified".
+
+        Concurrent checks for the same tag are coalesced onto one request:
+        `_verify_tags` fires up to ten of these at once, and on a cold cache
+        every one of them would otherwise reach the network before the first
+        answer was written back.
+        """
+        url = build_tag_api_url(image_name, tag)
+        if not url:
+            return None
+
+        cache_key = f"hubtag:{image_name}:{tag}"
+        if self._cache:
+            cached = await self._cache.get(cache_key)
+            if isinstance(cached, bool):
+                return cached
+
+        in_flight = self._tag_checks.get(cache_key)
+        if in_flight is not None:
+            return await asyncio.shield(in_flight)
+
+        future: asyncio.Future[bool | None] = asyncio.get_running_loop().create_future()
+        self._tag_checks[cache_key] = future
+        try:
+            result = await self._check_tag(image_name, tag, url, cache_key)
+        except BaseException as e:
+            future.set_exception(e)
+            # Nobody may be awaiting this future; without retrieving the
+            # exception asyncio logs a spurious "never retrieved" warning.
+            future.exception()
+            raise
+        else:
+            future.set_result(result)
+            return result
+        finally:
+            self._tag_checks.pop(cache_key, None)
+
+    async def _check_tag(self, image_name: str, tag: str, url: str, cache_key: str) -> bool | None:
+        try:
+            client = await self._get_client()
+            resp = await self._get_json(client, url)
+        except httpx.HTTPError as e:
+            logger.warning(f"Could not verify tag {image_name}:{tag} on Docker Hub: {e}")
+            return None
+
+        if resp.status_code == 404:
+            exists = False
+        elif resp.is_success:
+            exists = True
+        else:
+            logger.warning(
+                f"Unexpected status {resp.status_code} verifying {image_name}:{tag} on Docker Hub"
+            )
+            return None
+
+        if self._cache:
+            await self._cache.set(cache_key, exists, ttl_seconds=self._tag_ttl_seconds)
+        return exists
+
+    async def get_image_metadata(self, image_name: str, tag: str) -> DockerImage | None:
+        safe_name = sanitize_image_name(image_name)
+        namespace = "library" if "/" not in safe_name else safe_name.split("/")[0]
+        repo = safe_name if "/" not in safe_name else safe_name.split("/", 1)[1]
+
+        url = f"{self.BASE_URL}/repositories/{namespace}/{repo}/tags/{tag}"
+
+        client = await self._get_client()
+        try:
+            resp = await self._get_json(client, url)
+            if resp.status_code == 404:
+                return None
+            resp.raise_for_status()
+            data = resp.json()
+
+            last_updated = None
+            lu_str = data.get("last_updated")
+            if lu_str:
+                with contextlib.suppress(ValueError):
+                    last_updated = datetime.fromisoformat(lu_str.replace("Z", "+00:00"))
+
+            size, digest, arch, archs = self._parse_images(data.get("images", []))
+
+            return DockerImage(
+                name=safe_name,
+                tag=tag,
+                digest=digest,
+                size_bytes=size,
+                architecture=arch,
+                available_architectures=archs,
+                last_updated=last_updated,
+                is_official=namespace == "library",
+            )
+        except httpx.HTTPError as e:
+            logger.error(f"Failed to get metadata for {safe_name}:{tag}: {e}")
+            return None

dockerls/integrations/dockerhub/urls.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from urllib.parse import quote
+
+HUB_WEB_BASE = "https://hub.docker.com"
+HUB_API_BASE = "https://hub.docker.com/v2"
+
+
+def _is_registry_host(segment: str) -> bool:
+    """A leading path segment is a registry host (ghcr.io, cgr.dev,
+    registry.internal:5000) rather than a Docker Hub namespace when it
+    carries a dot or a port."""
+    return "." in segment or ":" in segment
+
+
+def split_repository(image: str) -> tuple[str, str] | None:
+    """Split a Docker Hub image name into (namespace, repository).
+
+    Official images ("node") map to the implicit "library" namespace, which
+    is what the Hub API expects. Returns None for references that are not
+    hosted on Docker Hub (e.g. "ghcr.io/org/app", "cgr.dev/chainguard/node")
+    so callers never build a link that would 404.
+    """
+    name = image.strip().strip("/")
+    if not name:
+        return None
+
+    # Strip any tag/digest suffix that leaked into the name.
+    if "@" in name:
+        name = name.split("@", 1)[0]
+
+    parts = name.split("/")
+    if _is_registry_host(parts[0]) and parts[0] not in ("docker.io", "index.docker.io"):
+        return None
+    if parts[0] in ("docker.io", "index.docker.io"):
+        parts = parts[1:]
+
+    if not parts or not parts[0]:
+        return None
+    if len(parts) == 1:
+        return "library", parts[0]
+    if len(parts) == 2:
+        return parts[0], parts[1]
+    return None
+
+
+def build_dockerhub_url(image: str, tag: str) -> str:
+    """Return the canonical Docker Hub web URL for `image`:`tag`.
+
+    Official images live under the `_/<repo>` path and expose their tags via
+    a query parameter; everything else lives under `r/<ns>/<repo>/tags`.
+    Returns "" when the image is not on Docker Hub.
+    """
+    split = split_repository(image)
+    if split is None:
+        return ""
+    namespace, repo = split
+    safe_tag = quote(tag, safe="")
+
+    if namespace == "library":
+        return f"{HUB_WEB_BASE}/_/{quote(repo, safe='')}?tab=tags&name={safe_tag}"
+    ns = quote(namespace, safe="")
+    return f"{HUB_WEB_BASE}/r/{ns}/{quote(repo, safe='')}/tags?name={safe_tag}"
+
+
+def build_tag_api_url(image: str, tag: str) -> str:
+    """Return the Hub API endpoint that confirms a tag actually exists."""
+    split = split_repository(image)
+    if split is None:
+        return ""
+    namespace, repo = split
+    return (
+        f"{HUB_API_BASE}/repositories/{quote(namespace, safe='')}/"
+        f"{quote(repo, safe='')}/tags/{quote(tag, safe='')}"
+    )