updates2mqtt 1.7.0__py3-none-any.whl → 1.7.2__py3-none-any.whl

updates2mqtt/integrations/docker.py

@@ -1,9 +1,10 @@
+import random
+import re
 import subprocess
 import time
 import typing
 from collections.abc import AsyncGenerator, Callable
 from enum import Enum
-from http import HTTPStatus
 from pathlib import Path
 from threading import Event
 from typing import Any, cast
@@ -11,32 +12,39 @@ from typing import Any, cast
 import docker
 import docker.errors
 import structlog
-from docker.auth import resolve_repository_name
 from docker.models.containers import Container
 
 from updates2mqtt.config import (
-    NO_KNOWN_IMAGE,
+    SEMVER_RE,
+    UNKNOWN_VERSION,
+    VERSION_RE,
     DockerConfig,
     NodeConfig,
     PackageUpdateInfo,
     PublishPolicy,
+    RegistryAPI,
     UpdatePolicy,
+    VersionPolicy,
 )
+from updates2mqtt.helpers import Selection, Throttler
 from updates2mqtt.integrations.docker_enrich import (
-    AuthError,
     CommonPackageEnricher,
+    ContainerDistributionAPIVersionLookup,
     DefaultPackageEnricher,
-    LabelEnricher,
+    DockerClientVersionLookup,
+    DockerImageInfo,
+    DockerServiceDetails,
     LinuxServerIOPackageEnricher,
+    LocalContainerInfo,
     PackageEnricher,
     SourceReleaseEnricher,
 )
-from updates2mqtt.model import Discovery, ReleaseProvider, Selection, VersionPolicy, select_version
+from updates2mqtt.model import Discovery, ReleaseDetail, ReleaseProvider
 
-from .git_utils import git_check_update_available, git_iso_timestamp, git_local_version, git_pull, git_trust
+from .git_utils import git_check_update_available, git_iso_timestamp, git_local_digest, git_pull, git_trust
 
 if typing.TYPE_CHECKING:
-    from docker.models.images import Image, RegistryData
+    from docker.models.images import Image
 
 # distinguish docker build from docker pull?
 
@@ -59,7 +67,7 @@ class ContainerCustomization:
     env_prefix: str = "UPD2MQTT_"
 
     def __init__(self, container: Container) -> None:
-        self.update: str = "PASSIVE"
+        self.update: UpdatePolicy = UpdatePolicy.PASSIVE  # was known as UPD2MQTT_UPDATE before policies and labels
         self.git_repo_path: str | None = None
         self.picture: str | None = None
         self.relnotes: str | None = None
@@ -108,11 +116,11 @@ class ContainerCustomization:
                         setattr(self, attr, v.upper() in ("TRUE", "YES", "1"))
                     elif isinstance(getattr(self, attr), VersionPolicy):
                         setattr(self, attr, VersionPolicy[v.upper()])
+                    elif isinstance(getattr(self, attr), UpdatePolicy):
+                        setattr(self, attr, UpdatePolicy[v.upper()])
                     else:
                         setattr(self, attr, v)
 
-        self.update = self.update.upper()
-
 
 class DockerProvider(ReleaseProvider):
     def __init__(
@@ -126,16 +134,19 @@ class DockerProvider(ReleaseProvider):
         self.cfg: DockerConfig = cfg
 
         # TODO: refresh discovered packages periodically
-        self.pause_api_until: dict[str, float] = {}
-        self.api_throttle_pause: int = cfg.default_api_backoff
+        self.throttler = Throttler(self.cfg.default_api_backoff, self.log, self.stopped)
         self.self_bounce: Event | None = self_bounce
         self.pkg_enrichers: list[PackageEnricher] = [
             CommonPackageEnricher(self.cfg),
             LinuxServerIOPackageEnricher(self.cfg),
             DefaultPackageEnricher(self.cfg),
         ]
-        self.label_enricher = LabelEnricher()
+        self.docker_client_image_lookup = DockerClientVersionLookup(
+            self.client, self.throttler, self.cfg.registry, self.cfg.default_api_backoff
+        )
+        self.registry_image_lookup = ContainerDistributionAPIVersionLookup(self.throttler, self.cfg.registry)
         self.release_enricher = SourceReleaseEnricher()
+        self.local_info_builder = LocalContainerInfo()
 
     def initialize(self) -> None:
         for enricher in self.pkg_enrichers:
@@ -152,19 +163,21 @@ class DockerProvider(ReleaseProvider):
 
     def fetch(self, discovery: Discovery) -> None:
         logger = self.log.bind(container=discovery.name, action="fetch")
+        installed_info: DockerImageInfo | None = cast("DockerImageInfo|None", discovery.current_detail)
+        service_info: DockerServiceDetails | None = cast("DockerServiceDetails|None", discovery.installation_detail)
 
-        image_ref: str | None = discovery.custom.get("image_ref")
-        platform: str | None = discovery.custom.get("platform")
-        if discovery.custom.get("can_pull") and image_ref:
+        image_ref: str | None = installed_info.ref if installed_info else None
+        platform: str | None = installed_info.platform if installed_info else None
+        if discovery.can_pull and image_ref:
             logger.info("Pulling", image_ref=image_ref, platform=platform)
             image: Image = self.client.images.pull(image_ref, platform=platform, all_tags=False)
             if image:
                 logger.info("Pulled", image_id=image.id, image_ref=image_ref, platform=platform)
             else:
                 logger.warn("Unable to pull", image_ref=image_ref, platform=platform)
-        elif discovery.can_build:
-            compose_path: str | None = discovery.custom.get("compose_path")
-            git_repo_path: str | None = discovery.custom.get("git_repo_path")
+        elif discovery.can_build and service_info:
+            compose_path: str | None = service_info.compose_path
+            git_repo_path: str | None = service_info.git_repo_path
             logger.debug("can_build check", git_repo=git_repo_path)
             if not compose_path or not git_repo_path:
                 logger.warn("No compose path or git repo path configured, skipped build")
@@ -172,10 +185,7 @@ class DockerProvider(ReleaseProvider):
 
             full_repo_path: Path = self.full_repo_path(compose_path, git_repo_path)
             if git_pull(full_repo_path, Path(self.node_cfg.git_path)):
-                if compose_path:
-                    self.build(discovery, compose_path)
-                else:
-                    logger.warn("No compose path configured, skipped build")
+                self.build(discovery)
             else:
                 logger.debug("Skipping git_pull, no update")
 
@@ -186,14 +196,19 @@ class DockerProvider(ReleaseProvider):
             return Path(compose_path) / git_repo_path
         return Path(git_repo_path)
 
-    def build(self, discovery: Discovery, compose_path: str) -> bool:
+    def build(self, discovery: Discovery) -> bool:
         logger = self.log.bind(container=discovery.name, action="build")
-        logger.info("Building", compose_path=compose_path)
+        service_info: DockerServiceDetails | None = cast("DockerServiceDetails|None", discovery.installation_detail)
+
+        if not service_info or not service_info.compose_path:
+            logger.warn("No service_info available on compose")
+            return False
+        logger.info("Building", compose_path=service_info.compose_path, service=service_info.compose_service)
         return self.execute_compose(
             command=DockerComposeCommand.BUILD,
             args="",
-            service=discovery.custom.get("compose_service"),
-            cwd=compose_path,
+            service=service_info.compose_service,
+            cwd=service_info.compose_path,
             logger=logger,
         )
 
@@ -227,16 +242,28 @@ class DockerProvider(ReleaseProvider):
 
     def restart(self, discovery: Discovery) -> bool:
         logger = self.log.bind(container=discovery.name, action="restart")
-        if self.self_bounce is not None and (
-            "ghcr.io/rhizomatics/updates2mqtt" in discovery.custom.get("image_ref", "")
-            or discovery.custom.get("git_repo_path", "").endswith("updates2mqtt")
+        installed_info: DockerImageInfo | None = cast("DockerImageInfo|None", discovery.current_detail)
+        service_info: DockerServiceDetails | None = cast("DockerServiceDetails|None", discovery.installation_detail)
+
+        if (
+            self.self_bounce is not None
+            and installed_info
+            and service_info
+            and (
+                "ghcr.io/rhizomatics/updates2mqtt" in installed_info.ref
+                or (service_info.git_repo_path and service_info.git_repo_path.endswith("updates2mqtt"))
+            )
         ):
             logger.warning("Attempting to self-bounce")
             self.self_bounce.set()
-        compose_path = discovery.custom.get("compose_path")
-        compose_service: str | None = discovery.custom.get("compose_service")
+        if service_info is None:
+            return False
         return self.execute_compose(
-            command=DockerComposeCommand.UP, args="--detach --yes", service=compose_service, cwd=compose_path, logger=logger
+            command=DockerComposeCommand.UP,
+            args="--detach --yes",
+            service=service_info.compose_service,
+            cwd=service_info.compose_path,
+            logger=logger,
         )
 
     def rescan(self, discovery: Discovery) -> Discovery | None:
@@ -245,7 +272,7 @@ class DockerProvider(ReleaseProvider):
             c: Container = self.client.containers.get(discovery.name)
             if c:
                 rediscovery = self.analyze(c, discovery.session, previous_discovery=discovery)
-                if rediscovery:
+                if rediscovery and not rediscovery.throttled:
                     self.discoveries[rediscovery.name] = rediscovery
                     return rediscovery
             logger.warn("Unable to find container for rescan")
@@ -255,22 +282,9 @@ class DockerProvider(ReleaseProvider):
             logger.exception("Docker API error retrieving container")
         return None
 
-    def check_throttle(self, repo_id: str) -> bool:
-        if self.pause_api_until.get(repo_id) is not None:
-            if self.pause_api_until[repo_id] < time.time():
-                del self.pause_api_until[repo_id]
-                self.log.info("%s throttling wait complete", repo_id)
-            else:
-                self.log.debug("%s throttling has %s secs left", repo_id, self.pause_api_until[repo_id] - time.time())
-                return True
-        return False
-
     def analyze(self, c: Container, session: str, previous_discovery: Discovery | None = None) -> Discovery | None:
         logger = self.log.bind(container=c.name, action="analyze")
 
-        image_ref: str | None = None
-        image_name: str | None = None
-        local_versions = None
         if c.attrs is None or not c.attrs:
             logger.warn("No container attributes found, discovery rejected")
             return None
@@ -282,252 +296,150 @@ class DockerProvider(ReleaseProvider):
         if customization.ignore:
             logger.info("Container ignored due to UPD2MQTT_IGNORE setting")
             return None
-
-        image: Image | None = c.image
-        repo_id: str = "DEFAULT"
-        if image is not None and image.tags and len(image.tags) > 0:
-            image_ref = image.tags[0]
-        else:
-            image_ref = c.attrs.get("Config", {}).get("Image")
-        if image_ref is None:
-            logger.warn("No image or image attributes found")
-        else:
-            repo_id, _ = resolve_repository_name(image_ref)
-            try:
-                image_name = image_ref.split(":")[0]
-            except Exception as e:
-                logger.warn("No tags found (%s) : %s", image, e)
-            if image is not None and image.attrs is not None:
-                try:
-                    local_versions = [i.split("@")[1][7:19] for i in image.attrs["RepoDigests"]]
-                except Exception as e:
-                    logger.warn("Cannot determine local version: %s", e)
-                    logger.warn("RepoDigests=%s", image.attrs.get("RepoDigests"))
-
-        selection = Selection(self.cfg.image_ref_select, image_ref)
-        publish_policy: PublishPolicy = PublishPolicy.MQTT if not selection.result else PublishPolicy.HOMEASSISTANT
-        version_policy: VersionPolicy = VersionPolicy.AUTO if not customization.version_policy else customization.version_policy
-
-        if customization.update == "AUTO":
+        version_policy: VersionPolicy = (
+            self.cfg.version_policy if not customization.version_policy else customization.version_policy
+        )
+        if customization.update == UpdatePolicy.AUTO:
             logger.debug("Auto update policy detected")
-            update_policy: UpdatePolicy = UpdatePolicy.AUTO
-        else:
-            update_policy = UpdatePolicy.PASSIVE
+        update_policy: UpdatePolicy = customization.update or UpdatePolicy.PASSIVE
 
-        platform: str = "Unknown"
-        pkg_info: PackageUpdateInfo = self.default_metadata(image_name, image_ref=image_ref)
+        local_info: DockerImageInfo
+        service_info: DockerServiceDetails
+        local_info, service_info = self.local_info_builder.build_image_info(c)
+        pkg_info: PackageUpdateInfo = self.default_metadata(local_info)
 
         try:
-            picture_url: str | None = customization.picture or pkg_info.logo_url
-            relnotes_url: str | None = customization.relnotes or pkg_info.release_notes_url
-            release_summary: str | None = None
-
-            if image is not None and image.attrs is not None:
-                platform = "/".join(
-                    filter(
-                        None,
-                        [
-                            image.attrs["Os"],
-                            image.attrs["Architecture"],
-                            image.attrs.get("Variant"),
-                        ],
-                    ),
-                )
-
-            reg_data: RegistryData | None = None
-            latest_digest: str | None = NO_KNOWN_IMAGE
-            latest_version: str | None = None
-
-            registry_throttled: bool = self.check_throttle(repo_id)
-
-            if image_ref and local_versions and not registry_throttled:
-                retries_left = 3
-                while reg_data is None and retries_left > 0 and not self.stopped.is_set():
-                    try:
-                        logger.debug("Fetching registry data", image_ref=image_ref)
-                        reg_data = self.client.images.get_registry_data(image_ref)
-                        logger.debug(
-                            "Registry Data: id:%s,image:%s, attrs:%s",
-                            reg_data.id,
-                            reg_data.image_name,
-                            reg_data.attrs,
-                        )
-                        latest_digest = reg_data.short_id[7:] if reg_data else None
-
-                    except docker.errors.APIError as e:
-                        if e.status_code == HTTPStatus.TOO_MANY_REQUESTS:
-                            retry_secs: int
-                            try:
-                                retry_secs = int(e.response.headers.get("Retry-After", self.api_throttle_pause))  # type: ignore[union-attr]
-                            except:  # noqa: E722
-                                retry_secs = self.api_throttle_pause
-                            logger.warn("Docker Registry throttling requests for %s seconds, %s", retry_secs, e.explanation)
-                            self.pause_api_until[repo_id] = time.time() + retries_left
-                            return None
-                        retries_left -= 1
-                        if retries_left == 0 or e.is_client_error():
-                            logger.warn("Failed to fetch registry data: [%s] %s", e.errno, e.explanation)
-                        else:
-                            logger.debug("Failed to fetch registry data, retrying: %s", e)
-
-            installed_digest: str | None = NO_KNOWN_IMAGE
-            installed_version: str | None = None
-            if local_versions:
-                # might be multiple RepoDigests if image has been pulled multiple times with diff manifests
-                installed_digest = latest_digest if latest_digest in local_versions else local_versions[0]
-                logger.debug(f"Setting local digest to {installed_digest}, local_versions:{local_versions}")
-
-            def save_if_set(key: str, val: str | None) -> None:
-                if val is not None:
-                    custom[key] = val
-
-            image_ref = image_ref or ""
-
-            custom: dict[str, str | bool | int | list[str] | dict[str, Any] | None] = {}
-            custom["platform"] = platform
-            custom["image_ref"] = image_ref
-            custom["installed_digest"] = installed_digest
-            custom["latest_digest"] = latest_digest
-            custom["repo_id"] = repo_id
-            custom["git_repo_path"] = customization.git_repo_path
-
-            if c.labels:
-                save_if_set("compose_path", c.labels.get("com.docker.compose.project.working_dir"))
-                save_if_set("compose_version", c.labels.get("com.docker.compose.version"))
-                save_if_set("compose_service", c.labels.get("com.docker.compose.service"))
-                save_if_set("documentation_url", c.labels.get("org.opencontainers.image.documentation"))
-                save_if_set("description", c.labels.get("org.opencontainers.image.description"))
-                save_if_set("current_image_created", c.labels.get("org.opencontainers.image.created"))
-                save_if_set("current_image_version", c.labels.get("org.opencontainers.image.version"))
-                save_if_set("vendor", c.labels.get("org.opencontainers.image.vendor"))
-                installed_version = c.labels.get("org.opencontainers.image.version")
+            service_info.git_repo_path = customization.git_repo_path
+
+            registry_selection = Selection(self.cfg.registry_select, local_info.index_name)
+            latest_info: DockerImageInfo
+            if local_info.pinned:
+                logger.debug("Skipping registry fetch for local pinned image, %s", local_info.ref)
+                latest_info = local_info.reuse()
+            elif registry_selection and local_info.ref and not local_info.local_build:
+                if self.cfg.registry.api == RegistryAPI.DOCKER_CLIENT:
+                    latest_info = self.docker_client_image_lookup.lookup(local_info)
+                elif self.cfg.registry.api == RegistryAPI.OCI_V2:
+                    latest_info = self.registry_image_lookup.lookup(local_info, token=customization.registry_token)
+                elif self.cfg.registry.api == RegistryAPI.OCI_V2_MINIMAL:
+                    latest_info = self.registry_image_lookup.lookup(
+                        local_info, token=customization.registry_token, minimal=True
+                    )
+                else:  # assuming RegistryAPI.DISABLED
+                    logger.debug(f"Skipping registry check, disabled in config {self.cfg.registry.api}")
+                    latest_info = local_info.reuse()
+            elif local_info.local_build:
+                # assume its a locally built image if no RepoDigests available
+                latest_info = local_info.reuse()
+                latest_info.short_digest = None
+                latest_info.image_digest = None
             else:
-                logger.debug("No annotations found on local container")
-                # save_if_set("apt_pkgs", c_env.get("UPD2MQTT_APT_PKGS"))
+                logger.debug("Registry selection rules suppressed metadata lookup")
+                latest_info = local_info.reuse()
 
-            if latest_digest is None or latest_digest == NO_KNOWN_IMAGE or registry_throttled:
-                logger.debug(
-                    "Skipping image manifest enrichment",
-                    latest_digest=latest_digest,
-                    image_ref=image_ref,
-                    platform=platform,
-                    throttled=registry_throttled,
-                )
-            else:
-                os, arch = platform.split("/")[:2] if "/" in platform else (platform, "Unknown")
-                try:
-                    annotations: dict[str, str] = self.label_enricher.fetch_annotations(
-                        image_ref, os, arch, token=customization.registry_token
-                    )
-                except AuthError as e:
-                    logger.warning("Authentication error prevented Docker Registry entichment: %s", e)
-                    annotations = {}
-
-                if annotations:
-                    save_if_set("latest_image_created", annotations.get("org.opencontainers.image.created"))
-                    save_if_set("source", annotations.get("org.opencontainers.image.source"))
-                    save_if_set("documentation_url", annotations.get("org.opencontainers.image.documentation"))
-                    save_if_set("description", annotations.get("org.opencontainers.image.description"))
-                    save_if_set("latest_image_version", annotations.get("org.opencontainers.image.version"))
-                    save_if_set("vendor", annotations.get("org.opencontainers.image.vendor"))
-                    latest_version = annotations.get("org.opencontainers.image.version")
-                    custom.update(self.release_enricher.enrich(annotations) or {})
-
-            if custom.get("git_repo_path") and custom.get("compose_path"):
-                full_repo_path: Path = Path(cast("str", custom.get("compose_path"))).joinpath(
-                    cast("str", custom.get("git_repo_path"))
-                )
+            release_info: ReleaseDetail | None = self.release_enricher.enrich(
+                latest_info,
+                source_repo_url=pkg_info.source_repo_url,
+                notes_url=customization.relnotes or pkg_info.release_notes_url,
+            )
+            logger.debug("Enriched release info: %s", release_info)
+
+            if service_info.git_repo_path and service_info.compose_path:
+                full_repo_path: Path = Path(service_info.compose_path).joinpath(service_info.git_repo_path)
 
                 git_trust(full_repo_path, Path(self.node_cfg.git_path))
-                save_if_set("git_local_timestamp", git_iso_timestamp(full_repo_path, Path(self.node_cfg.git_path)))
-            features: list[str] = []
+                service_info.git_local_timestamp = git_iso_timestamp(full_repo_path, Path(self.node_cfg.git_path))
+
             can_pull: bool = (
                 self.cfg.allow_pull
-                and image_ref is not None
-                and image_ref != ""
-                and (installed_digest != NO_KNOWN_IMAGE or latest_digest != NO_KNOWN_IMAGE)
+                and not local_info.local_build
+                and local_info.ref is not None
+                and local_info.ref != ""
+                and (local_info.short_digest is not None or latest_info.short_digest is not None)
             )
             if self.cfg.allow_pull and not can_pull:
                 logger.debug(
-                    f"Pull unavailable, image_ref:{image_ref},installed_digest:{installed_digest},latest_digest:{latest_digest}"
+                    f"Pull unavailable, ref:{local_info.ref},local:{local_info.short_digest},latest:{latest_info.short_digest}"
                 )
 
             can_build: bool = False
             if self.cfg.allow_build:
-                can_build = custom.get("git_repo_path") is not None and custom.get("compose_path") is not None
+                can_build = service_info.git_repo_path is not None and service_info.compose_path is not None
                 if not can_build:
-                    if custom.get("git_repo_path") is not None:
+                    if service_info.git_repo_path is not None:
                         logger.debug(
-                            "Local build ignored for git_repo_path=%s because no compose_path", custom.get("git_repo_path")
+                            "Local build ignored for git_repo_path=%s because no compose_path", service_info.git_repo_path
                         )
                 else:
                     full_repo_path = self.full_repo_path(
-                        cast("str", custom.get("compose_path")), cast("str", custom.get("git_repo_path"))
+                        cast("str", service_info.compose_path), cast("str", service_info.git_repo_path)
                     )
-                    if installed_digest is None or installed_digest == NO_KNOWN_IMAGE:
-                        installed_digest = git_local_version(full_repo_path, Path(self.node_cfg.git_path)) or NO_KNOWN_IMAGE
-
-                    behind_count: int = git_check_update_available(full_repo_path, Path(self.node_cfg.git_path))
-                    if behind_count > 0:
-                        if installed_digest is not None and installed_digest.startswith("git:"):
-                            latest_digest = f"{installed_digest}+{behind_count}"
-                            logger.info("Git update available, generating version %s", latest_digest)
-                    else:
-                        logger.debug(f"Git update not available, local repo:{full_repo_path}")
-
-            can_restart: bool = self.cfg.allow_restart and custom.get("compose_path") is not None
-
-            can_update: bool = False
+                    if local_info.local_build and full_repo_path:
+                        git_versionish = git_local_digest(full_repo_path, Path(self.node_cfg.git_path))
+                        if git_versionish:
+                            local_info.git_digest = git_versionish
+                            logger.debug("Git digest for local code %s", git_versionish)
+
+                            behind_count: int = git_check_update_available(full_repo_path, Path(self.node_cfg.git_path))
+                            if behind_count > 0:
+                                latest_info.git_digest = f"{git_versionish}+{behind_count}"
+                                logger.info("Git update available, generating version %s", latest_info.git_digest)
+                            else:
+                                logger.debug(f"Git update not available, local repo:{full_repo_path}")
+                                latest_info.git_digest = git_versionish
+
+            can_restart: bool = self.cfg.allow_restart and service_info.compose_path is not None
 
-            if can_pull or can_build or can_restart:
-                # public install-neutral capabilities and Home Assistant features
-                can_update = True
-                features.append("INSTALL")
-                features.append("PROGRESS")
-            elif any((self.cfg.allow_build, self.cfg.allow_restart, self.cfg.allow_pull)):
-                logger.info(f"Update not available, can_pull:{can_pull}, can_build:{can_build},can_restart{can_restart}")
-            if relnotes_url:
-                features.append("RELEASE_NOTES")
             if can_pull:
                 update_type = "Docker Image"
             elif can_build:
                 update_type = "Docker Build"
             else:
                 update_type = "Unavailable"
-            custom["can_pull"] = can_pull
+
             # can_pull,can_build etc are only info flags
             # the HASS update process is driven by comparing current and available versions
 
-            public_installed_version = select_version(
-                version_policy, installed_version, installed_digest, other_version=latest_version, other_digest=latest_digest
-            )
-            public_latest_version = select_version(
-                version_policy, latest_version, latest_digest, other_version=installed_version, other_digest=installed_digest
+            public_installed_version: str
+            public_latest_version: str
+            version_basis: str
+            public_installed_version, public_latest_version, version_basis = select_versions(
+                version_policy, local_info, latest_info
             )
 
+            publish_policy: PublishPolicy = PublishPolicy.HOMEASSISTANT
+            img_ref_selection = Selection(self.cfg.image_ref_select, local_info.ref)
+            version_selection = Selection(self.cfg.version_select, latest_info.version)
+            if not img_ref_selection or not version_selection:
+                self.log.info(
+                    "Excluding from HA Discovery for include/exclude rule: %s, %s", local_info.ref, latest_info.version
+                )
+                publish_policy = PublishPolicy.MQTT
+
             discovery: Discovery = Discovery(
                 self,
                 c.name,
                 session,
                 node=self.node_cfg.name,
-                entity_picture_url=picture_url,
-                release_url=relnotes_url,
-                release_summary=release_summary,
+                entity_picture_url=customization.picture or pkg_info.logo_url,
                 current_version=public_installed_version,
                 publish_policy=publish_policy,
                 update_policy=update_policy,
                 version_policy=version_policy,
+                version_basis=version_basis,
                 latest_version=public_latest_version,
                 device_icon=self.cfg.device_icon,
-                can_update=can_update,
+                can_pull=can_pull,
                 update_type=update_type,
                 can_build=can_build,
                 can_restart=can_restart,
                 status=(c.status == "running" and "on") or "off",
-                custom=custom,
-                features=features,
-                throttled=registry_throttled,
+                throttled=latest_info.throttled,
                 previous=previous_discovery,
+                release_detail=release_info,
+                installation_detail=service_info,
+                current_detail=local_info,
+                latest_detail=latest_info,
             )
             logger.debug("Analyze generated discovery: %s", discovery)
             return discovery
@@ -540,13 +452,17 @@ class DockerProvider(ReleaseProvider):
     #    metadata_version: str = c.labels.get("org.opencontainers.image.version")
     #    metadata_revision: str = c.labels.get("org.opencontainers.image.revision")
 
-    async def scan(self, session: str) -> AsyncGenerator[Discovery]:
+    async def scan(self, session: str, shuffle: bool = True) -> AsyncGenerator[Discovery]:
         logger = self.log.bind(session=session, action="scan", source=self.source_type)
         containers: int = 0
         results: int = 0
         throttled: int = 0
-        logger.debug("Starting container scan loop")
-        for c in self.client.containers.list():
+
+        targets: list[Container] = self.client.containers.list()
+        if shuffle:
+            random.shuffle(targets)
+        logger.debug("Starting scanning %s containers", len(targets))
+        for c in targets:
             logger.debug("Analyzing container", container=c.name)
             if self.stopped.is_set():
                 logger.info(f"Shutdown detected, aborting scan at {c}")
@@ -554,7 +470,7 @@ class DockerProvider(ReleaseProvider):
             containers = containers + 1
             result: Discovery | None = self.analyze(c, session)
             if result:
-                logger.debug("Analyzed container", result_name=result.name, custom=result.custom)
+                logger.debug("Analyzed container", result_name=result.name, throttled=result.throttled)
                 self.discoveries[result.name] = result
                 results = results + 1
                 throttled += 1 if result.throttled else 0
@@ -580,10 +496,10 @@ class DockerProvider(ReleaseProvider):
                     logger.info("Starting update ...")
                     on_update_start(discovery)
                     if self.update(discovery):
-                        logger.info("Rescanning ...")
+                        logger.debug("Rescanning ...")
                         rediscovery = self.rescan(discovery)
-                        updated = rediscovery is not None
-                        logger.info("Rescanned %s: %s", updated, rediscovery)
+                        updated = rediscovery is not None and not rediscovery.throttled
+                        logger.info("Rescanned, updated:%s", updated)
                     else:
                         logger.info("Rescan with no result")
                     on_update_end(rediscovery or discovery)
@@ -598,9 +514,149 @@ class DockerProvider(ReleaseProvider):
     def resolve(self, discovery_name: str) -> Discovery | None:
         return self.discoveries.get(discovery_name)
 
-    def default_metadata(self, image_name: str | None, image_ref: str | None) -> PackageUpdateInfo:
+    def default_metadata(self, image_info: DockerImageInfo) -> PackageUpdateInfo:
         for enricher in self.pkg_enrichers:
-            pkg_info = enricher.enrich(image_name, image_ref, self.log)
+            pkg_info = enricher.enrich(image_info)
             if pkg_info is not None:
                 return pkg_info
         raise ValueError("No enricher could provide metadata, not even default enricher")
+
+
+def select_versions(version_policy: VersionPolicy, installed: DockerImageInfo, latest: DockerImageInfo) -> tuple[str, str, str]:
+    """Pick the best version string to display based on the version policy and available data
+
+    Ensures that both local installed and remote latest versions are derived in same way
+    Falls back to digest if version not reliable or not consistent with current/available version
+    """
+    phase: int = 0
+    shortcircuit: str | None = None
+
+    def basis(rule: str) -> str:
+        return f"{rule}-{phase}" if not shortcircuit else f"{rule}-{phase}-{shortcircuit}"
+
+    # shortcircuit the logic if there's nothing to compare
+    if latest.throttled:
+        log.debug("Flattening versions for throttled update %s", installed.ref)
+        shortcircuit = "THR"
+        latest = installed
+    elif not any((latest.short_digest, latest.repo_digest, latest.git_digest, latest.version)):
+        log.debug("Flattening versions for empty update %s", installed.ref)
+        shortcircuit = "NUP"
+        latest = installed
+    elif latest.short_digest == installed.short_digest and latest.short_digest is not None:
+        log.debug("Flattening versions for identical update %s", installed.ref)
+        shortcircuit = "SDM"
+        latest = installed
+    elif installed.image_digest in latest.repo_digests:
+        # TODO: avoid this by better adaptations for different registries and single/multi manifests
+        log.debug(
+            "Matching new repo_digest against installed image digest for %s image %s", installed.index_name, installed.name
+        )
+        shortcircuit = "FGA"
+        latest = installed
+    elif latest.image_digest in installed.repo_digests:
+        # TODO: avoid this by better adaptations for different registries and single/multi manifests
+        log.debug(
+            "Matching new image_digest against installed repo digest for %s image %s", installed.index_name, installed.name
+        )
+        shortcircuit = "FGB"
+        latest = installed
+
+    if version_policy == VersionPolicy.VERSION and installed.version and latest.version:
+        return installed.version, latest.version, basis("version")
+
+    installed_digest_available: bool = installed.short_digest is not None and installed.short_digest != ""
+    latest_digest_available: bool = latest.short_digest is not None and latest.short_digest != ""
+
+    if version_policy == VersionPolicy.DIGEST and installed_digest_available and latest_digest_available:
+        return installed.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+    if (
+        version_policy == VersionPolicy.VERSION_DIGEST
+        and installed.version
+        and latest.version
+        and installed_digest_available
+        and latest_digest_available
+    ):
+        return (
+            f"{installed.version}:{installed.short_digest}",
+            f"{latest.version}:{latest.short_digest}",
+            basis("version-digest"),
+        )
+
+    phase = 1
+    if version_policy == VersionPolicy.AUTO and (
+        (installed.version == latest.version and installed.short_digest == latest.short_digest)
+        or (installed.version != latest.version and installed.short_digest != latest.short_digest)
+    ):
+        # detect semver, or casual semver (e.g. v1.030)
+        # only use this if both version and digest are consistently agreeing or disagreeing
+        # if the strict conditions work, people see nice version numbers on screen rather than hashes
+        if (
+            installed.version
+            and re.match(SEMVER_RE, installed.version or "")
+            and latest.version
+            and re.match(SEMVER_RE, latest.version or "")
+        ):
+            # Smells like semver, override if not using version_policy
+            return installed.version, latest.version, basis("semver")
+        if (
+            installed.version
+            and re.match(VERSION_RE, installed.version or "")
+            and latest.version
+            and re.match(VERSION_RE, latest.version or "")
+        ):
+            # Smells like casual semver, override if not using version_policy
+            return installed.version, latest.version, basis("causualver")
+
+    # AUTO or fallback
+    phase = 2
+    if installed.version and latest.version and installed_digest_available and latest_digest_available:
+        return (
+            f"{installed.version}:{installed.short_digest}",
+            f"{latest.version}:{latest.short_digest}",
+            basis("version-digest"),
+        )
+
+        # and ((other_digest is None and other_version is None) or (other_digest is not None and other_version is not None))
+
+    if installed.version and latest.version:
+        return installed.version, latest.version, basis("version")
+
+    # Check for local builds
+    phase = 3
+    if installed.git_digest and latest.git_digest:
+        return f"git:{installed.git_digest}", f"git:{latest.git_digest}", basis("git")
+
+    # Fall back to digests, image or repo index
+    phase = 4
+    if installed_digest_available and latest_digest_available:
+        return installed.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+    if installed.version and not latest.version and not latest.short_digest and not latest.repo_digest:
+        return installed.version, installed.version, basis("version")
+    phase = 5
+    if not installed_digest_available and latest_digest_available:
+        # odd condition if local image has no identity, even out versions so no update alert
+        return latest.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+
+    # Fall back to repo digests
+    phase = 6
+
+    def condense_repo_id(i: DockerImageInfo) -> str:
+        v: str | None = i.condense_digest(i.repo_digest) if i.repo_digest else None
+        return v or ""
+
+    if installed.repo_digest and latest.repo_digest:
+        # where the image digest isn't available, fall back to a repo digest
+        return condense_repo_id(installed), condense_repo_id(latest), basis("repo-digest")
+
+    phase = 7
+    if latest.repo_digest and latest.repo_digest in installed.repo_digests:
+        # installed has multiple RepoDigests from multiple pulls and one of them matches latest current repo digest
+        return condense_repo_id(latest), condense_repo_id(latest), basis("repo-digest")
+
+    if installed_digest_available and not latest_digest_available:
+        return installed.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+
+    log.warn("No versions can be determined for %s", installed.ref)
+    phase = 999
+    return UNKNOWN_VERSION, UNKNOWN_VERSION, basis("failure")