updates2mqtt 1.6.0__py3-none-any.whl → 1.7.2__py3-none-any.whl

updates2mqtt/integrations/docker.py

@@ -1,10 +1,10 @@
+import random
 import re
 import subprocess
 import time
 import typing
 from collections.abc import AsyncGenerator, Callable
 from enum import Enum
-from http import HTTPStatus
 from pathlib import Path
 from threading import Event
 from typing import Any, cast
@@ -12,22 +12,43 @@ from typing import Any, cast
 import docker
 import docker.errors
 import structlog
-from docker.auth import resolve_repository_name
 from docker.models.containers import Container
-from hishel.httpx import SyncCacheClient
 
-from updates2mqtt.config import DockerConfig, DockerPackageUpdateInfo, NodeConfig, PackageUpdateInfo
-from updates2mqtt.model import Discovery, ReleaseProvider
-
-from .git_utils import git_check_update_available, git_iso_timestamp, git_local_version, git_pull, git_trust
+from updates2mqtt.config import (
+    SEMVER_RE,
+    UNKNOWN_VERSION,
+    VERSION_RE,
+    DockerConfig,
+    NodeConfig,
+    PackageUpdateInfo,
+    PublishPolicy,
+    RegistryAPI,
+    UpdatePolicy,
+    VersionPolicy,
+)
+from updates2mqtt.helpers import Selection, Throttler
+from updates2mqtt.integrations.docker_enrich import (
+    CommonPackageEnricher,
+    ContainerDistributionAPIVersionLookup,
+    DefaultPackageEnricher,
+    DockerClientVersionLookup,
+    DockerImageInfo,
+    DockerServiceDetails,
+    LinuxServerIOPackageEnricher,
+    LocalContainerInfo,
+    PackageEnricher,
+    SourceReleaseEnricher,
+)
+from updates2mqtt.model import Discovery, ReleaseDetail, ReleaseProvider
+
+from .git_utils import git_check_update_available, git_iso_timestamp, git_local_digest, git_pull, git_trust
 
 if typing.TYPE_CHECKING:
-    from docker.models.images import Image, RegistryData
+    from docker.models.images import Image
 
 # distinguish docker build from docker pull?
 
 log = structlog.get_logger()
-NO_KNOWN_IMAGE = "UNKNOWN"
 
 
 class DockerComposeCommand(Enum):
@@ -46,13 +67,13 @@ class ContainerCustomization:
     env_prefix: str = "UPD2MQTT_"
 
     def __init__(self, container: Container) -> None:
-        self.update: str = "PASSIVE"
+        self.update: UpdatePolicy = UpdatePolicy.PASSIVE  # was known as UPD2MQTT_UPDATE before policies and labels
         self.git_repo_path: str | None = None
         self.picture: str | None = None
         self.relnotes: str | None = None
         self.ignore: bool = False
-        self.version_include: str | None = None
-        self.version_exclude: str | None = None
+        self.version_policy: VersionPolicy | None = None
+        self.registry_token: str | None = None
 
         if not container.attrs or container.attrs.get("Config") is None:
             return
@@ -93,30 +114,43 @@ class ContainerCustomization:
                 if v is not None:
                     if isinstance(getattr(self, attr), bool):
                         setattr(self, attr, v.upper() in ("TRUE", "YES", "1"))
+                    elif isinstance(getattr(self, attr), VersionPolicy):
+                        setattr(self, attr, VersionPolicy[v.upper()])
+                    elif isinstance(getattr(self, attr), UpdatePolicy):
+                        setattr(self, attr, UpdatePolicy[v.upper()])
                     else:
                         setattr(self, attr, v)
 
-        self.update = self.update.upper()
-
 
 class DockerProvider(ReleaseProvider):
     def __init__(
         self,
         cfg: DockerConfig,
-        common_pkg_cfg: dict[str, PackageUpdateInfo],
         node_cfg: NodeConfig,
         self_bounce: Event | None = None,
     ) -> None:
-        super().__init__("docker")
+        super().__init__(node_cfg, "docker")
         self.client: docker.DockerClient = docker.from_env()
         self.cfg: DockerConfig = cfg
-        self.node_cfg: NodeConfig = node_cfg
-        self.common_pkgs: dict[str, PackageUpdateInfo] = common_pkg_cfg if common_pkg_cfg else {}
+
         # TODO: refresh discovered packages periodically
-        self.discovered_pkgs: dict[str, PackageUpdateInfo] = self.discover_metadata()
-        self.pause_api_until: dict[str, float] = {}
-        self.api_throttle_pause: int = cfg.api_throttle_wait
+        self.throttler = Throttler(self.cfg.default_api_backoff, self.log, self.stopped)
         self.self_bounce: Event | None = self_bounce
+        self.pkg_enrichers: list[PackageEnricher] = [
+            CommonPackageEnricher(self.cfg),
+            LinuxServerIOPackageEnricher(self.cfg),
+            DefaultPackageEnricher(self.cfg),
+        ]
+        self.docker_client_image_lookup = DockerClientVersionLookup(
+            self.client, self.throttler, self.cfg.registry, self.cfg.default_api_backoff
+        )
+        self.registry_image_lookup = ContainerDistributionAPIVersionLookup(self.throttler, self.cfg.registry)
+        self.release_enricher = SourceReleaseEnricher()
+        self.local_info_builder = LocalContainerInfo()
+
+    def initialize(self) -> None:
+        for enricher in self.pkg_enrichers:
+            enricher.initialize()
 
     def update(self, discovery: Discovery) -> bool:
         logger: Any = self.log.bind(container=discovery.name, action="update")
@@ -129,19 +163,21 @@ class DockerProvider(ReleaseProvider):
 
     def fetch(self, discovery: Discovery) -> None:
         logger = self.log.bind(container=discovery.name, action="fetch")
+        installed_info: DockerImageInfo | None = cast("DockerImageInfo|None", discovery.current_detail)
+        service_info: DockerServiceDetails | None = cast("DockerServiceDetails|None", discovery.installation_detail)
 
-        image_ref: str | None = discovery.custom.get("image_ref")
-        platform: str | None = discovery.custom.get("platform")
-        if discovery.custom.get("can_pull") and image_ref:
+        image_ref: str | None = installed_info.ref if installed_info else None
+        platform: str | None = installed_info.platform if installed_info else None
+        if discovery.can_pull and image_ref:
             logger.info("Pulling", image_ref=image_ref, platform=platform)
             image: Image = self.client.images.pull(image_ref, platform=platform, all_tags=False)
             if image:
                 logger.info("Pulled", image_id=image.id, image_ref=image_ref, platform=platform)
             else:
                 logger.warn("Unable to pull", image_ref=image_ref, platform=platform)
-        elif discovery.can_build:
-            compose_path: str | None = discovery.custom.get("compose_path")
-            git_repo_path: str | None = discovery.custom.get("git_repo_path")
+        elif discovery.can_build and service_info:
+            compose_path: str | None = service_info.compose_path
+            git_repo_path: str | None = service_info.git_repo_path
             logger.debug("can_build check", git_repo=git_repo_path)
             if not compose_path or not git_repo_path:
                 logger.warn("No compose path or git repo path configured, skipped build")
@@ -149,10 +185,7 @@ class DockerProvider(ReleaseProvider):
 
             full_repo_path: Path = self.full_repo_path(compose_path, git_repo_path)
             if git_pull(full_repo_path, Path(self.node_cfg.git_path)):
-                if compose_path:
-                    self.build(discovery, compose_path)
-                else:
-                    logger.warn("No compose path configured, skipped build")
+                self.build(discovery)
             else:
                 logger.debug("Skipping git_pull, no update")
 
@@ -163,14 +196,19 @@ class DockerProvider(ReleaseProvider):
             return Path(compose_path) / git_repo_path
         return Path(git_repo_path)
 
-    def build(self, discovery: Discovery, compose_path: str) -> bool:
+    def build(self, discovery: Discovery) -> bool:
         logger = self.log.bind(container=discovery.name, action="build")
-        logger.info("Building", compose_path=compose_path)
+        service_info: DockerServiceDetails | None = cast("DockerServiceDetails|None", discovery.installation_detail)
+
+        if not service_info or not service_info.compose_path:
+            logger.warn("No service_info available on compose")
+            return False
+        logger.info("Building", compose_path=service_info.compose_path, service=service_info.compose_service)
         return self.execute_compose(
             command=DockerComposeCommand.BUILD,
             args="",
-            service=discovery.custom.get("compose_service"),
-            cwd=compose_path,
+            service=service_info.compose_service,
+            cwd=service_info.compose_path,
             logger=logger,
         )
 
@@ -204,16 +242,28 @@ class DockerProvider(ReleaseProvider):
 
     def restart(self, discovery: Discovery) -> bool:
         logger = self.log.bind(container=discovery.name, action="restart")
-        if self.self_bounce is not None and (
-            "ghcr.io/rhizomatics/updates2mqtt" in discovery.custom.get("image_ref", "")
-            or discovery.custom.get("git_repo_path", "").endswith("updates2mqtt")
+        installed_info: DockerImageInfo | None = cast("DockerImageInfo|None", discovery.current_detail)
+        service_info: DockerServiceDetails | None = cast("DockerServiceDetails|None", discovery.installation_detail)
+
+        if (
+            self.self_bounce is not None
+            and installed_info
+            and service_info
+            and (
+                "ghcr.io/rhizomatics/updates2mqtt" in installed_info.ref
+                or (service_info.git_repo_path and service_info.git_repo_path.endswith("updates2mqtt"))
+            )
         ):
             logger.warning("Attempting to self-bounce")
             self.self_bounce.set()
-        compose_path = discovery.custom.get("compose_path")
-        compose_service: str | None = discovery.custom.get("compose_service")
+        if service_info is None:
+            return False
         return self.execute_compose(
-            command=DockerComposeCommand.UP, args="--detach --yes", service=compose_service, cwd=compose_path, logger=logger
+            command=DockerComposeCommand.UP,
+            args="--detach --yes",
+            service=service_info.compose_service,
+            cwd=service_info.compose_path,
+            logger=logger,
         )
 
     def rescan(self, discovery: Discovery) -> Discovery | None:
@@ -221,8 +271,8 @@ class DockerProvider(ReleaseProvider):
         try:
             c: Container = self.client.containers.get(discovery.name)
             if c:
-                rediscovery = self.analyze(c, discovery.session, original_discovery=discovery)
-                if rediscovery:
+                rediscovery = self.analyze(c, discovery.session, previous_discovery=discovery)
+                if rediscovery and not rediscovery.throttled:
                     self.discoveries[rediscovery.name] = rediscovery
                     return rediscovery
             logger.warn("Unable to find container for rescan")
@@ -232,22 +282,9 @@ class DockerProvider(ReleaseProvider):
             logger.exception("Docker API error retrieving container")
         return None
 
-    def check_throttle(self, repo_id: str) -> bool:
-        if self.pause_api_until.get(repo_id) is not None:
-            if self.pause_api_until[repo_id] < time.time():
-                del self.pause_api_until[repo_id]
-                log.info("%s throttling wait complete", repo_id)
-            else:
-                log.debug("%s throttling has %s secs left", repo_id, self.pause_api_until[repo_id] - time.time())
-                return True
-        return False
-
-    def analyze(self, c: Container, session: str, original_discovery: Discovery | None = None) -> Discovery | None:
+    def analyze(self, c: Container, session: str, previous_discovery: Discovery | None = None) -> Discovery | None:
         logger = self.log.bind(container=c.name, action="analyze")
 
-        image_ref: str | None = None
-        image_name: str | None = None
-        local_versions = None
         if c.attrs is None or not c.attrs:
             logger.warn("No container attributes found, discovery rejected")
             return None
@@ -259,202 +296,150 @@ class DockerProvider(ReleaseProvider):
         if customization.ignore:
             logger.info("Container ignored due to UPD2MQTT_IGNORE setting")
             return None
+        version_policy: VersionPolicy = (
+            self.cfg.version_policy if not customization.version_policy else customization.version_policy
+        )
+        if customization.update == UpdatePolicy.AUTO:
+            logger.debug("Auto update policy detected")
+        update_policy: UpdatePolicy = customization.update or UpdatePolicy.PASSIVE
 
-        image: Image | None = c.image
-        repo_id: str = "DEFAULT"
-        if image is not None and image.tags and len(image.tags) > 0:
-            image_ref = image.tags[0]
-        else:
-            image_ref = c.attrs.get("Config", {}).get("Image")
-        if image_ref is None:
-            logger.warn("No image or image attributes found")
-        else:
-            repo_id, _ = resolve_repository_name(image_ref)
-            try:
-                image_name = image_ref.split(":")[0]
-            except Exception as e:
-                logger.warn("No tags found (%s) : %s", image, e)
-            if image is not None and image.attrs is not None:
-                try:
-                    local_versions = [i.split("@")[1][7:19] for i in image.attrs["RepoDigests"]]
-                except Exception as e:
-                    logger.warn("Cannot determine local version: %s", e)
-                    logger.warn("RepoDigests=%s", image.attrs.get("RepoDigests"))
-
-        platform: str = "Unknown"
-        pkg_info: PackageUpdateInfo = self.default_metadata(image_name, image_ref=image_ref)
+        local_info: DockerImageInfo
+        service_info: DockerServiceDetails
+        local_info, service_info = self.local_info_builder.build_image_info(c)
+        pkg_info: PackageUpdateInfo = self.default_metadata(local_info)
 
         try:
-            picture_url = customization.picture or pkg_info.logo_url
-            relnotes_url = customization.relnotes or pkg_info.release_notes_url
-            if image is not None and image.attrs is not None:
-                platform = "/".join(
-                    filter(
-                        None,
-                        [
-                            image.attrs["Os"],
-                            image.attrs["Architecture"],
-                            image.attrs.get("Variant"),
-                        ],
-                    ),
-                )
-
-            reg_data: RegistryData | None = None
-            latest_version: str | None = NO_KNOWN_IMAGE
-            registry_throttled = self.check_throttle(repo_id)
-
-            if image_ref and local_versions and not registry_throttled:
-                retries_left = 3
-                while reg_data is None and retries_left > 0 and not self.stopped.is_set():
-                    try:
-                        logger.debug("Fetching registry data", image_ref=image_ref)
-                        reg_data = self.client.images.get_registry_data(image_ref)
-                        log.debug(
-                            "Registry Data: id:%s,image:%s, attrs:%s",
-                            reg_data.id,
-                            reg_data.image_name,
-                            reg_data.attrs,
-                        )
-                        latest_version = reg_data.short_id[7:] if reg_data else None
-                    except docker.errors.APIError as e:
-                        if e.status_code == HTTPStatus.TOO_MANY_REQUESTS:
-                            logger.warn("Docker Registry throttling requests, %s", e.explanation)
-                            self.pause_api_until[repo_id] = time.time() + self.api_throttle_pause
-                            return None
-                        retries_left -= 1
-                        if retries_left == 0 or e.is_client_error():
-                            logger.warn("Failed to fetch registry data: [%s] %s", e.errno, e.explanation)
-                        else:
-                            logger.debug("Failed to fetch registry data, retrying: %s", e)
-
-            local_version: str | None = NO_KNOWN_IMAGE
-            if local_versions:
-                # might be multiple RepoDigests if image has been pulled multiple times with diff manifests
-                local_version = latest_version if latest_version in local_versions else local_versions[0]
-                log.debug(f"Setting local version to {local_version}, local_versions:{local_versions}")
-
-            def save_if_set(key: str, val: str | None) -> None:
-                if val is not None:
-                    custom[key] = val
-
-            image_ref = image_ref or ""
-
-            custom: dict[str, str | bool] = {}
-            custom["platform"] = platform
-            custom["image_ref"] = image_ref
-            custom["repo_id"] = repo_id
-            if registry_throttled:
-                custom["registry_throttled"] = True
-            save_if_set("compose_path", c.labels.get("com.docker.compose.project.working_dir"))
-            save_if_set("compose_version", c.labels.get("com.docker.compose.version"))
-            save_if_set("compose_service", c.labels.get("com.docker.compose.service"))
-            save_if_set("git_repo_path", customization.git_repo_path)
-            # save_if_set("apt_pkgs", c_env.get("UPD2MQTT_APT_PKGS"))
-
-            if customization.update == "AUTO":
-                logger.debug("Auto update policy detected")
-                update_policy = "Auto"
+            service_info.git_repo_path = customization.git_repo_path
+
+            registry_selection = Selection(self.cfg.registry_select, local_info.index_name)
+            latest_info: DockerImageInfo
+            if local_info.pinned:
+                logger.debug("Skipping registry fetch for local pinned image, %s", local_info.ref)
+                latest_info = local_info.reuse()
+            elif registry_selection and local_info.ref and not local_info.local_build:
+                if self.cfg.registry.api == RegistryAPI.DOCKER_CLIENT:
+                    latest_info = self.docker_client_image_lookup.lookup(local_info)
+                elif self.cfg.registry.api == RegistryAPI.OCI_V2:
+                    latest_info = self.registry_image_lookup.lookup(local_info, token=customization.registry_token)
+                elif self.cfg.registry.api == RegistryAPI.OCI_V2_MINIMAL:
+                    latest_info = self.registry_image_lookup.lookup(
+                        local_info, token=customization.registry_token, minimal=True
+                    )
+                else:  # assuming RegistryAPI.DISABLED
+                    logger.debug(f"Skipping registry check, disabled in config {self.cfg.registry.api}")
+                    latest_info = local_info.reuse()
+            elif local_info.local_build:
+                # assume its a locally built image if no RepoDigests available
+                latest_info = local_info.reuse()
+                latest_info.short_digest = None
+                latest_info.image_digest = None
             else:
-                update_policy = "Passive"
+                logger.debug("Registry selection rules suppressed metadata lookup")
+                latest_info = local_info.reuse()
 
-            if custom.get("git_repo_path") and custom.get("compose_path"):
-                full_repo_path: Path = Path(cast("str", custom.get("compose_path"))).joinpath(
-                    cast("str", custom.get("git_repo_path"))
-                )
+            release_info: ReleaseDetail | None = self.release_enricher.enrich(
+                latest_info,
+                source_repo_url=pkg_info.source_repo_url,
+                notes_url=customization.relnotes or pkg_info.release_notes_url,
+            )
+            logger.debug("Enriched release info: %s", release_info)
+
+            if service_info.git_repo_path and service_info.compose_path:
+                full_repo_path: Path = Path(service_info.compose_path).joinpath(service_info.git_repo_path)
 
                 git_trust(full_repo_path, Path(self.node_cfg.git_path))
-                save_if_set("git_local_timestamp", git_iso_timestamp(full_repo_path, Path(self.node_cfg.git_path)))
-            features: list[str] = []
+                service_info.git_local_timestamp = git_iso_timestamp(full_repo_path, Path(self.node_cfg.git_path))
+
             can_pull: bool = (
                 self.cfg.allow_pull
-                and image_ref is not None
-                and image_ref != ""
-                and (local_version != NO_KNOWN_IMAGE or latest_version != NO_KNOWN_IMAGE)
+                and not local_info.local_build
+                and local_info.ref is not None
+                and local_info.ref != ""
+                and (local_info.short_digest is not None or latest_info.short_digest is not None)
             )
             if self.cfg.allow_pull and not can_pull:
                 logger.debug(
-                    f"Pull not available, image_ref:{image_ref},local_version:{local_version},latest_version:{latest_version}"
+                    f"Pull unavailable, ref:{local_info.ref},local:{local_info.short_digest},latest:{latest_info.short_digest}"
                 )
-            skip_pull: bool = False
-            if can_pull and latest_version is not None:
-                if customization.version_include and not re.match(customization.version_include, latest_version):
-                    logger.info(f"Skipping version {latest_version} not matching include pattern")
-                    skip_pull = True
-                    latest_version = local_version
-                if customization.version_exclude and re.match(customization.version_exclude, latest_version):  # type: ignore[arg-type]
-                    logger.info(f"Skipping version {latest_version} matching exclude pattern")
-                    skip_pull = True
-                    latest_version = local_version
 
             can_build: bool = False
             if self.cfg.allow_build:
-                can_build = custom.get("git_repo_path") is not None and custom.get("compose_path") is not None
+                can_build = service_info.git_repo_path is not None and service_info.compose_path is not None
                 if not can_build:
-                    if custom.get("git_repo_path") is not None:
-                        log.debug(
-                            "Local build ignored for git_repo_path=%s because no compose_path", custom.get("git_repo_path")
+                    if service_info.git_repo_path is not None:
+                        logger.debug(
+                            "Local build ignored for git_repo_path=%s because no compose_path", service_info.git_repo_path
                         )
                 else:
                     full_repo_path = self.full_repo_path(
-                        cast("str", custom.get("compose_path")), cast("str", custom.get("git_repo_path"))
+                        cast("str", service_info.compose_path), cast("str", service_info.git_repo_path)
                     )
-                    if local_version is None or local_version == NO_KNOWN_IMAGE:
-                        local_version = git_local_version(full_repo_path, Path(self.node_cfg.git_path)) or NO_KNOWN_IMAGE
-
-                    behind_count: int = git_check_update_available(full_repo_path, Path(self.node_cfg.git_path))
-                    if behind_count > 0:
-                        if local_version is not None and local_version.startswith("git:"):
-                            latest_version = f"{local_version}+{behind_count}"
-                            log.info("Git update available, generating version %s", latest_version)
-                    else:
-                        logger.debug(f"Git update not available, local repo:{full_repo_path}")
-
-            can_restart: bool = self.cfg.allow_restart and custom.get("compose_path") is not None
-
-            can_update: bool = False
-
-            if can_pull or can_build or can_restart:
-                # public install-neutral capabilities and Home Assistant features
-                can_update = True
-                features.append("INSTALL")
-                features.append("PROGRESS")
-            elif any((self.cfg.allow_build, self.cfg.allow_restart, self.cfg.allow_pull)):
-                logger.info(f"Update not available, can_pull:{can_pull}, can_build:{can_build},can_restart{can_restart}")
-            if relnotes_url:
-                features.append("RELEASE_NOTES")
-            if skip_pull:
-                update_type: str = "Skipped"
-            elif can_pull:
+                    if local_info.local_build and full_repo_path:
+                        git_versionish = git_local_digest(full_repo_path, Path(self.node_cfg.git_path))
+                        if git_versionish:
+                            local_info.git_digest = git_versionish
+                            logger.debug("Git digest for local code %s", git_versionish)
+
+                            behind_count: int = git_check_update_available(full_repo_path, Path(self.node_cfg.git_path))
+                            if behind_count > 0:
+                                latest_info.git_digest = f"{git_versionish}+{behind_count}"
+                                logger.info("Git update available, generating version %s", latest_info.git_digest)
+                            else:
+                                logger.debug(f"Git update not available, local repo:{full_repo_path}")
+                                latest_info.git_digest = git_versionish
+
+            can_restart: bool = self.cfg.allow_restart and service_info.compose_path is not None
+
+            if can_pull:
                 update_type = "Docker Image"
             elif can_build:
                 update_type = "Docker Build"
             else:
                 update_type = "Unavailable"
-            custom["can_pull"] = can_pull
-            custom["skip_pull"] = skip_pull
+
             # can_pull,can_build etc are only info flags
             # the HASS update process is driven by comparing current and available versions
 
+            public_installed_version: str
+            public_latest_version: str
+            version_basis: str
+            public_installed_version, public_latest_version, version_basis = select_versions(
+                version_policy, local_info, latest_info
+            )
+
+            publish_policy: PublishPolicy = PublishPolicy.HOMEASSISTANT
+            img_ref_selection = Selection(self.cfg.image_ref_select, local_info.ref)
+            version_selection = Selection(self.cfg.version_select, latest_info.version)
+            if not img_ref_selection or not version_selection:
+                self.log.info(
+                    "Excluding from HA Discovery for include/exclude rule: %s, %s", local_info.ref, latest_info.version
+                )
+                publish_policy = PublishPolicy.MQTT
+
             discovery: Discovery = Discovery(
                 self,
                 c.name,
                 session,
                 node=self.node_cfg.name,
-                entity_picture_url=picture_url,
-                release_url=relnotes_url,
-                current_version=local_version,
+                entity_picture_url=customization.picture or pkg_info.logo_url,
+                current_version=public_installed_version,
+                publish_policy=publish_policy,
                 update_policy=update_policy,
-                update_last_attempt=original_discovery.update_last_attempt if original_discovery else None,
-                latest_version=latest_version if latest_version != NO_KNOWN_IMAGE else local_version,
+                version_policy=version_policy,
+                version_basis=version_basis,
+                latest_version=public_latest_version,
                 device_icon=self.cfg.device_icon,
-                can_update=can_update,
+                can_pull=can_pull,
                 update_type=update_type,
                 can_build=can_build,
                 can_restart=can_restart,
                 status=(c.status == "running" and "on") or "off",
-                custom=custom,
-                features=features,
-                throttled=registry_throttled,
+                throttled=latest_info.throttled,
+                previous=previous_discovery,
+                release_detail=release_info,
+                installation_detail=service_info,
+                current_detail=local_info,
+                latest_detail=latest_info,
             )
             logger.debug("Analyze generated discovery: %s", discovery)
             return discovery
@@ -463,13 +448,21 @@ class DockerProvider(ReleaseProvider):
         logger.debug("Analyze returned empty discovery")
         return None
 
-    async def scan(self, session: str) -> AsyncGenerator[Discovery]:
+    # def version(self, c: Container, version_type: str):
+    #    metadata_version: str = c.labels.get("org.opencontainers.image.version")
+    #    metadata_revision: str = c.labels.get("org.opencontainers.image.revision")
+
+    async def scan(self, session: str, shuffle: bool = True) -> AsyncGenerator[Discovery]:
         logger = self.log.bind(session=session, action="scan", source=self.source_type)
         containers: int = 0
         results: int = 0
         throttled: int = 0
-        logger.debug("Starting container scan loop")
-        for c in self.client.containers.list():
+
+        targets: list[Container] = self.client.containers.list()
+        if shuffle:
+            random.shuffle(targets)
+        logger.debug("Starting scanning %s containers", len(targets))
+        for c in targets:
             logger.debug("Analyzing container", container=c.name)
             if self.stopped.is_set():
                 logger.info(f"Shutdown detected, aborting scan at {c}")
@@ -477,7 +470,7 @@ class DockerProvider(ReleaseProvider):
             containers = containers + 1
             result: Discovery | None = self.analyze(c, session)
             if result:
-                logger.debug("Analyzed container", result_name=result.name, custom=result.custom)
+                logger.debug("Analyzed container", result_name=result.name, throttled=result.throttled)
                 self.discoveries[result.name] = result
                 results = results + 1
                 throttled += 1 if result.throttled else 0
@@ -503,10 +496,10 @@ class DockerProvider(ReleaseProvider):
                     logger.info("Starting update ...")
                     on_update_start(discovery)
                     if self.update(discovery):
-                        logger.info("Rescanning ...")
+                        logger.debug("Rescanning ...")
                         rediscovery = self.rescan(discovery)
-                        updated = rediscovery is not None
-                        logger.info("Rescanned %s: %s", updated, rediscovery)
+                        updated = rediscovery is not None and not rediscovery.throttled
+                        logger.info("Rescanned, updated:%s", updated)
                     else:
                         logger.info("Rescan with no result")
                     on_update_end(rediscovery or discovery)
@@ -521,87 +514,149 @@ class DockerProvider(ReleaseProvider):
     def resolve(self, discovery_name: str) -> Discovery | None:
         return self.discoveries.get(discovery_name)
 
-    def hass_state_format(self, discovery: Discovery) -> dict:  # noqa: ARG002
-        # disable since hass mqtt update has strict json schema for message
-        return {
-            # "docker_image_ref": discovery.custom.get("image_ref"),
-            # "last_update_attempt": safe_json_dt(discovery.update_last_attempt),
-            # "can_pull": discovery.custom.get("can_pull"),
-            # "can_build": discovery.custom.get("can_build"),
-            # "can_restart": discovery.custom.get("can_restart"),
-            # "git_repo_path": discovery.custom.get("git_repo_path"),
-            # "compose_path": discovery.custom.get("compose_path"),
-            # "platform": discovery.custom.get("platform"),
-        }
-
-    def default_metadata(self, image_name: str | None, image_ref: str | None) -> PackageUpdateInfo:
-        def match(pkg: PackageUpdateInfo) -> bool:
-            if pkg is not None and pkg.docker is not None and pkg.docker.image_name is not None:
-                if image_name is not None and image_name == pkg.docker.image_name:
-                    return True
-                if image_ref is not None and image_ref == pkg.docker.image_name:
-                    return True
-            return False
-
-        if image_name is not None and image_ref is not None:
-            for pkg in self.common_pkgs.values():
-                if match(pkg):
-                    self.log.debug(
-                        "Found common package",
-                        image_name=pkg.docker.image_name,  # type: ignore [union-attr]
-                        logo_url=pkg.logo_url,
-                        relnotes_url=pkg.release_notes_url,
-                    )
-                    return pkg
-            for pkg in self.discovered_pkgs.values():
-                if match(pkg):
-                    self.log.debug(
-                        "Found discovered package",
-                        pkg=pkg.docker.image_name,  # type: ignore [union-attr]
-                        logo_url=pkg.logo_url,
-                        relnotes_url=pkg.release_notes_url,
-                    )
-                    return pkg
+    def default_metadata(self, image_info: DockerImageInfo) -> PackageUpdateInfo:
+        for enricher in self.pkg_enrichers:
+            pkg_info = enricher.enrich(image_info)
+            if pkg_info is not None:
+                return pkg_info
+        raise ValueError("No enricher could provide metadata, not even default enricher")
+
+
+def select_versions(version_policy: VersionPolicy, installed: DockerImageInfo, latest: DockerImageInfo) -> tuple[str, str, str]:
+    """Pick the best version string to display based on the version policy and available data
+
+    Ensures that both local installed and remote latest versions are derived in same way
+    Falls back to digest if version not reliable or not consistent with current/available version
+    """
+    phase: int = 0
+    shortcircuit: str | None = None
+
+    def basis(rule: str) -> str:
+        return f"{rule}-{phase}" if not shortcircuit else f"{rule}-{phase}-{shortcircuit}"
+
+    # shortcircuit the logic if there's nothing to compare
+    if latest.throttled:
+        log.debug("Flattening versions for throttled update %s", installed.ref)
+        shortcircuit = "THR"
+        latest = installed
+    elif not any((latest.short_digest, latest.repo_digest, latest.git_digest, latest.version)):
+        log.debug("Flattening versions for empty update %s", installed.ref)
+        shortcircuit = "NUP"
+        latest = installed
+    elif latest.short_digest == installed.short_digest and latest.short_digest is not None:
+        log.debug("Flattening versions for identical update %s", installed.ref)
+        shortcircuit = "SDM"
+        latest = installed
+    elif installed.image_digest in latest.repo_digests:
+        # TODO: avoid this by better adaptations for different registries and single/multi manifests
+        log.debug(
+            "Matching new repo_digest against installed image digest for %s image %s", installed.index_name, installed.name
+        )
+        shortcircuit = "FGA"
+        latest = installed
+    elif latest.image_digest in installed.repo_digests:
+        # TODO: avoid this by better adaptations for different registries and single/multi manifests
+        log.debug(
+            "Matching new image_digest against installed repo digest for %s image %s", installed.index_name, installed.name
+        )
+        shortcircuit = "FGB"
+        latest = installed
+
+    if version_policy == VersionPolicy.VERSION and installed.version and latest.version:
+        return installed.version, latest.version, basis("version")
+
+    installed_digest_available: bool = installed.short_digest is not None and installed.short_digest != ""
+    latest_digest_available: bool = latest.short_digest is not None and latest.short_digest != ""
+
+    if version_policy == VersionPolicy.DIGEST and installed_digest_available and latest_digest_available:
+        return installed.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+    if (
+        version_policy == VersionPolicy.VERSION_DIGEST
+        and installed.version
+        and latest.version
+        and installed_digest_available
+        and latest_digest_available
+    ):
+        return (
+            f"{installed.version}:{installed.short_digest}",
+            f"{latest.version}:{latest.short_digest}",
+            basis("version-digest"),
+        )
 
-        self.log.debug("No common or discovered package found", image_name=image_name)
-        return PackageUpdateInfo(
-            DockerPackageUpdateInfo(image_name or NO_KNOWN_IMAGE),
-            logo_url=self.cfg.default_entity_picture_url,
-            release_notes_url=None,
+    phase = 1
+    if version_policy == VersionPolicy.AUTO and (
+        (installed.version == latest.version and installed.short_digest == latest.short_digest)
+        or (installed.version != latest.version and installed.short_digest != latest.short_digest)
+    ):
+        # detect semver, or casual semver (e.g. v1.030)
+        # only use this if both version and digest are consistently agreeing or disagreeing
+        # if the strict conditions work, people see nice version numbers on screen rather than hashes
+        if (
+            installed.version
+            and re.match(SEMVER_RE, installed.version or "")
+            and latest.version
+            and re.match(SEMVER_RE, latest.version or "")
+        ):
+            # Smells like semver, override if not using version_policy
+            return installed.version, latest.version, basis("semver")
+        if (
+            installed.version
+            and re.match(VERSION_RE, installed.version or "")
+            and latest.version
+            and re.match(VERSION_RE, latest.version or "")
+        ):
+            # Smells like casual semver, override if not using version_policy
+            return installed.version, latest.version, basis("causualver")
+
+    # AUTO or fallback
+    phase = 2
+    if installed.version and latest.version and installed_digest_available and latest_digest_available:
+        return (
+            f"{installed.version}:{installed.short_digest}",
+            f"{latest.version}:{latest.short_digest}",
+            basis("version-digest"),
         )
 
-    def discover_metadata(self) -> dict[str, PackageUpdateInfo]:
-        pkgs: dict[str, PackageUpdateInfo] = {}
-        cfg = self.cfg.discover_metadata.get("linuxserver.io")
-        if cfg and cfg.enabled:
-            linuxserver_metadata(pkgs, cache_ttl=cfg.cache_ttl)
-        return pkgs
-
-
-def linuxserver_metadata_api(cache_ttl: int) -> dict:
-    """Fetch and cache linuxserver.io API call for image metadata"""
-    try:
-        with SyncCacheClient(headers=[("cache-control", f"max-age={cache_ttl}")]) as client:
-            log.debug(f"Fetching linuxserver.io metadata from API, cache_ttl={cache_ttl}")
-            req = client.get("https://api.linuxserver.io/api/v1/images?include_config=false&include_deprecated=false")
-            return req.json()
-    except Exception:
-        log.exception("Failed to fetch linuxserver.io metadata")
-        return {}
-
-
-def linuxserver_metadata(discovered_pkgs: dict[str, PackageUpdateInfo], cache_ttl: int) -> None:
-    """Fetch linuxserver.io metadata for all their images via their API"""
-    repos: list = linuxserver_metadata_api(cache_ttl).get("data", {}).get("repositories", {}).get("linuxserver", [])
-    added = 0
-    for repo in repos:
-        image_name = repo.get("name")
-        if image_name and image_name not in discovered_pkgs:
-            discovered_pkgs[image_name] = PackageUpdateInfo(
-                DockerPackageUpdateInfo(f"lscr.io/linuxserver/{image_name}"),
-                logo_url=repo["project_logo"],
-                release_notes_url=f"{repo['github_url']}/releases",
-            )
-            added += 1
-            log.debug("Added linuxserver.io package", pkg=image_name)
-    log.info(f"Added {added} linuxserver.io package details")
+        # and ((other_digest is None and other_version is None) or (other_digest is not None and other_version is not None))
+
+    if installed.version and latest.version:
+        return installed.version, latest.version, basis("version")
+
+    # Check for local builds
+    phase = 3
+    if installed.git_digest and latest.git_digest:
+        return f"git:{installed.git_digest}", f"git:{latest.git_digest}", basis("git")
+
+    # Fall back to digests, image or repo index
+    phase = 4
+    if installed_digest_available and latest_digest_available:
+        return installed.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+    if installed.version and not latest.version and not latest.short_digest and not latest.repo_digest:
+        return installed.version, installed.version, basis("version")
+    phase = 5
+    if not installed_digest_available and latest_digest_available:
+        # odd condition if local image has no identity, even out versions so no update alert
+        return latest.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+
+    # Fall back to repo digests
+    phase = 6
+
+    def condense_repo_id(i: DockerImageInfo) -> str:
+        v: str | None = i.condense_digest(i.repo_digest) if i.repo_digest else None
+        return v or ""
+
+    if installed.repo_digest and latest.repo_digest:
+        # where the image digest isn't available, fall back to a repo digest
+        return condense_repo_id(installed), condense_repo_id(latest), basis("repo-digest")
+
+    phase = 7
+    if latest.repo_digest and latest.repo_digest in installed.repo_digests:
+        # installed has multiple RepoDigests from multiple pulls and one of them matches latest current repo digest
+        return condense_repo_id(latest), condense_repo_id(latest), basis("repo-digest")
+
+    if installed_digest_available and not latest_digest_available:
+        return installed.short_digest, latest.short_digest, basis("digest")  # type: ignore[return-value]
+
+    log.warn("No versions can be determined for %s", installed.ref)
+    phase = 999
+    return UNKNOWN_VERSION, UNKNOWN_VERSION, basis("failure")