updates2mqtt 1.6.0__py3-none-any.whl → 1.7.0__py3-none-any.whl

updates2mqtt/integrations/docker.py

@@ -1,4 +1,3 @@
-import re
 import subprocess
 import time
 import typing
@@ -14,10 +13,25 @@ import docker.errors
 import structlog
 from docker.auth import resolve_repository_name
 from docker.models.containers import Container
-from hishel.httpx import SyncCacheClient
 
-from updates2mqtt.config import DockerConfig, DockerPackageUpdateInfo, NodeConfig, PackageUpdateInfo
-from updates2mqtt.model import Discovery, ReleaseProvider
+from updates2mqtt.config import (
+    NO_KNOWN_IMAGE,
+    DockerConfig,
+    NodeConfig,
+    PackageUpdateInfo,
+    PublishPolicy,
+    UpdatePolicy,
+)
+from updates2mqtt.integrations.docker_enrich import (
+    AuthError,
+    CommonPackageEnricher,
+    DefaultPackageEnricher,
+    LabelEnricher,
+    LinuxServerIOPackageEnricher,
+    PackageEnricher,
+    SourceReleaseEnricher,
+)
+from updates2mqtt.model import Discovery, ReleaseProvider, Selection, VersionPolicy, select_version
 
 from .git_utils import git_check_update_available, git_iso_timestamp, git_local_version, git_pull, git_trust
 
@@ -27,7 +41,6 @@ if typing.TYPE_CHECKING:
 # distinguish docker build from docker pull?
 
 log = structlog.get_logger()
-NO_KNOWN_IMAGE = "UNKNOWN"
 
 
 class DockerComposeCommand(Enum):
@@ -51,8 +64,8 @@ class ContainerCustomization:
         self.picture: str | None = None
         self.relnotes: str | None = None
         self.ignore: bool = False
-        self.version_include: str | None = None
-        self.version_exclude: str | None = None
+        self.version_policy: VersionPolicy | None = None
+        self.registry_token: str | None = None
 
         if not container.attrs or container.attrs.get("Config") is None:
             return
@@ -93,6 +106,8 @@ class ContainerCustomization:
                 if v is not None:
                     if isinstance(getattr(self, attr), bool):
                         setattr(self, attr, v.upper() in ("TRUE", "YES", "1"))
+                    elif isinstance(getattr(self, attr), VersionPolicy):
+                        setattr(self, attr, VersionPolicy[v.upper()])
                     else:
                         setattr(self, attr, v)
 
@@ -103,20 +118,28 @@ class DockerProvider(ReleaseProvider):
     def __init__(
         self,
         cfg: DockerConfig,
-        common_pkg_cfg: dict[str, PackageUpdateInfo],
         node_cfg: NodeConfig,
         self_bounce: Event | None = None,
     ) -> None:
-        super().__init__("docker")
+        super().__init__(node_cfg, "docker")
         self.client: docker.DockerClient = docker.from_env()
         self.cfg: DockerConfig = cfg
-        self.node_cfg: NodeConfig = node_cfg
-        self.common_pkgs: dict[str, PackageUpdateInfo] = common_pkg_cfg if common_pkg_cfg else {}
+
         # TODO: refresh discovered packages periodically
-        self.discovered_pkgs: dict[str, PackageUpdateInfo] = self.discover_metadata()
         self.pause_api_until: dict[str, float] = {}
-        self.api_throttle_pause: int = cfg.api_throttle_wait
+        self.api_throttle_pause: int = cfg.default_api_backoff
         self.self_bounce: Event | None = self_bounce
+        self.pkg_enrichers: list[PackageEnricher] = [
+            CommonPackageEnricher(self.cfg),
+            LinuxServerIOPackageEnricher(self.cfg),
+            DefaultPackageEnricher(self.cfg),
+        ]
+        self.label_enricher = LabelEnricher()
+        self.release_enricher = SourceReleaseEnricher()
+
+    def initialize(self) -> None:
+        for enricher in self.pkg_enrichers:
+            enricher.initialize()
 
     def update(self, discovery: Discovery) -> bool:
         logger: Any = self.log.bind(container=discovery.name, action="update")
@@ -221,7 +244,7 @@ class DockerProvider(ReleaseProvider):
         try:
             c: Container = self.client.containers.get(discovery.name)
             if c:
-                rediscovery = self.analyze(c, discovery.session, original_discovery=discovery)
+                rediscovery = self.analyze(c, discovery.session, previous_discovery=discovery)
                 if rediscovery:
                     self.discoveries[rediscovery.name] = rediscovery
                     return rediscovery
@@ -236,13 +259,13 @@ class DockerProvider(ReleaseProvider):
         if self.pause_api_until.get(repo_id) is not None:
             if self.pause_api_until[repo_id] < time.time():
                 del self.pause_api_until[repo_id]
-                log.info("%s throttling wait complete", repo_id)
+                self.log.info("%s throttling wait complete", repo_id)
             else:
-                log.debug("%s throttling has %s secs left", repo_id, self.pause_api_until[repo_id] - time.time())
+                self.log.debug("%s throttling has %s secs left", repo_id, self.pause_api_until[repo_id] - time.time())
                 return True
         return False
 
-    def analyze(self, c: Container, session: str, original_discovery: Discovery | None = None) -> Discovery | None:
+    def analyze(self, c: Container, session: str, previous_discovery: Discovery | None = None) -> Discovery | None:
         logger = self.log.bind(container=c.name, action="analyze")
 
         image_ref: str | None = None
@@ -281,12 +304,24 @@ class DockerProvider(ReleaseProvider):
                     logger.warn("Cannot determine local version: %s", e)
                     logger.warn("RepoDigests=%s", image.attrs.get("RepoDigests"))
 
+        selection = Selection(self.cfg.image_ref_select, image_ref)
+        publish_policy: PublishPolicy = PublishPolicy.MQTT if not selection.result else PublishPolicy.HOMEASSISTANT
+        version_policy: VersionPolicy = VersionPolicy.AUTO if not customization.version_policy else customization.version_policy
+
+        if customization.update == "AUTO":
+            logger.debug("Auto update policy detected")
+            update_policy: UpdatePolicy = UpdatePolicy.AUTO
+        else:
+            update_policy = UpdatePolicy.PASSIVE
+
         platform: str = "Unknown"
         pkg_info: PackageUpdateInfo = self.default_metadata(image_name, image_ref=image_ref)
 
         try:
-            picture_url = customization.picture or pkg_info.logo_url
-            relnotes_url = customization.relnotes or pkg_info.release_notes_url
+            picture_url: str | None = customization.picture or pkg_info.logo_url
+            relnotes_url: str | None = customization.relnotes or pkg_info.release_notes_url
+            release_summary: str | None = None
+
             if image is not None and image.attrs is not None:
                 platform = "/".join(
                     filter(
@@ -300,8 +335,10 @@ class DockerProvider(ReleaseProvider):
                 )
 
             reg_data: RegistryData | None = None
-            latest_version: str | None = NO_KNOWN_IMAGE
-            registry_throttled = self.check_throttle(repo_id)
+            latest_digest: str | None = NO_KNOWN_IMAGE
+            latest_version: str | None = None
+
+            registry_throttled: bool = self.check_throttle(repo_id)
 
             if image_ref and local_versions and not registry_throttled:
                 retries_left = 3
@@ -309,17 +346,23 @@ class DockerProvider(ReleaseProvider):
                     try:
                         logger.debug("Fetching registry data", image_ref=image_ref)
                         reg_data = self.client.images.get_registry_data(image_ref)
-                        log.debug(
+                        logger.debug(
                             "Registry Data: id:%s,image:%s, attrs:%s",
                             reg_data.id,
                             reg_data.image_name,
                             reg_data.attrs,
                         )
-                        latest_version = reg_data.short_id[7:] if reg_data else None
+                        latest_digest = reg_data.short_id[7:] if reg_data else None
+
                     except docker.errors.APIError as e:
                         if e.status_code == HTTPStatus.TOO_MANY_REQUESTS:
-                            logger.warn("Docker Registry throttling requests, %s", e.explanation)
-                            self.pause_api_until[repo_id] = time.time() + self.api_throttle_pause
+                            retry_secs: int
+                            try:
+                                retry_secs = int(e.response.headers.get("Retry-After", self.api_throttle_pause))  # type: ignore[union-attr]
+                            except:  # noqa: E722
+                                retry_secs = self.api_throttle_pause
+                            logger.warn("Docker Registry throttling requests for %s seconds, %s", retry_secs, e.explanation)
+                            self.pause_api_until[repo_id] = time.time() + retries_left
                             return None
                         retries_left -= 1
                         if retries_left == 0 or e.is_client_error():
@@ -327,11 +370,12 @@ class DockerProvider(ReleaseProvider):
                         else:
                             logger.debug("Failed to fetch registry data, retrying: %s", e)
 
-            local_version: str | None = NO_KNOWN_IMAGE
+            installed_digest: str | None = NO_KNOWN_IMAGE
+            installed_version: str | None = None
             if local_versions:
                 # might be multiple RepoDigests if image has been pulled multiple times with diff manifests
-                local_version = latest_version if latest_version in local_versions else local_versions[0]
-                log.debug(f"Setting local version to {local_version}, local_versions:{local_versions}")
+                installed_digest = latest_digest if latest_digest in local_versions else local_versions[0]
+                logger.debug(f"Setting local digest to {installed_digest}, local_versions:{local_versions}")
 
             def save_if_set(key: str, val: str | None) -> None:
                 if val is not None:
@@ -339,23 +383,55 @@ class DockerProvider(ReleaseProvider):
 
             image_ref = image_ref or ""
 
-            custom: dict[str, str | bool] = {}
+            custom: dict[str, str | bool | int | list[str] | dict[str, Any] | None] = {}
             custom["platform"] = platform
             custom["image_ref"] = image_ref
+            custom["installed_digest"] = installed_digest
+            custom["latest_digest"] = latest_digest
             custom["repo_id"] = repo_id
-            if registry_throttled:
-                custom["registry_throttled"] = True
-            save_if_set("compose_path", c.labels.get("com.docker.compose.project.working_dir"))
-            save_if_set("compose_version", c.labels.get("com.docker.compose.version"))
-            save_if_set("compose_service", c.labels.get("com.docker.compose.service"))
-            save_if_set("git_repo_path", customization.git_repo_path)
-            # save_if_set("apt_pkgs", c_env.get("UPD2MQTT_APT_PKGS"))
-
-            if customization.update == "AUTO":
-                logger.debug("Auto update policy detected")
-                update_policy = "Auto"
+            custom["git_repo_path"] = customization.git_repo_path
+
+            if c.labels:
+                save_if_set("compose_path", c.labels.get("com.docker.compose.project.working_dir"))
+                save_if_set("compose_version", c.labels.get("com.docker.compose.version"))
+                save_if_set("compose_service", c.labels.get("com.docker.compose.service"))
+                save_if_set("documentation_url", c.labels.get("org.opencontainers.image.documentation"))
+                save_if_set("description", c.labels.get("org.opencontainers.image.description"))
+                save_if_set("current_image_created", c.labels.get("org.opencontainers.image.created"))
+                save_if_set("current_image_version", c.labels.get("org.opencontainers.image.version"))
+                save_if_set("vendor", c.labels.get("org.opencontainers.image.vendor"))
+                installed_version = c.labels.get("org.opencontainers.image.version")
             else:
-                update_policy = "Passive"
+                logger.debug("No annotations found on local container")
+                # save_if_set("apt_pkgs", c_env.get("UPD2MQTT_APT_PKGS"))
+
+            if latest_digest is None or latest_digest == NO_KNOWN_IMAGE or registry_throttled:
+                logger.debug(
+                    "Skipping image manifest enrichment",
+                    latest_digest=latest_digest,
+                    image_ref=image_ref,
+                    platform=platform,
+                    throttled=registry_throttled,
+                )
+            else:
+                os, arch = platform.split("/")[:2] if "/" in platform else (platform, "Unknown")
+                try:
+                    annotations: dict[str, str] = self.label_enricher.fetch_annotations(
+                        image_ref, os, arch, token=customization.registry_token
+                    )
+                except AuthError as e:
+                    logger.warning("Authentication error prevented Docker Registry entichment: %s", e)
+                    annotations = {}
+
+                if annotations:
+                    save_if_set("latest_image_created", annotations.get("org.opencontainers.image.created"))
+                    save_if_set("source", annotations.get("org.opencontainers.image.source"))
+                    save_if_set("documentation_url", annotations.get("org.opencontainers.image.documentation"))
+                    save_if_set("description", annotations.get("org.opencontainers.image.description"))
+                    save_if_set("latest_image_version", annotations.get("org.opencontainers.image.version"))
+                    save_if_set("vendor", annotations.get("org.opencontainers.image.vendor"))
+                    latest_version = annotations.get("org.opencontainers.image.version")
+                    custom.update(self.release_enricher.enrich(annotations) or {})
 
             if custom.get("git_repo_path") and custom.get("compose_path"):
                 full_repo_path: Path = Path(cast("str", custom.get("compose_path"))).joinpath(
@@ -369,43 +445,33 @@ class DockerProvider(ReleaseProvider):
                 self.cfg.allow_pull
                 and image_ref is not None
                 and image_ref != ""
-                and (local_version != NO_KNOWN_IMAGE or latest_version != NO_KNOWN_IMAGE)
+                and (installed_digest != NO_KNOWN_IMAGE or latest_digest != NO_KNOWN_IMAGE)
             )
             if self.cfg.allow_pull and not can_pull:
                 logger.debug(
-                    f"Pull not available, image_ref:{image_ref},local_version:{local_version},latest_version:{latest_version}"
+                    f"Pull unavailable, image_ref:{image_ref},installed_digest:{installed_digest},latest_digest:{latest_digest}"
                 )
-            skip_pull: bool = False
-            if can_pull and latest_version is not None:
-                if customization.version_include and not re.match(customization.version_include, latest_version):
-                    logger.info(f"Skipping version {latest_version} not matching include pattern")
-                    skip_pull = True
-                    latest_version = local_version
-                if customization.version_exclude and re.match(customization.version_exclude, latest_version):  # type: ignore[arg-type]
-                    logger.info(f"Skipping version {latest_version} matching exclude pattern")
-                    skip_pull = True
-                    latest_version = local_version
 
             can_build: bool = False
             if self.cfg.allow_build:
                 can_build = custom.get("git_repo_path") is not None and custom.get("compose_path") is not None
                 if not can_build:
                     if custom.get("git_repo_path") is not None:
-                        log.debug(
+                        logger.debug(
                             "Local build ignored for git_repo_path=%s because no compose_path", custom.get("git_repo_path")
                         )
                 else:
                     full_repo_path = self.full_repo_path(
                         cast("str", custom.get("compose_path")), cast("str", custom.get("git_repo_path"))
                     )
-                    if local_version is None or local_version == NO_KNOWN_IMAGE:
-                        local_version = git_local_version(full_repo_path, Path(self.node_cfg.git_path)) or NO_KNOWN_IMAGE
+                    if installed_digest is None or installed_digest == NO_KNOWN_IMAGE:
+                        installed_digest = git_local_version(full_repo_path, Path(self.node_cfg.git_path)) or NO_KNOWN_IMAGE
 
                     behind_count: int = git_check_update_available(full_repo_path, Path(self.node_cfg.git_path))
                     if behind_count > 0:
-                        if local_version is not None and local_version.startswith("git:"):
-                            latest_version = f"{local_version}+{behind_count}"
-                            log.info("Git update available, generating version %s", latest_version)
+                        if installed_digest is not None and installed_digest.startswith("git:"):
+                            latest_digest = f"{installed_digest}+{behind_count}"
+                            logger.info("Git update available, generating version %s", latest_digest)
                     else:
                         logger.debug(f"Git update not available, local repo:{full_repo_path}")
 
@@ -422,19 +488,23 @@ class DockerProvider(ReleaseProvider):
                 logger.info(f"Update not available, can_pull:{can_pull}, can_build:{can_build},can_restart{can_restart}")
             if relnotes_url:
                 features.append("RELEASE_NOTES")
-            if skip_pull:
-                update_type: str = "Skipped"
-            elif can_pull:
+            if can_pull:
                 update_type = "Docker Image"
             elif can_build:
                 update_type = "Docker Build"
             else:
                 update_type = "Unavailable"
             custom["can_pull"] = can_pull
-            custom["skip_pull"] = skip_pull
             # can_pull,can_build etc are only info flags
             # the HASS update process is driven by comparing current and available versions
 
+            public_installed_version = select_version(
+                version_policy, installed_version, installed_digest, other_version=latest_version, other_digest=latest_digest
+            )
+            public_latest_version = select_version(
+                version_policy, latest_version, latest_digest, other_version=installed_version, other_digest=installed_digest
+            )
+
             discovery: Discovery = Discovery(
                 self,
                 c.name,
@@ -442,10 +512,12 @@ class DockerProvider(ReleaseProvider):
                 node=self.node_cfg.name,
                 entity_picture_url=picture_url,
                 release_url=relnotes_url,
-                current_version=local_version,
+                release_summary=release_summary,
+                current_version=public_installed_version,
+                publish_policy=publish_policy,
                 update_policy=update_policy,
-                update_last_attempt=original_discovery.update_last_attempt if original_discovery else None,
-                latest_version=latest_version if latest_version != NO_KNOWN_IMAGE else local_version,
+                version_policy=version_policy,
+                latest_version=public_latest_version,
                 device_icon=self.cfg.device_icon,
                 can_update=can_update,
                 update_type=update_type,
@@ -455,6 +527,7 @@ class DockerProvider(ReleaseProvider):
                 custom=custom,
                 features=features,
                 throttled=registry_throttled,
+                previous=previous_discovery,
             )
             logger.debug("Analyze generated discovery: %s", discovery)
             return discovery
@@ -463,6 +536,10 @@ class DockerProvider(ReleaseProvider):
         logger.debug("Analyze returned empty discovery")
         return None
 
+    # def version(self, c: Container, version_type: str):
+    #    metadata_version: str = c.labels.get("org.opencontainers.image.version")
+    #    metadata_revision: str = c.labels.get("org.opencontainers.image.revision")
+
     async def scan(self, session: str) -> AsyncGenerator[Discovery]:
         logger = self.log.bind(session=session, action="scan", source=self.source_type)
         containers: int = 0
@@ -521,87 +598,9 @@ class DockerProvider(ReleaseProvider):
     def resolve(self, discovery_name: str) -> Discovery | None:
         return self.discoveries.get(discovery_name)
 
-    def hass_state_format(self, discovery: Discovery) -> dict:  # noqa: ARG002
-        # disable since hass mqtt update has strict json schema for message
-        return {
-            # "docker_image_ref": discovery.custom.get("image_ref"),
-            # "last_update_attempt": safe_json_dt(discovery.update_last_attempt),
-            # "can_pull": discovery.custom.get("can_pull"),
-            # "can_build": discovery.custom.get("can_build"),
-            # "can_restart": discovery.custom.get("can_restart"),
-            # "git_repo_path": discovery.custom.get("git_repo_path"),
-            # "compose_path": discovery.custom.get("compose_path"),
-            # "platform": discovery.custom.get("platform"),
-        }
-
     def default_metadata(self, image_name: str | None, image_ref: str | None) -> PackageUpdateInfo:
-        def match(pkg: PackageUpdateInfo) -> bool:
-            if pkg is not None and pkg.docker is not None and pkg.docker.image_name is not None:
-                if image_name is not None and image_name == pkg.docker.image_name:
-                    return True
-                if image_ref is not None and image_ref == pkg.docker.image_name:
-                    return True
-            return False
-
-        if image_name is not None and image_ref is not None:
-            for pkg in self.common_pkgs.values():
-                if match(pkg):
-                    self.log.debug(
-                        "Found common package",
-                        image_name=pkg.docker.image_name,  # type: ignore [union-attr]
-                        logo_url=pkg.logo_url,
-                        relnotes_url=pkg.release_notes_url,
-                    )
-                    return pkg
-            for pkg in self.discovered_pkgs.values():
-                if match(pkg):
-                    self.log.debug(
-                        "Found discovered package",
-                        pkg=pkg.docker.image_name,  # type: ignore [union-attr]
-                        logo_url=pkg.logo_url,
-                        relnotes_url=pkg.release_notes_url,
-                    )
-                    return pkg
-
-        self.log.debug("No common or discovered package found", image_name=image_name)
-        return PackageUpdateInfo(
-            DockerPackageUpdateInfo(image_name or NO_KNOWN_IMAGE),
-            logo_url=self.cfg.default_entity_picture_url,
-            release_notes_url=None,
-        )
-
-    def discover_metadata(self) -> dict[str, PackageUpdateInfo]:
-        pkgs: dict[str, PackageUpdateInfo] = {}
-        cfg = self.cfg.discover_metadata.get("linuxserver.io")
-        if cfg and cfg.enabled:
-            linuxserver_metadata(pkgs, cache_ttl=cfg.cache_ttl)
-        return pkgs
-
-
-def linuxserver_metadata_api(cache_ttl: int) -> dict:
-    """Fetch and cache linuxserver.io API call for image metadata"""
-    try:
-        with SyncCacheClient(headers=[("cache-control", f"max-age={cache_ttl}")]) as client:
-            log.debug(f"Fetching linuxserver.io metadata from API, cache_ttl={cache_ttl}")
-            req = client.get("https://api.linuxserver.io/api/v1/images?include_config=false&include_deprecated=false")
-            return req.json()
-    except Exception:
-        log.exception("Failed to fetch linuxserver.io metadata")
-        return {}
-
-
-def linuxserver_metadata(discovered_pkgs: dict[str, PackageUpdateInfo], cache_ttl: int) -> None:
-    """Fetch linuxserver.io metadata for all their images via their API"""
-    repos: list = linuxserver_metadata_api(cache_ttl).get("data", {}).get("repositories", {}).get("linuxserver", [])
-    added = 0
-    for repo in repos:
-        image_name = repo.get("name")
-        if image_name and image_name not in discovered_pkgs:
-            discovered_pkgs[image_name] = PackageUpdateInfo(
-                DockerPackageUpdateInfo(f"lscr.io/linuxserver/{image_name}"),
-                logo_url=repo["project_logo"],
-                release_notes_url=f"{repo['github_url']}/releases",
-            )
-            added += 1
-            log.debug("Added linuxserver.io package", pkg=image_name)
-    log.info(f"Added {added} linuxserver.io package details")
+        for enricher in self.pkg_enrichers:
+            pkg_info = enricher.enrich(image_name, image_ref, self.log)
+            if pkg_info is not None:
+                return pkg_info
+        raise ValueError("No enricher could provide metadata, not even default enricher")