updates2mqtt 1.7.3__py3-none-any.whl → 1.8.1__py3-none-any.whl

updates2mqtt/integrations/docker_enrich.py

@@ -14,6 +14,7 @@ from updates2mqtt.model import DiscoveryArtefactDetail, DiscoveryInstallationDet
 
 if typing.TYPE_CHECKING:
     from docker.models.images import RegistryData
+    from omegaconf.dictconfig import DictConfig
 from http import HTTPStatus
 
 import docker
@@ -21,23 +22,35 @@ import docker.errors
 
 from updates2mqtt.config import (
     PKG_INFO_FILE,
+    CommonPackages,
     DockerConfig,
     DockerPackageUpdateInfo,
+    GitHubConfig,
     PackageUpdateInfo,
     RegistryConfig,
-    UpdateInfoConfig,
+    VersionPolicy,
 )
 
-log = structlog.get_logger()
+log: Any = structlog.get_logger()
 
 SOURCE_PLATFORM_GITHUB = "GitHub"
 SOURCE_PLATFORM_CODEBERG = "CodeBerg"
-SOURCE_PLATFORMS = {SOURCE_PLATFORM_GITHUB: r"https://github.com/.*"}
+SOURCE_PLATFORM_GITLAB = "GitLab"
+SOURCE_PLATFORMS = {
+    SOURCE_PLATFORM_GITHUB: r"https://github.com/.*",
+    SOURCE_PLATFORM_GITLAB: r"https://gitlab.com/.*",
+    SOURCE_PLATFORM_CODEBERG: r"https://codeberg.org/.*",
+}
 DIFF_URL_TEMPLATES = {
     SOURCE_PLATFORM_GITHUB: "{repo}/commit/{revision}",
 }
-RELEASE_URL_TEMPLATES = {SOURCE_PLATFORM_GITHUB: "{repo}/releases/tag/{version}"}
-UNKNOWN_RELEASE_URL_TEMPLATES = {SOURCE_PLATFORM_GITHUB: "{repo}/releases"}
+RELEASE_URL_TEMPLATES = {
+    SOURCE_PLATFORM_GITHUB: "{repo}/releases/tag/{version}",
+}
+UNKNOWN_RELEASE_URL_TEMPLATES = {
+    SOURCE_PLATFORM_GITHUB: "{repo}/releases",
+    SOURCE_PLATFORM_GITLAB: "{repo}/container_registry",
+}
 MISSING_VAL = "**MISSING**"
 UNKNOWN_REGISTRY = "**UNKNOWN_REGISTRY**"
 
@@ -45,18 +58,27 @@ HEADER_DOCKER_DIGEST = "docker-content-digest"
 HEADER_DOCKER_API = "docker-distribution-api-version"
 
 TOKEN_URL_TEMPLATE = "https://{auth_host}/token?scope=repository:{image_name}:pull&service={service}"  # noqa: S105 # nosec
-REGISTRIES = {
-    # registry: (auth_host, api_host, service, url_template)
-    "docker.io": ("auth.docker.io", "registry-1.docker.io", "registry.docker.io", TOKEN_URL_TEMPLATE),
-    "mcr.microsoft.com": (None, "mcr.microsoft.com", "mcr.microsoft.com", TOKEN_URL_TEMPLATE),
-    "ghcr.io": ("ghcr.io", "ghcr.io", "ghcr.io", TOKEN_URL_TEMPLATE),
-    "lscr.io": ("ghcr.io", "lscr.io", "ghcr.io", TOKEN_URL_TEMPLATE),
-    "codeberg.org": ("codeberg.org", "codeberg.org", "container_registry", TOKEN_URL_TEMPLATE),
+
+REGISTRIES: dict[str, tuple[str | None, str, str, str | None, str | None]] = {
+    # registry: (auth_host, api_host, service, url_template, repo_template)
+    "docker.io": ("auth.docker.io", "registry-1.docker.io", "registry.docker.io", TOKEN_URL_TEMPLATE, None),
+    "mcr.microsoft.com": (None, "mcr.microsoft.com", "mcr.microsoft.com", None, None),
+    "quay.io": (None, "quay.io", "quay.io", TOKEN_URL_TEMPLATE, None),
+    "ghcr.io": ("ghcr.io", "ghcr.io", "ghcr.io", TOKEN_URL_TEMPLATE, "https://github.com/{image_name}"),
+    "lscr.io": ("ghcr.io", "lscr.io", "ghcr.io", TOKEN_URL_TEMPLATE, None),
+    "codeberg.org": (
+        "codeberg.org",
+        "codeberg.org",
+        "container_registry",
+        TOKEN_URL_TEMPLATE,
+        "https://codeberg.org/{image_name}",
+    ),
     "registry.gitlab.com": (
         "www.gitlab.com",
         "registry.gitlab.com",
         "container_registry",
         "https://{auth_host}/jwt/auth?service={service}&scope=repository:{image_name}:pull&offline_token=true&client_id=docker",
+        "https://gitlab.com/{image_name}",
     ),
 }
 
@@ -83,7 +105,9 @@ class DockerImageInfo(DiscoveryArtefactDetail):
         annotations: dict[str, Any] | None = None,
         platform: str | None = None,  # test harness simplification
         version: str | None = None,  # test harness simplification
+        created: str | None = None,
     ) -> None:
+        super().__init__()
         self.ref: str = ref
         self.version: str | None = version
         self.image_digest: str | None = image_digest
@@ -105,6 +129,7 @@ class DockerImageInfo(DiscoveryArtefactDetail):
         self.error: str | None = None
         self.platform: str | None = platform
         self.custom: dict[str, str | float | int | bool | None] = {}
+        self.created: str | None = created
 
         self.local_build: bool = not self.repo_digests
         self.index_name, remote_name = resolve_repository_name(ref)
@@ -190,16 +215,20 @@ class DockerImageInfo(DiscoveryArtefactDetail):
                 digest = digest.split(":")[1] if ":" in digest else digest  # remove digest type prefix
                 return digest[0:12]
             return digest
-        except Exception:
+        except Exception as e:
+            log.warning("Unable to condense digest %s: %s", digest, e)
             return None
 
     def reuse(self) -> "DockerImageInfo":
-        cloned = DockerImageInfo(self.ref, self.image_digest, self.tags, self.attributes, self.annotations, self.version)
+        cloned = DockerImageInfo(
+            self.ref, self.image_digest, self.tags, self.attributes, self.annotations, self.version, self.created
+        )
         cloned.origin = "REUSED"
         return cloned
 
     def as_dict(self, minimal: bool = True) -> dict[str, str | list | dict | bool | int | None]:
         result: dict[str, str | list | dict | bool | int | None] = {
+            "captured": self.captured.isoformat(),
             "image_ref": self.ref,
             "name": self.name,
             "version": self.version,
@@ -246,7 +275,7 @@ def _select_annotation(
 
 
 def cherrypick_annotations(
-    local_info: DockerImageInfo | None, registry_info: DockerImageInfo | None
+    local_info: DockerImageInfo, registry_info: DockerImageInfo | None
 ) -> dict[str, str | float | int | bool | None]:
     """https://github.com/opencontainers/image-spec/blob/main/annotations.md"""
     results: dict[str, str | float | int | bool | None] = {}
@@ -258,11 +287,25 @@ def cherrypick_annotations(
         ("image_created", "org.opencontainers.image.created"),
         ("image_version", "org.opencontainers.image.version"),
         ("image_revision", "org.opencontainers.image.revision"),
+        ("ref_name", "org.opencontainers.image.ref.name"),
         ("title", "org.opencontainers.image.title"),
         ("vendor", "org.opencontainers.image.vendor"),
         ("source", "org.opencontainers.image.source"),
     ]:
         results.update(_select_annotation(either_name, either_label, local_info, registry_info))
+    if (
+        results.get("ref_name") == "ubuntu"
+        and local_info.name != "ubuntu"
+        and results.get("image_version")
+        and re.fullmatch(r"^2\d\.\d\d$", cast("str", results["image_version"]))
+    ):
+        log.debug(
+            "Suppressing %s base %s version leaking into image version: %s",
+            local_info.name,
+            results["ref_name"],
+            results["image_version"],
+        )
+        del results["image_version"]
     return results
 
 
@@ -322,14 +365,17 @@ class LocalContainerInfo:
         labels: dict[str, str | float | int | bool | None] = cherrypick_annotations(image_info, None)
         # capture container labels/annotations, not image ones
         labels = labels or {}
+        if container.image and container.image.attrs:
+            image_info.created = container.image.attrs.get("Created")
         image_info.custom = labels
         image_info.version = cast("str|None", labels.get("image_version"))
+
         return image_info, service_info
 
 
 class PackageEnricher:
-    def __init__(self, docker_cfg: DockerConfig) -> None:
-        self.pkgs: dict[str, PackageUpdateInfo] = {}
+    def __init__(self, docker_cfg: DockerConfig, packages: dict[str, PackageUpdateInfo] | None = None) -> None:
+        self.pkgs: dict[str, PackageUpdateInfo] = packages or {}
         self.cfg: DockerConfig = docker_cfg
         self.log: Any = structlog.get_logger().bind(integration="docker")
 
@@ -362,7 +408,7 @@ class DefaultPackageEnricher(PackageEnricher):
     def enrich(self, image_info: DockerImageInfo) -> PackageUpdateInfo | None:
         self.log.debug("Default pkg info", image_name=image_info.untagged_ref, image_ref=image_info.ref)
         return PackageUpdateInfo(
-            DockerPackageUpdateInfo(image_info.untagged_ref or image_info.ref),
+            DockerPackageUpdateInfo(image_info.untagged_ref or image_info.ref, version_policy=VersionPolicy.AUTO),
             logo_url=self.cfg.default_entity_picture_url,
             release_notes_url=None,
         )
@@ -370,19 +416,25 @@ class DefaultPackageEnricher(PackageEnricher):
 
 class CommonPackageEnricher(PackageEnricher):
     def initialize(self) -> None:
+        base_cfg: DictConfig = OmegaConf.structured(CommonPackages)
         if PKG_INFO_FILE.exists():
-            log.debug("Loading common package update info", path=PKG_INFO_FILE)
-            cfg = OmegaConf.load(PKG_INFO_FILE)
+            self.log.debug("Loading common package update info", path=PKG_INFO_FILE)
+            cfg: DictConfig = typing.cast("DictConfig", OmegaConf.merge(base_cfg, OmegaConf.load(PKG_INFO_FILE)))
+
+            OmegaConf.to_container(cfg, throw_on_missing=True)
+            OmegaConf.set_readonly(cfg, True)
         else:
-            log.warn("No common package update info found", path=PKG_INFO_FILE)
-            cfg = OmegaConf.structured(UpdateInfoConfig)
+            self.log.warn("No common package update info found", path=PKG_INFO_FILE)
+            cfg = base_cfg
         try:
+            common_config: CommonPackages = typing.cast("CommonPackages", cfg)
             # omegaconf broken-ness on optional fields and converting to backclasses
-            self.pkgs: dict[str, PackageUpdateInfo] = {
-                pkg: PackageUpdateInfo(**pkg_cfg) for pkg, pkg_cfg in cfg.common_packages.items()
-            }
+            self.pkgs = common_config.common_packages
+            # self.pkgs: dict[str, PackageUpdateInfo] = {
+            #    pkg: PackageUpdateInfo(**pkg_cfg) for pkg, pkg_cfg in cfg.common_packages.items() if pkg not in self.pkgs
+            # }
         except (MissingMandatoryValue, ValidationError) as e:
-            log.error("Configuration error %s", e, path=PKG_INFO_FILE.as_posix())
+            self.log.serror("Configuration error %s", e, path=PKG_INFO_FILE.as_posix())
             raise
 
 
@@ -392,7 +444,7 @@ class LinuxServerIOPackageEnricher(PackageEnricher):
         if cfg is None or not cfg.enabled:
             return
 
-        log.debug(f"Fetching linuxserver.io metadata from API, cache_ttl={cfg.cache_ttl}")
+        self.log.debug(f"Fetching linuxserver.io metadata from API, cache_ttl={cfg.cache_ttl}")
         response: Response | None = fetch_url(
             "https://api.linuxserver.io/api/v1/images?include_config=false&include_deprecated=false",
             cache_ttl=cfg.cache_ttl,
@@ -413,26 +465,38 @@ class LinuxServerIOPackageEnricher(PackageEnricher):
                     release_notes_url=f"{repo['github_url']}/releases",
                 )
                 added += 1
-                log.debug("Added linuxserver.io package", pkg=image_name)
-        log.info(f"Added {added} linuxserver.io package details")
+        self.log.info(f"Added {added} linuxserver.io package details")
 
 
 class SourceReleaseEnricher:
-    def __init__(self) -> None:
+    def __init__(self, gh_cfg: GitHubConfig | None = None) -> None:
         self.log: Any = structlog.get_logger().bind(integration="docker")
+        self.gh_cfg: GitHubConfig | None = gh_cfg
 
     def enrich(
         self, registry_info: DockerImageInfo, source_repo_url: str | None = None, notes_url: str | None = None
     ) -> ReleaseDetail | None:
-        if not registry_info.annotations and not source_repo_url and not notes_url:
-            return None
-
         detail = ReleaseDetail()
 
         detail.notes_url = notes_url
         detail.version = registry_info.annotations.get("org.opencontainers.image.version")
         detail.revision = registry_info.annotations.get("org.opencontainers.image.revision")
-        detail.source_url = registry_info.annotations.get("org.opencontainers.image.source") or source_repo_url
+        # explicit source_repo_url overrides container, e.g. where container source is only the docker wrapper
+        detail.source_url = source_repo_url or registry_info.annotations.get("org.opencontainers.image.source")
+
+        if detail.source_url is None and registry_info is not None and registry_info.index_name is not None:
+            registry_config: tuple[str | None, str, str, str | None, str | None] | None = REGISTRIES.get(
+                registry_info.index_name
+            )
+            repo_template: str | None = registry_config[4] if registry_config else None
+            if repo_template:
+                source_url = repo_template.format(image_name=registry_info.name)
+                if validate_url(source_url, cache_ttl=86400):
+                    detail.source_url = source_url
+                    self.log.info("Implied source from registry: %s", detail.source_url)
+
+        if detail.source_url is None and detail.notes_url is None and detail.revision is None and detail.version is None:
+            return None
 
         if detail.source_url and "#" in detail.source_url:
             detail.source_repo_url = detail.source_url.split("#", 1)[0]
@@ -452,30 +516,77 @@ class SourceReleaseEnricher:
             "source": detail.source_url or MISSING_VAL,
         }
 
-        diff_url: str | None = DIFF_URL_TEMPLATES[detail.source_platform].format(**template_vars)
-        if diff_url and MISSING_VAL not in diff_url and validate_url(diff_url):
+        diff_url_template: str | None = DIFF_URL_TEMPLATES.get(detail.source_platform)
+        diff_url: str | None = diff_url_template.format(**template_vars) if diff_url_template else None
+        if diff_url and MISSING_VAL not in diff_url and validate_url(diff_url, cache_ttl=3600):
             detail.diff_url = diff_url
         else:
             diff_url = None
 
-        if detail.notes_url is None:
-            detail.notes_url = RELEASE_URL_TEMPLATES[detail.source_platform].format(**template_vars)
-
-            if MISSING_VAL in detail.notes_url or not validate_url(detail.notes_url):
-                detail.notes_url = UNKNOWN_RELEASE_URL_TEMPLATES[detail.source_platform].format(**template_vars)
-                if MISSING_VAL in detail.notes_url or not validate_url(detail.notes_url):
-                    detail.notes_url = None
-
-        if detail.source_platform == SOURCE_PLATFORM_GITHUB and detail.source_repo_url:
+        if detail.notes_url is None and detail.source_platform in RELEASE_URL_TEMPLATES:
+            platform_notes_url: str | None = RELEASE_URL_TEMPLATES[detail.source_platform].format(**template_vars)
+            if (
+                platform_notes_url
+                and MISSING_VAL not in platform_notes_url
+                and validate_url(platform_notes_url, cache_ttl=86400)
+            ):
+                self.log.debug("Setting default known release notes url: %s", platform_notes_url)
+                detail.notes_url = platform_notes_url
+
+        if detail.notes_url is None and detail.source_platform in UNKNOWN_RELEASE_URL_TEMPLATES:
+            platform_notes_url = UNKNOWN_RELEASE_URL_TEMPLATES[detail.source_platform].format(**template_vars)
+            if (
+                platform_notes_url
+                and MISSING_VAL not in platform_notes_url
+                and validate_url(platform_notes_url, cache_ttl=86400)
+            ):
+                self.log.debug("Setting default unknown release notes url: %s", platform_notes_url)
+                detail.notes_url = platform_notes_url
+
+        if detail.source_platform == SOURCE_PLATFORM_GITHUB and detail.source_repo_url and detail.version is not None:
+            access_token: str | None = self.gh_cfg.access_token if self.gh_cfg else None
+            if access_token:
+                self.log.debug("Using configured bearer token (%s chars) for GitHub API", len(access_token))
             base_api = detail.source_repo_url.replace("https://github.com", "https://api.github.com/repos")
 
-            api_response: Response | None = fetch_url(f"{base_api}/releases/tags/{detail.version}")
+            api_response: Response | None = fetch_url(
+                f"{base_api}/releases/tags/{detail.version}", bearer_token=access_token, allow_stale=True
+            )
+            if api_response and api_response.status_code == 404:
+                # possible that source version doesn't match release gag
+                alt_api_response: Response | None = fetch_url(f"{base_api}/releases/latest", bearer_token=access_token)
+                if alt_api_response and alt_api_response.is_success:
+                    alt_api_results = httpx_json_content(alt_api_response, {})
+                    if alt_api_results and re.fullmatch(f"(V|v|r|R)?{detail.version}", alt_api_results.get("tag_name")):
+                        self.log.info(
+                            f"Matched {registry_info.name} {detail.version} to latest release {alt_api_results['tag_name']}"
+                        )
+                        api_response = alt_api_response
+                    elif alt_api_results:
+                        self.log.debug(
+                            "Failed to match %s release %s on GitHub, found tag %s for name %s published at %s",
+                            registry_info.name,
+                            detail.version,
+                            alt_api_results.get("tag_name"),
+                            alt_api_results.get("name"),
+                            alt_api_results.get("published_at"),
+                        )
+
             if api_response and api_response.is_success:
                 api_results: Any = httpx_json_content(api_response, {})
                 detail.summary = api_results.get("body")  # ty:ignore[possibly-missing-attribute]
                 reactions = api_results.get("reactions")  # ty:ignore[possibly-missing-attribute]
                 if reactions:
                     detail.net_score = reactions.get("+1", 0) - reactions.get("-1", 0)
+            elif api_response:
+                api_results = httpx_json_content(api_response, default={})
+                self.log.debug(
+                    "Failed to find %s release %s on GitHub, status %s, errors; %s",
+                    registry_info.name,
+                    detail.version,
+                    api_response.status_code,
+                    api_results.get("errors"),
+                )
             else:
                 self.log.debug(
                     "Failed to fetch GitHub release info",
@@ -523,14 +634,18 @@ class ContainerDistributionAPIVersionLookup(VersionLookup):
         self.api_stats = APIStatsCounter()
 
     def fetch_token(self, registry: str, image_name: str) -> str | None:
-        default_host: tuple[str, str, str, str] = (registry, registry, registry, TOKEN_URL_TEMPLATE)
+        default_host: tuple[str, str, str, str, None] = (registry, registry, registry, TOKEN_URL_TEMPLATE, None)
         auth_host: str | None = REGISTRIES.get(registry, default_host)[0]
         if auth_host is None:
             return None
 
         service: str = REGISTRIES.get(registry, default_host)[2]
-        url_template: str = REGISTRIES.get(registry, default_host)[3]
-        auth_url: str = url_template.format(auth_host=auth_host, image_name=image_name, service=service)
+        url_template: str | None = REGISTRIES.get(registry, default_host)[3]
+        auth_url: str | None = (
+            url_template.format(auth_host=auth_host, image_name=image_name, service=service) if url_template else None
+        )
+        if auth_url is None:
+            return None
         response: Response | None = fetch_url(
             auth_url, cache_ttl=self.cfg.token_cache_ttl, follow_redirects=True, api_stats_counter=self.api_stats
         )
@@ -737,7 +852,7 @@ class ContainerDistributionAPIVersionLookup(VersionLookup):
                     if index_digest:
                         result.image_digest = index_digest
                         result.short_digest = result.condense_digest(index_digest)
-                        log.debug("Setting %s image digest %s", result.name, result.short_digest)
+                        self.log.debug("Setting %s image digest %s", result.name, result.short_digest)
 
                     digest: str | None = m.get("digest")
                     media_type = m.get("mediaType")
@@ -757,7 +872,7 @@ class ContainerDistributionAPIVersionLookup(VersionLookup):
                             self.log.warning("Empty digest for %s %s %s", api_host, digest, media_type)
                         else:
                             result.repo_digest = result.condense_digest(digest, short=False)
-                            log.debug("Setting %s repo digest: %s", result.name, result.repo_digest)
+                            self.log.debug("Setting %s repo digest: %s", result.name, result.repo_digest)
 
                         if manifest.get("annotations"):
                             result.annotations.update(manifest.get("annotations", {}))
@@ -780,6 +895,7 @@ class ContainerDistributionAPIVersionLookup(VersionLookup):
                                     if config and "Labels" in config:
                                         result.annotations.update(config.get("Labels") or {})
                                     result.annotations.update(img_config.get("annotations") or {})
+                                    result.created = config.get("created") or config.get("Created")
                                 else:
                                     self.log.debug("No config found: %s", manifest)
                             except Exception as e:
@@ -872,5 +988,6 @@ class DockerClientVersionLookup(VersionLookup):
         labels: dict[str, str | float | int | bool | None] = cherrypick_annotations(local_image_info, result)
         result.custom = labels or {}
         result.version = cast("str|None", labels.get("image_version"))
+        result.created = cast("str|None", labels.get("image_created"))
         result.origin = "DOCKER_CLIENT"
         return result

updates2mqtt/model.py

@@ -1,3 +1,4 @@
+import datetime as dt
 import json
 import time
 from abc import abstractmethod
@@ -6,27 +7,40 @@ from threading import Event
 from typing import Any
 
 import structlog
+from tzlocal import get_localzone
 
 from updates2mqtt.config import NodeConfig, PublishPolicy, UpdatePolicy, VersionPolicy
-from updates2mqtt.helpers import timestamp
+from updates2mqtt.helpers import sanitize_name, timestamp
 
 
-class DiscoveryArtefactDetail:
-    """Provider specific detail"""
+class DiscoveryDetail:
+    def __init__(self) -> None:
+        self.captured: dt.datetime = dt.datetime.now(tz=get_localzone())
 
+    @abstractmethod
     def as_dict(self) -> dict[str, str | list | dict | bool | int | None]:
         return {}
 
+    def __str__(self) -> str:
+        """Log friendly"""
+        return ",".join(f"{k}:{v}" for k, v in self.as_dict().items())
+
 
-class DiscoveryInstallationDetail:
+class DiscoveryArtefactDetail(DiscoveryDetail):
     """Provider specific detail"""
 
-    @abstractmethod
-    def as_dict(self) -> dict[str, str | list | dict | bool | int | None]:
-        return {}
+    def __init__(self) -> None:
+        super().__init__()
+
+
+class DiscoveryInstallationDetail(DiscoveryDetail):
+    """Provider specific detail"""
+
+    def __init__(self) -> None:
+        super().__init__()
 
 
-class ReleaseDetail:
+class ReleaseDetail(DiscoveryDetail):
     """The artefact source details
 
     Note this may be an actual software package, or the source details of the wrapping of it
@@ -34,6 +48,7 @@ class ReleaseDetail:
     """
 
     def __init__(self, notes_url: str | None = None, summary: str | None = None) -> None:
+        super().__init__()
         self.source_platform: str | None = None
         self.source_repo_url: str | None = None
         self.source_url: str | None = None
@@ -45,8 +60,9 @@ class ReleaseDetail:
         self.summary: str | None = summary
         self.net_score: int | None = None
 
-    def as_dict(self) -> dict[str, str | None]:
+    def as_dict(self) -> dict[str, str | list | dict | bool | int | None]:
         return {
+            "captured": self.captured.isoformat(),
             "title": self.title,
             "version": self.version,
             "source_platform": self.source_platform,
@@ -94,7 +110,7 @@ class Discovery:
         self.provider: ReleaseProvider = provider
         self.source_type: str = provider.source_type
         self.session: str = session
-        self.name: str = name
+        self.name: str = sanitize_name(name)
         self.node: str = node
         self.entity_picture_url: str | None = entity_picture_url
         self.current_version: str | None = current_version